Prevent CMake from adding the library export directory the the executable RPath

Question

I have a small project with a shared library and an executable. I specify the rpath to be $ORIGIN/. so that it is the same folder as the executable. When I make the program, it adds /home/?????/test/lib/release to it. The question marks are my name so I censored it. This is the same as my library output path. Is there a way to prevent CMake from adding this to the rpath? Here are both of my CMakeLists.txt files.

project(test)
cmake_minimum_required(VERSION 3.1)
set(CMAKE_CXX_STANDARD 11)

set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin/)
set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib/)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_SOURCE_DIR}/bin/debug/)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/lib/)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/lib/release/)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${PROJECT_SOURCE_DIR}/lib/debug/)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/bin/release/)

set(CMAKE_CXX_FLAGS "-Wall")
set(CMAKE_CXX_FLAGS_DEBUG "-g -O0")
set(CMAKE_CXX_FLAGS_RELEASE "-O3")

if(NOT CMAKE_BUILD_TYPE)
    set(CMAKE_BUILD_TYPE Release)
endif()

set(CMAKE_BUILD_RPATH $ORIGIN/.)

add_subdirectory(${PROJECT_SOURCE_DIR}/src/)

and

add_library(testlib SHARED inlib.cpp)

add_executable(test main.cpp)

target_link_libraries(test testlib)