cmake not adding needed -fopenmp to linker command

Question

I'm building a program which is mostly Fortran, but includes a C++ interface library. This is a cmake project, and you can even find the specific branch on github here. I'm building on Linux using cmake 3.25.1

My issue is that there is a cmake option to build with OpenMP. The full CMakeLists.txt for the C++ library is shown below:

#
# Copyright 2016 National Renewable Energy Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0")
  cmake_policy(SET CMP0074 NEW)
endif()

set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(MPI REQUIRED)
find_package(LibXml2 REQUIRED)
find_package(ZLIB REQUIRED)
find_package(HDF5 REQUIRED COMPONENTS C HL)
find_package(yaml-cpp REQUIRED)

include_directories(${YAML_CPP_INCLUDE_DIRS})
include_directories(${HDF5_INCLUDES})
include_directories(${HDF5_INCLUDE_DIR})
include_directories(${ZLIB_INCLUDES})
include_directories(${LIBXML2_INCLUDE_DIR})
include_directories(${CMAKE_SOURCE_DIR}/modules/openfast-library/src/)
include_directories(${CMAKE_BINARY_DIR}/modules/openfoam/)
include_directories(${CMAKE_BINARY_DIR}/modules/supercontroller/)
include_directories(${MPI_INCLUDE_PATH})

add_library(openfastcpplib
  src/OpenFAST.cpp 
  src/SC.cpp
  $<TARGET_OBJECTS:openfastlib_common_obj>
  $<TARGET_OBJECTS:openfast_prelib_obj>
  $<TARGET_OBJECTS:maplib_obj>
  $<TARGET_OBJECTS:nwtcbaselib_obj> 
  $<TARGET_OBJECTS:nwtcsyslib_obj>
  $<TARGET_OBJECTS:nwtclibs_obj>
  $<TARGET_OBJECTS:ifwlib_obj>
  $<TARGET_OBJECTS:fvwlib_obj>
  $<TARGET_OBJECTS:uaaerolib_obj>
  $<TARGET_OBJECTS:afinfolib_obj>
  $<TARGET_OBJECTS:aeroacoustics_obj>
  $<TARGET_OBJECTS:aerodynlib_obj>
  $<TARGET_OBJECTS:aerodyn14lib_obj>
  $<TARGET_OBJECTS:servodynlib_obj>
  $<TARGET_OBJECTS:elastodynlib_obj>
  $<TARGET_OBJECTS:beamdynlib_obj>
  $<TARGET_OBJECTS:subdynlib_obj>
  $<TARGET_OBJECTS:hydrodynlib_obj>
  $<TARGET_OBJECTS:orcaflexlib_obj>
  $<TARGET_OBJECTS:extptfm_mckflib_obj>
  $<TARGET_OBJECTS:openfoamtypeslib_obj>
  $<TARGET_OBJECTS:foamfastlib_obj>
  $<TARGET_OBJECTS:scdataextypeslib_obj>
  $<TARGET_OBJECTS:scdataexlib_obj>
  $<TARGET_OBJECTS:feamlib_obj>
  $<TARGET_OBJECTS:moordynlib_obj>
  $<TARGET_OBJECTS:icedynlib_obj>
  $<TARGET_OBJECTS:icefloelib_obj>
  $<TARGET_OBJECTS:openfast_postlib_obj>
  $<TARGET_OBJECTS:versioninfolib_obj>
)
set_property(TARGET openfastcpplib PROPERTY POSITION_INDEPENDENT_CODE ON)
target_link_libraries(openfastcpplib
openfast_prelib_obj
maplib_obj
nwtclibs_obj
nwtcsyslib_obj
nwtcbaselib_obj
ifwlib
fvwlib
uaaerolib
afinfolib
aeroacoustics
aerodynlib
aerodyn14lib
servodynlib
elastodynlib
beamdynlib
subdynlib
hydrodynlib
orcaflexlib
extptfm_mckflib
openfoamtypeslib
foamfastlib
scdataextypeslib
scdataexlib
feamlib
moordynlib
icedynlib
icefloelib
openfast_postlib
versioninfolib
${HDF5_C_LIBRARIES}
${HDF5_HL_LIBRARIES}
${ZLIB_LIBRARIES}
${LIBXML2_LIBRARIES}
${MPI_LIBRARIES}
${CMAKE_DL_LIBS}
)

add_executable(openfastcpp src/FAST_Prog.cpp)
target_link_libraries(openfastcpp openfastcpplib
  ${MPI_LIBRARIES}
  ${YAML_CPP_LIBRARIES}
  ${HDF5_C_LIBRARIES}
  ${HDF5_HL_LIBRARIES}
  ${ZLIB_LIBRARIES}
  ${LIBXML2_LIBRARIES}
  ${CMAKE_DL_LIBS})

if (OPENMP)
  find_package(OpenMP)
  if(OpenMP_CXX_FOUND)
    target_link_libraries(openfastcpplib OpenMP::OpenMP_CXX)
    target_link_libraries(openfastcpp OpenMP::OpenMP_CXX)
  endif()
endif(OPENMP)

if(MPI_COMPILE_FLAGS)
  set_target_properties(openfastcpp PROPERTIES
    COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
endif(MPI_COMPILE_FLAGS)

if(MPI_LINK_FLAGS)
  set_target_properties(openfastcpp PROPERTIES
    LINK_FLAGS "${MPI_LINK_FLAGS}")
endif(MPI_LINK_FLAGS)

set_property(TARGET openfastcpp PROPERTY LINKER_LANGUAGE CXX)

install(TARGETS openfastcpplib
  EXPORT "${CMAKE_PROJECT_NAME}Libraries"
  RUNTIME DESTINATION lib
  ARCHIVE DESTINATION lib
  LIBRARY DESTINATION lib)

install(FILES
  src/OpenFAST.H src/SC.h
  DESTINATION include)

install(TARGETS openfastcpp
  RUNTIME DESTINATION bin)

It uses the modern OpenMP cmake invocations:

if (OPENMP)
  find_package(OpenMP)
  if(OpenMP_CXX_FOUND)
    target_link_libraries(openfastcpplib OpenMP::OpenMP_CXX)
    target_link_libraries(openfastcpp OpenMP::OpenMP_CXX)
  endif()
endif(OPENMP)

But when I build the program, I receive an error like this:

[ 95%] Linking CXX executable openfast_cpp
cd /home/rcrozier/build/openfast-reos-git/glue-codes/openfast/glue-codes/openfast && /snap/cmake/1210/bin/cmake -E cmake_link_script CMakeFiles/openfast_cpp.dir/link.txt --verbose=1
/bin/c++  -fpic -O3 -DNDEBUG -rdynamic CMakeFiles/openfast_cpp.dir/src/FAST_Prog.cpp.o CMakeFiles/openfast_cpp.dir/src/FastLibAPI.cpp.o -o openfast_cpp  -Wl,-rpath,:::::::::::::: ../../modules/openfast-library/libopenfastlib.a ../../dependencies/src/lapack-build/lib/liblapack.a ../../dependencies/src/lapack-build/lib/libblas.a -ldl -lgfortran -lquadmath 
/bin/ld: ../../modules/openfast-library/libopenfastlib.a(VersionInfo.f90.o): in function `__versioninfo_MOD_dispcompileruntimeinfo._omp_fn.0':
VersionInfo.f90:(.text+0xf): undefined reference to `omp_get_thread_num_'
/bin/ld: VersionInfo.f90:(.text+0x31): undefined reference to `omp_get_num_threads_'
/bin/ld: VersionInfo.f90:(.text+0xf4): undefined reference to `omp_get_max_threads_'
/bin/ld: ../../modules/openfast-library/libopenfastlib.a(VersionInfo.f90.o): in function `__versioninfo_MOD_dispcompileruntimeinfo':
VersionInfo.f90:(.text+0x847): undefined reference to `GOMP_parallel'

If I copy the command, but manually add the -fopenmp option, it seems to work, there is no error message

cd /home/rcrozier/build/openfast-reos-git/glue-codes/openfast/glue-codes/openfast && /snap/cmake/1210/bin/cmake -E cmake_link_script CMakeFiles/openfast_cpp.dir/link.txt --verbose=1
/bin/c++  -fpic -O3 -DNDEBUG -rdynamic CMakeFiles/openfast_cpp.dir/src/FAST_Prog.cpp.o CMakeFiles/openfast_cpp.dir/src/FastLibAPI.cpp.o -o openfast_cpp  -Wl,-rpath,:::::::::::::: ../../modules/openfast-library/libopenfastlib.a ../../dependencies/src/lapack-build/lib/liblapack.a ../../dependencies/src/lapack-build/lib/libblas.a -ldl -lgfortran -lquadmath -fopenmp

So my question is, assuming the -fopenmp is needed, and there is not some other issue I don't understand, how do I best get the -fopenmp option added by cmake?

EDIT with further information

I forgot to mention I am building static libraries, using -DBUILD_SHARED_LIBS=OFF.

The cmake command I am using to generate is the following:

cmake -LH /home/rcrozier/src/openfast-reos-git -DBUILD_FASTFARM=ON -DBUILD_OPENFAST_CPP_API=ON -DBUILD_OPENFAST_SIMULINK_API=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DDOUBLE_PRECISION=ON -DOPENMP=ON -DUSE_LOCAL_STATIC_LAPACK=ON

I build out of tree in a different directory from the source directory.

I have attempted the following fixes:

1. I added the REQUIRED potion to find_package to ensure OpenMP is available (it is)

2. adding -fopenmp to target_link_libraries (the cmake docs state that link flags can be passed this way)

   target_link_libraries(openfastcpplib "-fopenmp") target_link_libraries(openfastcpp "-fopenmp")

3. adding -fopenmp to target_link_options

   target_link_options(openfastcpplib PUBLIC "-fopenmp") target_link_options(openfastcpp PUBLIC "-fopenmp")

4. using set_property

   set_property(TARGET openfastcpp APPEND PROPERTY LINK_FLAGS "-fopenmp") set_property(TARGET openfastcpp APPEND PROPERTY COMPILE_FLAGS "-fopenmp")

5. using STATIC_LIBRARY_OPTIONS as I am building with shared libraries disabled

   set_property (TARGET openfastcpp APPEND PROPERTY STATIC_LIBRARY_OPTIONS "-fopenmp")

I also used message to see if the flag is ever added in the following places

message(CMAKE_CXX_LINK_FLAGS="${CMAKE_CXX_LINK_FLAGS}")
message(CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}")
message(CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}")

It is not

6. using target_link_options only

   target_link_options(openfastcpp PUBLIC "-fopenmp")

7 using target_link_options and target_compile_options

target_compile_options(openfastcpp PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-std=c++14>)
target_link_options(openfastcpp PUBLIC "-fopenmp")

TRYING NOW WILL EDIT Q WITH RESULTS

Not one of the above has added linker commands where necessary.

I also noticed that there was a cmake github issue which seems to describe a similar problem here. Unfortuantely the discussion is rather advanced I'm not sure if it is supposed to be fixed in cmake 3.25.1, or how to implement a workaround in my case in the same way described in the link.

Answer 1

First I will assume that you generate your cmake files with the -DOPENMP project as the code itself suggests it's needed and even the documentation here mentions it in the section OpenFAST CMake options.

Secondly I would make sure as dreschejrm pointed out, that OpenMP is trully linked against and found correctly, I would assume this option is part of the OpenMP alias that you are using (i.e. OpenMP::OpenMP_CXX). But it might not (maybe you are using an older version or maybe it's something else very specific)

If you've checked all of the above and you are in the same situation then read further:

So to answer your question on how to quickly add the option to your build, you have a minimum of 2 ways on how to do this and one of them you are already using in the provided CMakeLists.txt.

1. set_target_properties() - Which requires you to specify which property (e.g. LINK_FLAGS, COMPILE_FLAGS, etc.) You are for example using it in here:

if(MPI_COMPILE_FLAGS)
  set_target_properties(openfastcpp PROPERTIES
    COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
endif(MPI_COMPILE_FLAGS)

if(MPI_LINK_FLAGS)
  set_target_properties(openfastcpp PROPERTIES
    LINK_FLAGS "${MPI_LINK_FLAGS}")
endif(MPI_LINK_FLAGS)

2. target_compile_options() or target_link_options(), e.g. (exempli gratia):

target_compile_options(openfastcpp PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-std=c++14>)

target_link_options(openfastcpp PUBLIC "-fopenmp")

Please note that the above are just examples i.e. DO NOT JUST COPY PASTE THEM. The target_link_options/target_compile_options are guaranteed to just add an option (and not replace the already specified ones). I am not 100% sure about the set_target_properties as I've never used it for COMPILE_FLAGS/LINK_FLAGS.

EDIT: I will state again that the above was just an example. You need to (most likely) add the linker options to openfastlib target as well.

In regards to the github issue they are in essence doing exactly as I'm telling you here, i.e. manually adding the linker option. If you wish to use their solution you need to update your FindOpenMP.cmake file with the commit that one of the people posted there. It may seem "more complicated" than what I'm explaining here but it is the exact same.

set_target_properties(OpenMP::OpenMP_${LANG} PROPERTIES
      INTERFACE_COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:${LANG}>:${_OpenMP_${LANG}_OPTIONS}>"
      INTERFACE_LINK_OPTIONS "$<$<COMPILE_LANGUAGE:${LANG}>:${_OpenMP_${LANG}_OPTIONS}>")

To explain the above snippet: ${LANG} is just a variable representing one of the languages supported by this .cmake file. In your case it expands to CXX. The "$<$<COMPILE_LANGUAGE:${LANG}>:${_OpenMP_${LANG}_OPTIONS}>" is a generator expression that is dependent on the selected language. I.e. if COMPILE_LANGUAGE is CXX it will use the specified variable ${_OpenMP_CXX_OPTIONS} in place, etc.

They populate the specified targets which are: INTERFACE_LINK_OPTIONS and INTERFACE_COMPILE_OPTIONS. That means that instead of the PUBLIC keyword in target_link_options you should use INTERFACE.

Also I will repeat this again, you need to specify these options for both openfast and openfastlib targets. i.e. you need the following lines:

target_link_options(openfastcpplib INTERFACE "-fopenmp")
target_link_options(openfastcpp INTERFACE "-fopenmp")