Dynamically generated source file list for cmake

Question

We have a code generator that generates communication protocol definitions into multiple c++ files and want to switch our build system to use cmake. I have made a sample project to demonstrate the problem we're facing.

The code generator takes an input file and may generate hundreds of output files so enumerating them in CMakeLists.txt is not an option especially since the protocol changes during development. We neither want to use wildcards for this.

We did not find a way to make cmake regenerate the makefiles if one of it's input file changes, according to the documentation it is our belief that configure_file should be able to do that but it apparently fails at it's job.

We want the makefile for the main project (in main folder in example) to be regenerated by cmake whenever either the generator (in gen folder in example) code changes or the protocol definition file (in our example this is in.txt) is updated.

I have the following file structure:

src
├── CMakeLists.txt
├── gen
│   ├── CMakeLists.txt
│   └── gen.cpp
├── in.txt
└── main
    ├── CMakeLists.txt
    └── main.cpp

with the following contents: src/CMakeLists.txt:

cmake_minimum_required(VERSION 3.5.1)
project(cmake-config)

add_subdirectory(main)
add_subdirectory(gen)

add_dependencies(main gen run_gen)

gen/CMakeLists.txt:

cmake_minimum_required(VERSION 3.5.1)
project(gen)

add_executable(gen gen.cpp)

add_custom_target(
    run_gen
    COMMAND ./gen ${CMAKE_SOURCE_DIR}/in.txt
    COMMENT running gen
)

add_dependencies(run_gen gen)

gen/gen.cpp:

#include <fstream>
#include <string>

using namespace std;

int main(int argc, char *argv[]){
    ifstream inf(argv[1]);
    ofstream outf("input.txt");
    string word;
    while(inf >> word)
        outf << "set(SOURCES ${SOURCES} " << word << ")\n";

}

main/CMakeLists.txt:

cmake_minimum_required(VERSION 3.5.1)
project(main)

if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/../gen/input.txt)
    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/../gen/input.txt "") 
endif()

configure_file(${CMAKE_CURRENT_BINARY_DIR}/../gen/input.txt ${CMAKE_CURRENT_BINARY_DIR}/input.txt COPYONLY)

set(SOURCES main.cpp)

include(${CMAKE_CURRENT_BINARY_DIR}/input.txt)

message(${SOURCES})

add_executable(main ${SOURCES})

main/main.cpp:

int main(){}

gen generates input.txt using src/in.txt. input.txt contains a list of source files which I want to use to build main:

set(SOURCES ${SOURCES} a.cpp)
set(SOURCES ${SOURCES} b.cpp)
set(SOURCES ${SOURCES} c.cpp)

We are looking for a solution which does not require bootstrapping cmake or needing to rerun it every time the protocol changes during development. This is easily achievable but undesirable as it leads to problematic code which would end up using older protocol definitions if someone from the team does not realize the protocol file (or generator) has changed.

Answer 1

We have found a way to do it. The generator gen is put in a separate project, without changing the file structure. gen is built and executed when parsing the top level CMakeLists.txt file. When top level make is called, it modifies CMakeLists.txt, therefore the next time make is called cmake runs automatically. Project gen has a custom command which regenerates the output if in.txt changes.

src/CMakeLists.txt:

cmake_minimum_required(VERSION 3.5.1)
project(cmake-config)

if(NOT EXISTS ${CMAKE_BINARY_DIR}/gen/Makefile)
    execute_process(COMMAND cmake -B${CMAKE_BINARY_DIR}/gen -H${CMAKE_SOURCE_DIR}/gen)
endif()
execute_process(COMMAND make -C ${CMAKE_BINARY_DIR}/gen)

add_custom_target(touch_cmakelists.txt ALL
    COMMAND touch ${CMAKE_SOURCE_DIR}/CMakeLists.txt
)

add_subdirectory(main)

gen/CMakeLists.txt:

cmake_minimum_required(VERSION 3.5)
project(gen)

add_executable(gen gen.cpp)

add_custom_command(
    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/input.txt
    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/gen ${CMAKE_SOURCE_DIR}/../in.txt
    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/gen ${CMAKE_SOURCE_DIR}/../in.txt
    COMMENT "Running gen"
)

add_custom_target(run_generator ALL
    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input.txt
)

main/CMakeLists.txt:

project(main)

set(SOURCES main.cpp)

include(${CMAKE_CURRENT_BINARY_DIR}/../gen/input.txt)

add_executable(main ${SOURCES})

Answer 2

I think you can fully automate the process. You need a macro that can process a list of file argument. Here is an example that I use to process interface files for the Zeroc Ice middleware that become source files for the rest of the project. What you were also missing was the "MAIN_DEPENDENCY" argument of "ADD_CUSTOM_COMMAND".

macro(sliceCpp outfiles)
    foreach(i ${ARGN})
        GET_FILENAME_COMPONENT(name ${i} NAME)
        GET_FILENAME_COMPONENT(name2 ${i} NAME_WE)
        SET(infile ${CMAKE_CURRENT_SOURCE_DIR}/${name})
        SET(out1 ${CMAKE_CURRENT_BINARY_DIR}/${name2}.cpp)
        SET(out2 ${CMAKE_CURRENT_BINARY_DIR}/${name2}.h)
        ADD_CUSTOM_COMMAND(
            OUTPUT ${out1} ${out2}
            COMMAND ${ICE_SLICE2CPP}
            ARGS -I${ICE_SLICE_DIR} -I${CMAKE_CURRENT_SOURCE_DIR} --output-dir ${CMAKE_CURRENT_BINARY_DIR} ${infile}
            MAIN_DEPENDENCY ${infile}
        )
        SET(${outfiles} ${${outfiles}} ${out1} ${out2})
    endforeach(i)
    INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
endmacro(sliceCpp)

As a result, CMake will understand the dependencies and only re-build what is necessary. Other frameworks that generate cpp files for interfaces may behave similarly, you may have a look at ROS generate_messages() https://github.com/ros/ros_comm/blob/kinetic-devel/clients/roscpp/rosbuild/roscpp.cmake