CMake ninja custom target used as pre-build command modifies the files but ninja see the change of the depndecies during next build

Question

I run some command before the actual build

add_custom_target("${PROJECT_BUILD_NUM_FILE}" 
    COMMAND ${PROJECT_BUILD_NUM_UPDATE} "-p" "-x" -f ${PROJECT_SOURCE_DIR}/${MAIN_PATH}/${BOARD_NAME}/${PROJECT_BUILD_NUM_FILE}
    )

add_dependencies(${EXECUTABLE_NAME} "${PROJECT_BUILD_NUM_FILE}" )

When I build first time (the project was build before)

>------ Build started: Project: CMakeLists, Configuration: Debug ------
  [1/1] cmd.exe /C "cd /D C:\Users\Piotr\git\gPIMS-EG\out\build\IoT-Debug && C:\Users\Piotr\git\gPIMS-EG\stm-libs\pc-exe\buildnum.exe -p -x -f C:/Users/Piotr/git/gPIMS-EG/src/EG/BuildID.h"
  Build number file used:C:/Users/Piotr/git/gPIMS-EG/src/EG/BuildID.h
  Build file updated: Build Number: 22 DateCode: 0x1d6d

Build succeeded.

BuildID.h file was modified and saved. Modification date is set correctly.

On the second build it behaves correctly

>------ Build started: Project: CMakeLists, Configuration: Debug ------
  [1/4] cmd.exe /C "cd /D C:\Users\Piotr\git\gPIMS-EG\out\build\IoT-Debug && C:\Users\Piotr\git\gPIMS-EG\stm-libs\pc-exe\buildnum.exe -p -x -f C:/Users/Piotr/git/gPIMS-EG/src/EG/BuildID.h"
  Build number file used:C:/Users/Piotr/git/gPIMS-EG/src/EG/BuildID.h
  Build file updated: Build Number: 23 DateCode: 0x1d6d
  [2/4] C:\PROGRA~2\Atollic\TRUEST~1.0\ARMTools\bin\AR097D~1.EXE -DALLOW_SOFTWARE_BKPTS=1 -DARM_MATH_CM4 -DBOOTLOADER_VERSION=1 -DCOMPILE_FOR_EG=1 -DDEBUGFILEWRITE=1 -DDEBUG_RUN_WITHOUT_CHECKS=1 -DFORCE_CUBE_USB -DHSE_VALUE=8000000UL -DSLOWSPIDEBUG=0 -DSTM32L -DSTM32L476xx -DSTM32L4xx -DUSE_EMBEDDED_PHY=1 -DUSE_FULL_ASSERT -DUSE_FULL_LL_DRIVER="" -DUSE_HAL_DRIVER -DUSE_SWOTRACE=1 -DUSE_USB_OTG_FS=1 -DUSE_USB_OTG_HS=1 -D_DEBUG=1 -D_GNU_SOURCE=1 -D__USE_SIMPLE_SWO=0 -I../../../src -I../../../src/EG -I../../../src/ZM -I../../../src/inc -I../../../stm-libs/inc -I../../../stm-libs/Drivers -I../../../stm-libs/Services -I../../../stm-libs/Services/Utils -I../../../stm-libs/RTOS/Source/include -I../../../stm-libs/RTOS/Source/portable/GCC/ARM_CM4F -I../../../stm-libs/Libraries/CMSIS/Include -I../../../stm-libs/Libraries/CMSIS/Device/ST/STM32L4xx/Include -I../../../stm-libs/Libraries/CryptoLib/Inc -I../../../stm-libs/Libraries/CryptoLib/Inc/RNG -I../../../stm-libs/drivers/USBD -I../../../stm-libs/Libraries/Middlewares/ST/STM32_USB_Device_Library/Core/Inc -I../../../stm-libs/Libraries/USB/Device/Class/CDC/Inc -I../../../stm-libs/Libraries/USB/Device/Class/MSC_HID/Inc -I../../../stm-libs/Libraries/USB/Device/Core/Inc -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7 -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7/arch -I../../../stm-libs/Libraries/lwip/src -I../../../stm-libs/Libraries/lwip/src/include -I../../../stm-libs/drivers/USBH -I../../../stm-libs/Libraries/USB/Host/Class/AUDIO/Inc -I../../../stm-libs/Libraries/USB/Host/Class/CDC/Inc -I../../../stm-libs/Libraries/USB/Host/Class/HID/Inc -I../../../stm-libs/Libraries/USB/Host/Class/MSC/Inc -I../../../stm-libs/Libraries/USB/Host/Class/MTP/Inc -I../../../stm-libs/Libraries/USB/Host/Core/Inc -I../../../stm-libs/Libraries/USB/Host/Class/Template/Inc -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7/FreeRTOS -I../../../stm-libs/Libraries/misc -I../../../stm-libs/Libraries/FatFs/src -I../../../stm-libs/Libraries/littleFS -I../../../stm-libs/Apps -I../../../stm-libsApps/SeqZM -I../../../stm-libs/Libraries/libc_e -I../../../stm-libs/Libraries -I../../../stm-hal-l4/Inc/Legacy -I../../../stm-hal-l4/Inc -I../../../stm-libs/Apps/Other -mthumb -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -g3 -T"C:/Users/Piotr/git/gPIMS-EG/src/EG/startup/EG.ld" -L"C:/Users/Piotr/git/gPIMS-EG/stm-libs/Libraries/CryptoLib" -lSTM32CryptographicV3.0.0_CM4_GCC_FPU -specs=nosys.specs -specs=nano.specs -Wl,-Map="EG.map" -static -Wl,-u,Reset_Handler  -Wl,--gc-sections -Wl,--defsym=malloc_getpagesize_P=0x80 -Wl,--start-group -lc -lm -Wl,--end-group -ffunction-sections -fdata-sections  -g -MD -MT CMakeFiles/GCEG-FW.dir/stm-libs/Apps/common/SC_Ver.c.obj -MF CMakeFiles\GCEG-FW.dir\stm-libs\Apps\common\SC_Ver.c.obj.d -o CMakeFiles/GCEG-FW.dir/stm-libs/Apps/common/SC_Ver.c.obj   -c ../../../stm-libs/Apps/common/SC_Ver.c
  In file included from ../../../stm-libs/Apps/common/SC_Ver.c:9:0:
C:\Users\Piotr\git\gPIMS-EG\stm-libs\Drivers\CAN-COB.h(267,2): warning GF07C7A44: #warning added to compile BP to change [-Wcpp]
   #warning added to compile BP to change
    ^~~~~~~
  [3/4] C:\PROGRA~2\Atollic\TRUEST~1.0\ARMTools\bin\AR097D~1.EXE -DALLOW_SOFTWARE_BKPTS=1 -DARM_MATH_CM4 -DBOOTLOADER_VERSION=1 -DCOMPILE_FOR_EG=1 -DDEBUGFILEWRITE=1 -DDEBUG_RUN_WITHOUT_CHECKS=1 -DFORCE_CUBE_USB -DHSE_VALUE=8000000UL -DSLOWSPIDEBUG=0 -DSTM32L -DSTM32L476xx -DSTM32L4xx -DUSE_EMBEDDED_PHY=1 -DUSE_FULL_ASSERT -DUSE_FULL_LL_DRIVER="" -DUSE_HAL_DRIVER -DUSE_SWOTRACE=1 -DUSE_USB_OTG_FS=1 -DUSE_USB_OTG_HS=1 -D_DEBUG=1 -D_GNU_SOURCE=1 -D__USE_SIMPLE_SWO=0 -I../../../src -I../../../src/EG -I../../../src/ZM -I../../../src/inc -I../../../stm-libs/inc -I../../../stm-libs/Drivers -I../../../stm-libs/Services -I../../../stm-libs/Services/Utils -I../../../stm-libs/RTOS/Source/include -I../../../stm-libs/RTOS/Source/portable/GCC/ARM_CM4F -I../../../stm-libs/Libraries/CMSIS/Include -I../../../stm-libs/Libraries/CMSIS/Device/ST/STM32L4xx/Include -I../../../stm-libs/Libraries/CryptoLib/Inc -I../../../stm-libs/Libraries/CryptoLib/Inc/RNG -I../../../stm-libs/drivers/USBD -I../../../stm-libs/Libraries/Middlewares/ST/STM32_USB_Device_Library/Core/Inc -I../../../stm-libs/Libraries/USB/Device/Class/CDC/Inc -I../../../stm-libs/Libraries/USB/Device/Class/MSC_HID/Inc -I../../../stm-libs/Libraries/USB/Device/Core/Inc -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7 -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7/arch -I../../../stm-libs/Libraries/lwip/src -I../../../stm-libs/Libraries/lwip/src/include -I../../../stm-libs/drivers/USBH -I../../../stm-libs/Libraries/USB/Host/Class/AUDIO/Inc -I../../../stm-libs/Libraries/USB/Host/Class/CDC/Inc -I../../../stm-libs/Libraries/USB/Host/Class/HID/Inc -I../../../stm-libs/Libraries/USB/Host/Class/MSC/Inc -I../../../stm-libs/Libraries/USB/Host/Class/MTP/Inc -I../../../stm-libs/Libraries/USB/Host/Core/Inc -I../../../stm-libs/Libraries/USB/Host/Class/Template/Inc -I../../../stm-libs/Libraries/lwip/ports/STM32F4x7/FreeRTOS -I../../../stm-libs/Libraries/misc -I../../../stm-libs/Libraries/FatFs/src -I../../../stm-libs/Libraries/littleFS -I../../../stm-libs/Apps -I../../../stm-libsApps/SeqZM -I../../../stm-libs/Libraries/libc_e -I../../../stm-libs/Libraries -I../../../stm-hal-l4/Inc/Legacy -I../../../stm-hal-l4/Inc -I../../../stm-libs/Apps/Other -mthumb -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -g3 -T"C:/Users/Piotr/git/gPIMS-EG/src/EG/startup/EG.ld" -L"C:/Users/Piotr/git/gPIMS-EG/stm-libs/Libraries/CryptoLib" -lSTM32CryptographicV3.0.0_CM4_GCC_FPU -specs=nosys.specs -specs=nano.specs -Wl,-Map="EG.map" -static -Wl,-u,Reset_Handler  -Wl,--gc-sections -Wl,--defsym=malloc_getpagesize_P=0x80 -Wl,--start-group -lc -lm -Wl,--end-group -ffunction-sections -fdata-sections  -g -MD -MT CMakeFiles/GCEG-FW.dir/stm-libs/Apps/WaveMC3/waveMC3SeqGUI.c.obj -MF CMakeFiles\GCEG-FW.dir\stm-libs\Apps\WaveMC3\waveMC3SeqGUI.c.obj.d -o CMakeFiles/GCEG-FW.dir/stm-libs/Apps/WaveMC3/waveMC3SeqGUI.c.obj   -c ../../../stm-libs/Apps/WaveMC3/waveMC3SeqGUI.c
  In file included from ../../../stm-libs/Drivers/CAN-G3M.h:31:0,
                   from ../../../stm-libs/Drivers/ioCmd.h:12,
                   from ../../../stm-libs/Services/srvCmd.h:9,
                   from ../../../src/EG/../ZM/zmServices.h:10,
                   from ../../../src/EG/ourServices.h:12,
                   from ../../../stm-libs/Apps/WaveMC3/waveMC3SeqGUI.c:5:
C:\Users\Piotr\git\gPIMS-EG\stm-libs\Drivers\CAN-COB.h(267,2): warning GF07C7A44: #warning added to compile BP to change [-Wcpp]
   #warning added to compile BP to change
    ^~~~~~~
  [4/4] cmd.exe /C "cd . && C:\PROGRA~2\Atollic\TRUEST~1.0\ARMTools\bin\AR097D~1.EXE -mthumb -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 -Og -g3 -T"C:/Users/Piotr/git/gPIMS-EG/src/EG/startup/EG.ld" -L"C:/Users/Piotr/git/gPIMS-EG/stm-libs/Libraries/CryptoLib" -lSTM32CryptographicV3.0.0_CM4_GCC_FPU -specs=nosys.specs -specs=nano.specs -Wl,-Map="EG.map" -static -Wl,-u,Reset_Handler  -Wl,--gc-sections -Wl,--defsym=malloc_getpagesize_P=0x80 -Wl,--start-group -lc -lm -Wl,--end-group -ffunction-sections -fdata-sections  -g   @CMakeFiles\GCEG-FW.rsp  -o GCEG-FW.elf  && cmd.exe /C "cd /D C:\Users\Piotr\git\gPIMS-EG\out\build\IoT-Debug && "C:\Program Files (x86)\Atollic\TrueSTUDIO for STM32 9.3.0\ARMTools\bin\arm-atollic-eabi-objcopy.exe" GCEG-FW.elf --output-target=binary GCEG-FW.bin -j .isr_vector -j .VERSION_INFO -j .text -j .rodata -j .ARM.extab -j .ARM -j .preinit_array -j .init_array -j .fini_array -j .data -j .RAM_VectorTable -j .RAM_DATAfunctions -j .RAM_functions -v && "C:\Program Files (x86)\Atollic\TrueSTUDIO for STM32 9.3.0\ARMTools\bin\arm-atollic-eabi-size.exe" GCEG-FW.elf --format=Berkeley""
  copy from `GCEG-FW.elf' [elf32-littlearm] to `GCEG-FW.bin' [binary]
     text      data     bss     dec     hex filename
   210428     36680   54020  301128   49848 GCEG-FW.elf

Build succeeded.

It looks like ninja is checking dependencies before the actual build starts and ignores meanwhile file changed. Is there any way to force cmake/ninja to recheck dependecies after custom target is build (ie prgram that thenges the build is run).

Answer 1

It looks like ninja is checking dependencies before the actual build starts and ignores meanwhile file change

Hey! Yes. Because DEPFILEs from C source files are generated at the same time as they are compiled, cmake has no way to know beforehand which file depends on which. It would have to do two passes - first to get dependencies and then to compile (which would be a nice feature, but actually hard to implement). It happens in one pass during compilation (-MD -MT flags), so cmake has no way of knowing that one file depends on that header. I also advise:

• Do not modify files in your source tree. Keep all changes in BINARY_DIR.
• Do not modify files. Generate new files. It's easier and generally, less state = less problems.

Try it such:

# Inside binary_dir
set(PROJECT_BUILD_NUM_FILE ${CMAKE_CURRENT_BINARY_DIR}/gen/buildid_gen.h)
# Add #include <buildid_gen.h> to your buildid.h
add_custom_command(
     OUTPUT ${PROJECT_BUILD_NUM_FILE}
     COMMAND ${PROJECT_BUILD_NUM_UPDATE} "-p" "-x" -f ${PROJECT_BUILD_NUM_FILE}
     DEPENDS ${PROJECT_BUILD_NUM_UPDATE}
)
add_custom_target(buildid_gen
    COMMENT "Yooohooo!"
    DEPENDS ${PROJECT_BUILD_NUM_FILE}
)
add_executable(${EXECUTABLE_NAME} ${PROJECT_BUILD_NUM_FILE} ${some_other_sources})
target_add_include_directories(${EXECTABLE_NAME} PUBLIC
     # add include directory, so that buildid_gen is found
     ${CMAKE_CURRENT_BINARY_DIR}/gen
)
# this *helps*
add_dependencies(${EXECUTABLE_NAME} buildid_gen)

Is there any way to force cmake/ninja to recheck dependecies after custom target is build (ie prgram that thenges the build is run).

The real proper way ™ would be to manually track all files that depend on the file (you could think of grep -l '#include <buildid_gen.h> to get the list files) and communicate to cmake the dependency relation:

add_source_file_properties(${sources_that_include_buildid_gen} PROPERTIES OBJECT_DEPENDS ${PROJECT_BUILD_NUM_FILE})

Tracking dependencies is easier if you use a runtime interface (ie. long get_build_id();) cause then only one .c file that implements the interface will depend on the generated file.
There are also some projects that are compiled in two passes because of such dependencies, you have to first cmake .. --target buildid_gen and then cmake .. --target actually_compile each time you compile.
Also maybe execute_process is more appropriate if the file is setup once per build.