After NDK22 upgrade the build fails with "cannot open crtbegin_so.o / crtend_so.o" error

Question

My app is using Android SDK and NDK. Until recently I was using SDK27 and NDK17c and all was working just fine. Then I decided to update my app and switched to SDK29 and NDK22 and the hell broke loose.

After taking into account the NDK directory structure changes and getting rid of all the remnants of GCC I was able to compile my .so library but during the build, there is this error:

19:38:58 **** Incremental Build of configuration android_ndk22_android-29-arm64-v8a for project MyProjectJNI ****
make all 
Building target: ../../../../../../bin/android_ndk22_android-29-arm64-v8a/libMyProjectJNI.so
Invoking: LLVM Clang C++ linker
clang++ -v -L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\bin" -L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29" -L"../../../../../../bin/android_ndk22_android-29-arm64-v8a/" -L"D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/" -L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sources/cxx-stl/llvm-libc++/libs/arm64-v8a" -L"D:\Projects\Libs\boost_1_67_0/lib/" -L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/aarch64-linux-android-4.9/prebuilt/windows-x86_64/lib/gcc/aarch64-linux-android/4.9.x" --sysroot="C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/" -target aarch64-linux-android29 -shared -std=c++11 -O1 --no-undefined -Wl,-verbose -o "../../../../../../bin/android_ndk22_android-29-arm64-v8a/libMyProjectJNI.so"  ./BaseJNI.bc ./CalibrationJNI.bc ./UtilsJNI.bc ./dllmain.bc   -lMyOtherLib -lboost_system -lboost_filesystem -lboost_thread -lboost_atomic -lboost_chrono -ljnigraphics -llog
clang version 3.7.0 (tags/RELEASE_370/final)
Target: aarch64--linux-android
Thread model: posix
 "C:\\Program Files\\LLVM\\bin\\clang++.exe" -cc1 -triple aarch64--linux-android29 -emit-obj -disable-free -main-file-name BaseJNI.bc -mrelocation-model pic -pic-level 1 -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -fuse-init-array -target-cpu generic -target-feature +neon -target-abi aapcs -backend-option -aarch64-fix-cortex-a53-835769=1 -v -dwarf-column-info -resource-dir "C:\\Program Files\\LLVM\\bin\\..\\lib\\clang\\3.7.0" -O1 -std=c++11 -fdebug-compilation-dir "D:\\Projects\\Development\\trunk\\Host\\dev\\src\\MyProjectJNI\\android_ndk22_android-29-arm64-v8a" -ferror-limit 19 -fmessage-length 0 -mstackrealign -fallow-half-arguments-and-returns -fno-signed-char -fobjc-runtime=gcc -fdiagnostics-show-option -o "C:\\Users\\UserPro\\AppData\\Local\\Temp\\BaseJNI-9a764d.o" -x ir ./BaseJNI.bc
clang -cc1 version 3.7.0 based upon LLVM 3.7.0 default target x86_64-w64-windows-gnu
 "C:\\Program Files\\LLVM\\bin\\clang++.exe" -cc1 -triple aarch64--linux-android29 -emit-obj -disable-free -main-file-name CalibrationJNI.bc -mrelocation-model pic -pic-level 1 -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -fuse-init-array -target-cpu generic -target-feature +neon -target-abi aapcs -backend-option -aarch64-fix-cortex-a53-835769=1 -v -dwarf-column-info -resource-dir "C:\\Program Files\\LLVM\\bin\\..\\lib\\clang\\3.7.0" -O1 -std=c++11 -fdebug-compilation-dir "D:\\Projects\\Development\\trunk\\Host\\dev\\src\\MyProjectJNI\\android_ndk22_android-29-arm64-v8a" -ferror-limit 19 -fmessage-length 0 -mstackrealign -fallow-half-arguments-and-returns -fno-signed-char -fobjc-runtime=gcc -fdiagnostics-show-option -o "C:\\Users\\UserPro\\AppData\\Local\\Temp\\CalibrationJNI-cc675c.o" -x ir ./CalibrationDataJNI.bc
clang -cc1 version 3.7.0 based upon LLVM 3.7.0 default target x86_64-w64-windows-gnu
 "C:\\Program Files\\LLVM\\bin\\clang++.exe" -cc1 -triple aarch64--linux-android29 -emit-obj -disable-free -main-file-name UtilsJNI.bc -mrelocation-model pic -pic-level 1 -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -fuse-init-array -target-cpu generic -target-feature +neon -target-abi aapcs -backend-option -aarch64-fix-cortex-a53-835769=1 -v -dwarf-column-info -resource-dir "C:\\Program Files\\LLVM\\bin\\..\\lib\\clang\\3.7.0" -O1 -std=c++11 -fdebug-compilation-dir "D:\\Projects\\Development\\trunk\\Host\\dev\\src\\MyProjectJNI\\android_ndk22_android-29-arm64-v8a" -ferror-limit 19 -fmessage-length 0 -mstackrealign -fallow-half-arguments-and-returns -fno-signed-char -fobjc-runtime=gcc -fdiagnostics-show-option -o "C:\\Users\\UserPro\\AppData\\Local\\Temp\\UtilsJNI-6102ad.o" -x ir ./CalibrationJNI
clang -cc1 version 3.7.0 based upon LLVM 3.7.0 default target x86_64-w64-windows-gnu
 "C:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670\\toolchains\\llvm\\prebuilt\\windows-x86_64\\aarch64-linux-android\\bin\\ld.exe" "--sysroot=C:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/" --eh-frame-hdr -m aarch64linux -shared -o ../../../../../../bin/android_ndk22_android-29-arm64-v8a/libMyProjectJNI.so crtbegin_so.o "-LC:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670\\toolchains\\llvm\\prebuilt\\windows-x86_64\\bin" "-LC:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670\\toolchains\\llvm\\prebuilt\\windows-x86_64\\sysroot\\usr\\lib\\aarch64-linux-android\\29" -L../../../../../../bin/android_ndk22_android-29-arm64-v8a/ "-LD:\\Projects\\Libs\\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/" "-LC:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670/sources/cxx-stl/llvm-libc++/libs/arm64-v8a" "-LD:\\Projects\\Libs\\boost_1_67_0/lib/" "-LC:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670/toolchains/aarch64-linux-android-4.9/prebuilt/windows-x86_64/lib/gcc/aarch64-linux-android/4.9.x" "-LC:\\Users\\UserPro\\AppData\\Local\\Android\\sdk\\ndk\\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot//usr/lib" --no-undefined -verbose "C:\\Users\\UserPro\\AppData\\Local\\Temp\\BaseFrameJNI-9a764d.o" "C:\\Users\\UserPro\\AppData\\Local\\Temp\\CalibrationJNI-cc675c.o" "C:\\Users\\UserPro\\AppData\\Local\\Temp\\UtilsJNI-09f434.o" "C:\\Users\\UserPro\\AppData\\Local\\Temp\\dllmain-63e5e7.o" -lMyOtherLib -lboost_system -lboost_filesystem -lboost_thread -lboost_atomic -lboost_chrono -ljnigraphics -llog -lstdc++ -lm -lgcc -ldl -lc -lgcc -ldl crtend_so.o
ld: crtbegin_so.o
ld: error: cannot open crtbegin_so.o: No such file or directory
ld: C:\Users\UserPro\AppData\Local\Temp\BaseJNI-9a764d.o
ld: C:\Users\UserPro\AppData\Local\Temp\CalibrationJNI-cc675c.o
ld: C:\Users\UserPro\AppData\Local\Temp\UtilsJNI-09f434.o
ld: C:\Users\UserPro\AppData\Local\Temp\dllmain-63e5e7.o
ld: ../../../../../../bin/android_ndk22_android-29-arm64-v8a/liblMyOtherLib.so
ld: D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/libboost_system.a
ld: D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/libboost_filesystem.a
ld: D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/libboost_thread.a
ld: D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/libboost_atomic.a
ld: D:\Projects\Libs\boost_1_67_0/android_ndk22_android-29-arm64-v8a/lib/libboost_chrono.a
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libjnigraphics.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\liblog.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libstdc++.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libm.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/aarch64-linux-android-4.9/prebuilt/windows-x86_64/lib/gcc/aarch64-linux-android/4.9.x\libgcc.a
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libdl.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libc.so
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/aarch64-linux-android-4.9/prebuilt/windows-x86_64/lib/gcc/aarch64-linux-android/4.9.x\libgcc.a
ld: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\libdl.so
ld: crtend_so.o
ld: error: cannot open crtend_so.o: No such file or directory
clang++.exe: error: linker command failed with exit code 1 (use -v to see invocation)
make: *** [makefile:53: ../../../../../../bin/android_ndk22_android-29-arm64-v8a/libMyProjectJNI.so] Error 1

The files that the linker cannot find, crtbegin_so.o and crtend_so.o, are definitely here: C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29\

I know that the major change between ndk17 and ndk22 is that

"Clang, binutils, the sysroot, and other toolchain pieces are now all installed to $NDK/toolchains/llvm/prebuilt/ and Clang will automatically find them. "

It looks to me like the linker was able to automatically find .so libraries but cannot find .o objects and is ignoring all paths I supply. Has anybody else encountered this problem? Any suggestions on how to solve it?

UPDATE: I respectfully disagree with the editing of the question. The switching from ndk17 to ndk22 includes switching from GCC to LLVM and probably the problem lies somewhere there.

UPDATE2 My clang++ command-line options for compiler:

-DNDEBUG 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sources/cxx-stl/llvm-libc++/include" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sysroot/usr/include/aarch64-linux-android" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sources/android/cpufeatures" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sysroot/usr/include" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/usr/include" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/usr/include/x86_64-linux-android" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/usr/lib/aarch64-linux-android/29" 
-I"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22\AppData\Local\Android\sdk\ndk\22.1.7171670/sysroot/usr/include/arm-linux-androideabi" 
-I../../../include -I../../CommonUtilities 
-I"$(BOOST_PATH)" -I../../../../../../Common/include -O2 -emit-llvm -g -Wall -c -fmessage-length=0 -target aarch64-linux-android -std=c++0x

and for linker:

-v 
-L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\bin" 
-L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670\toolchains\llvm\prebuilt\windows-x86_64\sysroot\usr\lib\aarch64-linux-android\29" 
-L"../../../../../../bin/android_ndk23_android-29-arm64-v8a/" 
-L"D:\Projects\Libs\boost_1_67_0/android_ndk23_android-29-arm64-v8a/lib/" 
-L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/sources/cxx-stl/llvm-libc++/libs/arm64-v8a" 
-L"D:\Projects\Libs\boost_1_67_0/lib/" 
-L"C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/aarch64-linux-android-4.9/prebuilt/windows-x86_64/lib/gcc/aarch64-linux-android/4.9.x" 
--sysroot="C:\Users\UserPro\AppData\Local\Android\sdk\ndk\22.1.7171670/toolchains/llvm/prebuilt/windows-x86_64/sysroot/" 
-target aarch64-linux-android -shared -std=c++11 -O1 --no-undefined -Wl,-verbose

Answer 1

I ended up using -nostartfiles option. Worked for me