Different behavior in Jam and CMake when trying to check for `-Wl,--no-undefined` support with sanitizers

[BobIsOnFire]

Description

Building boost-filesystem with clang+sanitizers, I can see that the build system fails on trying to build the check executable, and detects no support for -Wl,--no-undefined for current build. After that, the actual build succeeds ok and tests pass:

$ cat ~/user-config.jam 
using clang : : /opt/llvm-20/bin/clang++ : <triple>none ;

$ ./b2 -j 16 toolset=clang cxxstd=23 variant=debug cxxflags="-fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1" linkflags="-fsanitize=address -fuse-ld=lld" libs/filesystem/test -d+2 | tee build.log 
...

$ echo $?
0

$ cat bin.v2/config.log
...
clang-linux.link.dll bin.v2/libs/filesystem/config/clang-linux-20/debug/x86_64/cxxstd-23-iso/threading-multi/visibility-hidden/libha
s_linkflag_no_undefined.so.1.92.0

    "/opt/llvm-20/bin/clang++"      -o "bin.v2/libs/filesystem/config/clang-linux-20/debug/x86_64/cxxstd-23-iso/threading-multi/visi
bility-hidden/libhas_linkflag_no_undefined.so.1.92.0" -Wl,-soname -Wl,"libhas_linkflag_no_undefined.so.1.92.0" -shared -Wl,--start-g
roup "bin.v2/libs/filesystem/config/clang-linux-20/debug/x86_64/cxxstd-23-iso/threading-multi/visibility-hidden/has_linkflag_no_unde
fined.o"  -Wl,-Bstatic  -Wl,-Bdynamic -lrt -Wl,--end-group -fPIC -pthread -std=c++23 -g -fvisibility=hidden -fvisibility-inlines-hid
den -m64 -Wl,--no-undefined -fsanitize=address -fuse-ld=lld

ld.lld: error: undefined symbol: __asan_init
>>> referenced by has_linkflag_no_undefined.cpp
>>>               bin.v2/libs/filesystem/config/clang-linux-20/debug/x86_64/cxxstd-23-iso/threading-multi/visibility-hidden/has_linkflag_no_undefined.o:(asan.module_ctor)

ld.lld: error: undefined symbol: __asan_version_mismatch_check_v8
...

$ grep no-undefined build.log
    - has -Wl,--no-undefined   : no [2]

Doing the same with CMake -- the check succeeds, but the actual binary build fails with undefined symbol errors:

$ cmake -S . -B build-cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_COMPILER=/opt/llvm-20/bin/clang++ -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1" -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address -fuse-ld=lld" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fuse-ld=lld"
...
-- Performing Test BOOST_FILESYSTEM_HAS_LINKFLAG_NO_UNDEFINED
-- Performing Test BOOST_FILESYSTEM_HAS_LINKFLAG_NO_UNDEFINED - Success
...

$ tail build-cmake/CMakeFiles/CMakeConfigureLog.yaml 
        Linking CXX executable cmTC_7ec5d
        /export/home/nikita.akatiev/tb/toolchain/x86_64-unknown-linux/bin/cmake -E cmake_link_script CMakeFiles/cmTC_7ec5d.dir/link.txt --verbose=1
        /opt/llvm-20/bin/clang++ -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1  -fsanitize=address -fuse-ld=lld  CMakeFiles/cmTC_7ec5d.dir/src.cxx.o -o cmTC_7ec5d  -Wl,--no-undefined
        gmake[1]: Leaving directory '/export/home/nikita.akatiev/rmme/boost-root/build-cmake/CMakeFiles/CMakeScratch/TryCompile-1ryjmr'
        
      exitCode: 0
...

$ cmake --build build-cmake/ --target boost_filesystem -j16 --verbose |& build-cmake.log
...
[100%] Linking CXX shared library ../../stage/lib/libboost_filesystem.so
cd /export/home/nikita.akatiev/rmme/boost-root/build-cmake/libs/filesystem && /export/home/nikita.akatiev/tb/toolchain/x86_64-unknown-linux/bin/cmake -E cmake_link_script CMakeFiles/boost_filesystem.dir/link.txt --verbose=1
ld.lld: error: undefined symbol: __asan_report_load1
>>> referenced by asan_rtl_static.cpp:25 (/rpmbuild/BUILD/tb-llvm20-20.1.7/src/compiler-rt/lib/asan/asan_rtl_static.cpp:25)
>>>               asan_rtl_static.cpp.o:(__asan_report_load1_asm) in archive /opt/llvm-20/lib64/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan_static.a
>>> referenced by path.cpp:233 (/export/home/nikita.akatiev/rmme/boost-root/libs/filesystem/src/path.cpp:233)
>>>               CMakeFiles/boost_filesystem.dir/src/path.cpp.o:(boost::filesystem::detail::path_algorithms::lexically_normal_v3(boost::filesystem::path const&))
>>> referenced by path.cpp:265 (/export/home/nikita.akatiev/rmme/boost-root/libs/filesystem/src/path.cpp:265)
>>>               CMakeFiles/boost_filesystem.dir/src/path.cpp.o:(boost::filesystem::detail::path_algorithms::lexically_normal_v3(boost::filesystem::path const&))
>>> referenced 273 more times

ld.lld: error: undefined symbol: __asan_report_store1
...

$ echo $?
2

I can see in the logs that Jam is building the shared library from has_linkflag_no_undefined.cpp while CMake builds an executable from it. Locally I've applied following patch to CMakeLists.txt so that CMake also builds shared lib here and now my CMake build succeeds:

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4619ee9..b38f485 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -90,11 +90,11 @@ target_compile_features(boost_filesystem PUBLIC
 
 get_target_property(BOOST_FILESYSTEM_TARGET_TYPE boost_filesystem TYPE)
 if(BOOST_FILESYSTEM_TARGET_TYPE STREQUAL "SHARED_LIBRARY")
-    set(CMAKE_REQUIRED_LIBRARIES "-Wl,--no-undefined")
+    set(CMAKE_REQUIRED_LIBRARIES "-shared" "-Wl,--no-undefined")
     check_cxx_source_compiles("#include <${CMAKE_CURRENT_SOURCE_DIR}/config/has_linkflag_no_undefined.cpp>" BOOST_FILESYSTEM_HAS_LINKFLAG_NO_UNDEFINED)
     unset(CMAKE_REQUIRED_LIBRARIES)
     if(NOT BOOST_FILESYSTEM_HAS_LINKFLAG_NO_UNDEFINED)
-        set(CMAKE_REQUIRED_LIBRARIES "-Wl,-undefined,error")
+        set(CMAKE_REQUIRED_LIBRARIES "-shared" "-Wl,-undefined,error")
         check_cxx_source_compiles("#include <${CMAKE_CURRENT_SOURCE_DIR}/config/has_linkflag_no_undefined.cpp>" BOOST_FILESYSTEM_HAS_LINKFLAG_UNDEFINED_ERROR)
         unset(CMAKE_REQUIRED_LIBRARIES)
     endif()

Environment

Boost 1.91.0, clang 20.1.8, libstdc++ 11.5.0, CMake 3.31.7

Expected behavior

Both CMake and Jam should make the same decision regarding -Wl,--no-undefined flag at configure stage -- i.e. both build systems should either succeed the check and add the flag to actual library build, or fail the check and skip adding the flag.

[Lastique]

The issue here is not that -Wl,--no-undefined is falsely detected as supported - it is supported in both cases. The linking error is exactly the effect of that flag. The issue is that presumably ASAN requires explicit linking with libasan. See here for a similar issue with UBSAN. I.e. both checks should pass on Linux. Try adding -lasan to your link flags.

I'm not sure if it is possible to remove the sanitizer flags from the command line when compiling the config checks to test specifically for -Wl,--no-undefined and no other things to avoid false negatives. It doesn't seem like check_cxx_source_compiles supports that.

[Lastique]

Duplicates #297.