Index: zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh +++ zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh @@ -11,35 +11,32 @@ PLATFORM=`uname` export PATH="/usr/local/bin:$PATH" +STAGE1_DIR=llvm_build0 +STAGE2_ASAN_DIR=llvm_build_asan +STAGE2_MSAN_DIR=llvm_build_msan +STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan +STAGE2_UBSAN_DIR=llvm_build_ubsan +STAGE3_ASAN_DIR=llvm_build2_asan +STAGE3_MSAN_DIR=llvm_build2_msan +MAKE_JOBS=${MAX_MAKE_JOBS:-16} +LLVM=$ROOT/llvm +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3" + if [ "$BUILDBOT_CLOBBER" != "" ]; then echo @@@BUILD_STEP clobber@@@ rm -rf llvm - rm -rf llvm_build0 -fi - -MEMORY_SANITIZER_KIND="Memory" -BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} -if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then - MEMORY_SANITIZER_KIND="MemoryWithOrigins" + rm -rf ${STAGE1_DIR} fi # CMake does not notice that the compiler itself has changed. # Anyway, incremental builds of stage2 and stage3 compilers don't make sense. # Clobber the build trees. -rm -rf libcxx_build_msan -rm -rf llvm_build_msan -rm -rf llvm_build2_msan -rm -rf llvm_build_asan -rm -rf llvm_build2_asan -rm -rf llvm_build_ubsan - -MAKE_JOBS=${MAX_MAKE_JOBS:-16} -LLVM=$ROOT/llvm - -type -a gcc -type -a g++ -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3" -CMAKE_STAGE1_OPTIONS="${CMAKE_COMMON_OPTIONS}" +rm -rf ${STAGE2_LIBCXX_MSAN_DIR} +rm -rf ${STAGE2_MSAN_DIR} +rm -rf ${STAGE3_MSAN_DIR} +rm -rf ${STAGE2_ASAN_DIR} +rm -rf ${STAGE3_ASAN_DIR} +rm -rf ${STAGE2_UBSAN_DIR} echo @@@BUILD_STEP update@@@ buildbot_update @@ -47,130 +44,49 @@ # Stage 1 echo @@@BUILD_STEP build stage1 clang@@@ -if [ ! -d llvm_build0 ]; then - mkdir llvm_build0 -fi -# Build cxx/cxxabi to fool the compiler check in MSan stage 2. -(cd llvm_build0 && cmake ${CMAKE_STAGE1_OPTIONS} $LLVM && \ - ninja clang compiler-rt llvm-symbolizer) || \ - echo @@@STEP_FAILURE@@@ -CLANG_PATH=$ROOT/llvm_build0/bin -CMAKE_STAGE2_COMMON_OPTIONS="\ - ${CMAKE_COMMON_OPTIONS} \ - -DCMAKE_C_COMPILER=${CLANG_PATH}/clang \ - -DCMAKE_CXX_COMPILER=${CLANG_PATH}/clang++ \ - " -LLVM_SYMBOLIZER_PATH=${CLANG_PATH}/llvm-symbolizer -export ASAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH} -export MSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH} +build_stage1_clang # Stage 2 / Memory Sanitizer -echo @@@BUILD_STEP build libcxx/msan@@@ -if [ ! -d libcxx_build_msan ]; then - mkdir libcxx_build_msan -fi - -(cd libcxx_build_msan && \ - cmake \ - ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \ - $LLVM && \ - ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP build clang/msan@@@ -if [ ! -d llvm_build_msan ]; then - mkdir llvm_build_msan -fi - -MSAN_LDFLAGS="-lc++abi -Wl,--rpath=${ROOT}/libcxx_build_msan/lib -L${ROOT}/libcxx_build_msan/lib" -# See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264 -CMAKE_BUG_WORKAROUND_CFLAGS="$MSAN_LDFLAGS -fsanitize=memory -w" -MSAN_CFLAGS="-I${ROOT}/libcxx_build_msan/include -I${ROOT}/libcxx_build_msan/include/c++/v1 $CMAKE_BUG_WORKAROUND_CFLAGS" - -(cd llvm_build_msan && \ - cmake ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \ - -DLLVM_ENABLE_LIBCXX=ON \ - -DCMAKE_C_FLAGS="${MSAN_CFLAGS}" \ - -DCMAKE_CXX_FLAGS="${MSAN_CFLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="${MSAN_LDFLAGS}" \ - $LLVM && \ - ninja clang) || echo @@@STEP_FAILURE@@@ - -echo @@@BUILD_STEP check-llvm msan@@@ - -(cd llvm_build_msan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@ - -echo @@@BUILD_STEP check-clang msan@@@ - -(cd llvm_build_msan && ninja check-clang) || echo @@@STEP_FAILURE@@@ +build_stage2_msan +check_stage2_msan # Stage 3 / MemorySanitizer echo @@@BUILD_STEP build stage3/msan clang@@@ -if [ ! -d llvm_build2_msan ]; then - mkdir llvm_build2_msan -fi +mkdir -p ${STAGE3_MSAN_DIR} -CLANG_MSAN_PATH=$ROOT/llvm_build_msan/bin -CMAKE_STAGE3_MSAN_OPTIONS="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${CLANG_MSAN_PATH}/clang -DCMAKE_CXX_COMPILER=${CLANG_MSAN_PATH}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=15" +clang_msan_path=$ROOT/${STAGE2_MSAN_DIR}/bin +cmake_stage3_msan_options="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${clang_msan_path}/clang -DCMAKE_CXX_COMPILER=${clang_msan_path}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=15" -(cd llvm_build2_msan && cmake ${CMAKE_STAGE3_MSAN_OPTIONS} $LLVM && ninja) || \ +(cd ${STAGE3_MSAN_DIR} && cmake ${cmake_stage3_msan_options} $LLVM && ninja) || \ echo @@@STEP_FAILURE@@@ echo @@@BUILD_STEP check-all stage3/msan@@@ -(cd llvm_build2_msan && ninja check-all) || echo @@@STEP_FAILURE@@@ +(cd ${STAGE3_MSAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@ # Stage 2 / AddressSanitizer -echo @@@BUILD_STEP build clang/asan@@@ - -# Turn on init-order checker as ASan runtime option. -export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" -CMAKE_ASAN_OPTIONS=" \ - ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=Address \ - " - -if [ ! -d llvm_build_asan ]; then - mkdir llvm_build_asan -fi - -(cd llvm_build_asan && \ - cmake ${CMAKE_ASAN_OPTIONS} $LLVM && \ - ninja clang) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP check-llvm asan@@@ - -(cd llvm_build_asan && ninja check-llvm) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP check-clang asan@@@ - -(cd llvm_build_asan && ninja check-clang) || echo @@@STEP_FAILURE@@@ +build_stage2_asan +check_stage2_asan # Stage 3 / AddressSanitizer echo @@@BUILD_STEP build stage3/asan clang@@@ -if [ ! -d llvm_build2_asan ]; then - mkdir llvm_build2_asan -fi +mkdir -p ${STAGE3_ASAN_DIR} -CLANG_ASAN_PATH=$ROOT/llvm_build_asan/bin -CMAKE_STAGE3_ASAN_OPTIONS="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${CLANG_ASAN_PATH}/clang -DCMAKE_CXX_COMPILER=${CLANG_ASAN_PATH}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=10" +clang_asan_path=$ROOT/${STAGE2_ASAN_DIR}/bin +cmake_stage3_asan_options="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${clang_asan_path}/clang -DCMAKE_CXX_COMPILER=${clang_asan_path}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=10" -(cd llvm_build2_asan && cmake ${CMAKE_STAGE3_ASAN_OPTIONS} $LLVM && ninja) || \ +(cd ${STAGE3_ASAN_DIR} && cmake ${cmake_stage3_asan_options} $LLVM && ninja) || \ echo @@@STEP_FAILURE@@@ @@ -178,34 +94,16 @@ export ASAN_OPTIONS="check_initialization_order=true:detect_leaks=1" -(cd llvm_build2_asan && ninja check-all) || echo @@@STEP_FAILURE@@@ +(cd ${STAGE3_ASAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@ echo @@@BUILD_STEP check-all stage3/asan-uar@@@ export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" -(cd llvm_build2_asan && ninja check-all) || echo @@@STEP_FAILURE@@@ +(cd ${STAGE3_ASAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@ # Stage 2 / UndefinedBehaviorSanitizer -echo @@@BUILD_STEP build clang/ubsan@@@ - -export UBSAN_OPTIONS="external_symbolizer_path=${LLVM_SYMBOLIZER_PATH}:print_stacktrace=1" -CMAKE_UBSAN_OPTIONS=" \ - ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DCMAKE_BUILD_TYPE=Debug \ - -DLLVM_USE_SANITIZER=Undefined \ - " - -if [ ! -d llvm_build_ubsan ]; then - mkdir llvm_build_ubsan -fi - -(cd llvm_build_ubsan && - cmake ${CMAKE_UBSAN_OPTIONS} $LLVM && \ - ninja clang) || echo @@@STEP_FAILURE@@@ -echo @@@BUILD_STEP check-llvm ubsan@@@ -(cd llvm_build_ubsan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@ +build_stage2_ubsan -echo @@@BUILD_STEP check-clang ubsan@@@ -(cd llvm_build_ubsan && ninja check-clang) || echo @@@STEP_WARNINGS@@@ +check_stage2_ubsan Index: zorg/buildbot/builders/sanitizers/buildbot_fast.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_fast.sh +++ zorg/buildbot/builders/sanitizers/buildbot_fast.sh @@ -11,27 +11,20 @@ PLATFORM=`uname` export PATH="/usr/local/bin:$PATH" -if [ "$BUILDBOT_CLOBBER" != "" ]; then - echo @@@BUILD_STEP clobber@@@ - rm -rf llvm - rm -rf llvm_build0 -fi - +STAGE1_DIR=llvm_build0 +STAGE2_ASAN_DIR=llvm_build_asan +STAGE2_MSAN_DIR=llvm_build_msan +STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan HOST_CLANG_REVISION=223108 - -MEMORY_SANITIZER_KIND="Memory" -BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} -if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then - MEMORY_SANITIZER_KIND="MemoryWithOrigins" -fi - MAKE_JOBS=${MAX_MAKE_JOBS:-16} LLVM=$ROOT/llvm - -type -a gcc -type -a g++ CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3" -CMAKE_STAGE1_OPTIONS="${CMAKE_COMMON_OPTIONS}" + +if [ "$BUILDBOT_CLOBBER" != "" ]; then + echo @@@BUILD_STEP clobber@@@ + rm -rf llvm + rm -rf ${STAGE1_DIR} +fi # Stage 1 @@ -42,7 +35,7 @@ echo @@@BUILD_STEP using pre-built stage1 clang at r$HOST_CLANG_REVISION@@@ else echo @@@BUILD_STEP sync to r$HOST_CLANG_REVISION@@@ - REAL_BUILDBOT_REVISION=$BUILDBOT_REVISION + local real_buildbot_revision=$BUILDBOT_REVISION BUILDBOT_REVISION=$HOST_CLANG_REVISION buildbot_update @@ -53,104 +46,28 @@ # CMake does not notice that the compiler itself has changed. Anyway, # incremental builds of stage2 don't make sense if stage1 compiler has # changed. Clobber the build trees. - rm -rf libcxx_build_msan - rm -rf llvm_build_msan - rm -rf llvm_build_asan - - if [ ! -d llvm_build0 ]; then - mkdir llvm_build0 - fi - # Build cxx/cxxabi to fool the compiler check in MSan stage 2. - (cd llvm_build0 && cmake ${CMAKE_STAGE1_OPTIONS} $LLVM && \ - ninja clang compiler-rt llvm-symbolizer) + rm -rf ${STAGE2_LIBCXX_MSAN_DIR} + rm -rf ${STAGE2_MSAN_DIR} + rm -rf ${STAGE2_ASAN_DIR} + + build_stage1_clang + echo $HOST_CLANG_REVISION > host_clang_revision - BUILDBOT_REVISION=$REAL_BUILDBOT_REVISION + BUILDBOT_REVISION=$real_buildbot_revision fi echo @@@BUILD_STEP update@@@ buildbot_update -CLANG_PATH=$ROOT/llvm_build0/bin -CMAKE_STAGE2_COMMON_OPTIONS="\ - ${CMAKE_COMMON_OPTIONS} \ - -DCMAKE_C_COMPILER=${CLANG_PATH}/clang \ - -DCMAKE_CXX_COMPILER=${CLANG_PATH}/clang++ \ - " -LLVM_SYMBOLIZER_PATH=${CLANG_PATH}/llvm-symbolizer -export ASAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH} -export MSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH} - - # Stage 2 / Memory Sanitizer -echo @@@BUILD_STEP build libcxx/msan@@@ -if [ ! -d libcxx_build_msan ]; then - mkdir libcxx_build_msan -fi - -(cd libcxx_build_msan && \ - cmake \ - ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \ - $LLVM && \ - ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP build clang/msan@@@ -if [ ! -d llvm_build_msan ]; then - mkdir llvm_build_msan -fi - -MSAN_LDFLAGS="-lc++abi -Wl,--rpath=${ROOT}/libcxx_build_msan/lib -L${ROOT}/libcxx_build_msan/lib" -# See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264 -CMAKE_BUG_WORKAROUND_CFLAGS="$MSAN_LDFLAGS -fsanitize=memory -w" -MSAN_CFLAGS="-I${ROOT}/libcxx_build_msan/include -I${ROOT}/libcxx_build_msan/include/c++/v1 $CMAKE_BUG_WORKAROUND_CFLAGS" - -(cd llvm_build_msan && \ - cmake ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \ - -DLLVM_ENABLE_LIBCXX=ON \ - -DCMAKE_C_FLAGS="${MSAN_CFLAGS}" \ - -DCMAKE_CXX_FLAGS="${MSAN_CFLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="${MSAN_LDFLAGS}" \ - $LLVM && \ - ninja clang) || echo @@@STEP_FAILURE@@@ - -echo @@@BUILD_STEP check-llvm msan@@@ - -(cd llvm_build_msan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@ - -echo @@@BUILD_STEP check-clang msan@@@ - -(cd llvm_build_msan && ninja check-clang) || echo @@@STEP_FAILURE@@@ +build_stage2_msan +check_stage2_msan # Stage 2 / AddressSanitizer -echo @@@BUILD_STEP build clang/asan@@@ - -# Turn on init-order checker as ASan runtime option. -export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" -CMAKE_ASAN_OPTIONS=" \ - ${CMAKE_STAGE2_COMMON_OPTIONS} \ - -DLLVM_USE_SANITIZER=Address \ - " - -if [ ! -d llvm_build_asan ]; then - mkdir llvm_build_asan -fi - -(cd llvm_build_asan && \ - cmake ${CMAKE_ASAN_OPTIONS} $LLVM && \ - ninja clang) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP check-llvm asan@@@ - -(cd llvm_build_asan && ninja check-llvm) || echo @@@STEP_FAILURE@@@ - - -echo @@@BUILD_STEP check-clang asan@@@ +build_stage2_asan -(cd llvm_build_asan && ninja check-clang) || echo @@@STEP_FAILURE@@@ +check_stage2_asan Index: zorg/buildbot/builders/sanitizers/buildbot_functions.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -93,3 +93,118 @@ gclient runhooks ) } + +function build_stage1_clang { + type -a gcc + type -a g++ + mkdir -p ${STAGE1_DIR} + # Build cxx/cxxabi to fool the compiler check in MSan stage 2. + cmake_stage1_options="${CMAKE_COMMON_OPTIONS}" + (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && \ + ninja clang compiler-rt llvm-symbolizer) +} + +function common_stage2_variables { + local stage1_clang_path=$ROOT/${STAGE1_DIR}/bin + cmake_stage2_common_options="\ + ${CMAKE_COMMON_OPTIONS} \ + -DCMAKE_C_COMPILER=${stage1_clang_path}/clang \ + -DCMAKE_CXX_COMPILER=${stage1_clang_path}/clang++ \ + " + llvm_symbolizer_path=${stage1_clang_path}/llvm-symbolizer +} + +function build_stage2_msan { + echo @@@BUILD_STEP build libcxx/msan@@@ + + common_stage2_variables + export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + + memory_sanitizer_kind="Memory" + BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} + if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then + memory_sanitizer_kind="MemoryWithOrigins" + fi + + mkdir -p ${STAGE2_LIBCXX_MSAN_DIR} + (cd ${STAGE2_LIBCXX_MSAN_DIR} && \ + cmake \ + ${cmake_stage2_common_options} \ + -DLLVM_USE_SANITIZER=${memory_sanitizer_kind} \ + $LLVM && \ + ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@ + + echo @@@BUILD_STEP build clang/msan@@@ + + local msan_ldflags="-lc++abi -Wl,--rpath=${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/lib -L${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/lib" + # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264 + local cmake_bug_workaround_cflags="$msan_ldflags -fsanitize=memory -w" + local msan_cflags="-I${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/include -I${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/include/c++/v1 $cmake_bug_workaround_cflags" + mkdir -p ${STAGE2_MSAN_DIR} + (cd ${STAGE2_MSAN_DIR} && \ + cmake ${cmake_stage2_common_options} \ + -DLLVM_USE_SANITIZER=${memory_sanitizer_kind} \ + -DLLVM_ENABLE_LIBCXX=ON \ + -DCMAKE_C_FLAGS="${msan_cflags}" \ + -DCMAKE_CXX_FLAGS="${msan_cflags}" \ + -DCMAKE_EXE_LINKER_FLAGS="${msan_ldflags}" \ + $LLVM && \ + ninja clang) || echo @@@STEP_FAILURE@@@ +} + +function build_stage2_asan { + echo @@@BUILD_STEP build clang/asan@@@ + + common_stage2_variables + # Turn on init-order checker as ASan runtime option. + export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" + local cmake_asan_options=" \ + ${cmake_stage2_common_options} \ + -DLLVM_USE_SANITIZER=Address \ + " + mkdir -p ${STAGE2_ASAN_DIR} + (cd ${STAGE2_ASAN_DIR} && \ + cmake ${cmake_asan_options} $LLVM && \ + ninja clang) || echo @@@STEP_FAILURE@@@ +} + +function build_stage2_ubsan { + echo @@@BUILD_STEP build clang/ubsan@@@ + + common_stage2_variables + export UBSAN_OPTIONS="external_symbolizer_path=${llvm_symbolizer_path}:print_stacktrace=1" + local cmake_ubsan_options=" \ + ${cmake_stage2_common_options} \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLVM_USE_SANITIZER=Undefined \ + " + mkdir -p ${STAGE2_UBSAN_DIR} + (cd ${STAGE2_UBSAN_DIR} && + cmake ${cmake_ubsan_options} $LLVM && \ + ninja clang) || echo @@@STEP_FAILURE@@@ +} + +function check_stage2 { + local sanitizer_name=$1 + local build_dir=$2 + echo @@@BUILD_STEP check-llvm ${sanitizer_name}@@@ + + (cd ${build_dir} && ninja check-llvm) || echo @@@STEP_WARNINGS@@@ + + echo @@@BUILD_STEP check-clang ${sanitizer_name}@@@ + + (cd ${build_dir} && ninja check-clang) || echo @@@STEP_FAILURE@@@ +} + +function check_stage2_msan { + check_stage2 msan "${STAGE2_MSAN_DIR}" +} + +function check_stage2_asan { + check_stage2 asan "${STAGE2_ASAN_DIR}" +} + +function check_stage2_ubsan { + check_stage2 ubsan "${STAGE2_UBSAN_DIR}" +}