Index: zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh +++ zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh @@ -23,7 +23,13 @@ STAGE3_ASAN_DIR=llvm_build2_asan STAGE3_MSAN_DIR=llvm_build2_msan LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=20" +ZLIB=$ROOT/zlib +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10" + +function build_symbolizer { + check_stage2 msan "${STAGE2_MSAN_DIR}" @@@STEP_FAILURE@@@ + (cd $1 && $LLVM/ninja check-llvm) || echo @@@STEP_FAILURE@@@ +} if [ "$BUILDBOT_CLOBBER" != "" ]; then echo @@@BUILD_STEP clobber@@@ @@ -34,18 +40,21 @@ # CMake does not notice that the compiler itself has changed. # Anyway, incremental builds of stage2 and stage3 compilers don't make sense. # Clobber the build trees. -rm -rf ${STAGE2_LIBCXX_MSAN_DIR} -rm -rf ${STAGE2_LIBCXX_ASAN_DIR} -rm -rf ${STAGE2_LIBCXX_UBSAN_DIR} -rm -rf ${STAGE2_MSAN_DIR} -rm -rf ${STAGE3_MSAN_DIR} -rm -rf ${STAGE2_ASAN_DIR} -rm -rf ${STAGE3_ASAN_DIR} -rm -rf ${STAGE2_UBSAN_DIR} +# rm -rf ${STAGE2_LIBCXX_MSAN_DIR} +# rm -rf ${STAGE2_LIBCXX_ASAN_DIR} +# rm -rf ${STAGE2_LIBCXX_UBSAN_DIR} +# rm -rf ${STAGE2_MSAN_DIR} +# rm -rf ${STAGE3_MSAN_DIR} +# rm -rf ${STAGE2_ASAN_DIR} +# rm -rf ${STAGE3_ASAN_DIR} +# rm -rf ${STAGE2_UBSAN_DIR} echo @@@BUILD_STEP update@@@ buildbot_update +echo @@@BUILD_STEP update zlib@@@ +git -C $ZLIB pull --rebase || git clone https://github.com/madler/zlib.git $ZLIB || echo @@@STEP_FAILURE@@@ + # Stage 1 echo @@@BUILD_STEP build stage1 clang@@@ @@ -58,6 +67,12 @@ check_stage2_msan +build_symbolizer "${STAGE2_MSAN_DIR}" + +check_stage2_msan + +exit 1 + # Stage 3 / MemorySanitizer echo @@@BUILD_STEP build stage3/msan clang@@@ Index: zorg/buildbot/builders/sanitizers/buildbot_cmake.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_cmake.sh +++ zorg/buildbot/builders/sanitizers/buildbot_cmake.sh @@ -135,36 +135,22 @@ # Do a sanity check on Linux: build and test sanitizers using gcc as a host # compiler. if [ "$PLATFORM" == "Linux" ]; then - echo @@@BUILD_STEP check-sanitizer in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-sanitizer) || echo @@@STEP_FAILURE@@@ - if [ "$CHECK_ASAN" == "1" ]; then - echo @@@BUILD_STEP check-asan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-asan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_UBSAN" == "1" ]; then - echo @@@BUILD_STEP check-ubsan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-ubsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_LSAN" == "1" ]; then - echo @@@BUILD_STEP check-lsan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-lsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_MSAN" == "1" ]; then - echo @@@BUILD_STEP check-msan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-msan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_TSAN" == "1" ]; then - echo @@@BUILD_STEP check-tsan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-tsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_DFSAN" == "1" ]; then - echo @@@BUILD_STEP check-dfsan in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-dfsan) || echo @@@STEP_WARNINGS@@@ - fi - if [ "$CHECK_CFI" == "1" ]; then - echo @@@BUILD_STEP check-cfi-and-supported in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-cfi-and-supported) || echo @@@STEP_FAILURE@@@ - fi + check_in_gcc() { + CONDITION=$1 + SANITIZER=$2 + if [ "$CONDITION" == "1" ]; then + echo @@@BUILD_STEP check-$SANITIZER in gcc build@@@ + (cd clang_build && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + fi + } + check_in_gcc 1 sanitizer + check_in_gcc $CHECK_ASAN asan + check_in_gcc $CHECK_CFI cfi-and-supported + check_in_gcc $CHECK_DFSAN dfsan + check_in_gcc $CHECK_LSAN lsan + check_in_gcc $CHECK_MSAN msan + check_in_gcc $CHECK_TSAN tsan + check_in_gcc $CHECK_UBSAN ubsan fi ### From now on we use just-built Clang as a host compiler ### @@ -191,44 +177,32 @@ # Now build everything else. (cd llvm_build64 && make -j$MAKE_JOBS) || echo @@@STEP_FAILURE@@@ +# FIXME: Make these true dependencies of check-cfi-and-supported when + # compiler-rt is configured as an external project. +(cd llvm_build64 && make -j$MAKE_JOBS LLVMgold opt sanstats) || echo @@@STEP_FAILURE@@@ + FRESH_CLANG_PATH=${ROOT}/llvm_build64/bin COMPILER_RT_BUILD_PATH=projects/compiler-rt/src/compiler-rt-build -if [ "$CHECK_ASAN" == "1" ]; then - echo @@@BUILD_STEP run asan tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-asan) || echo @@@STEP_FAILURE@@@ -fi - -if [ "$PLATFORM" == "Linux" -a "$CHECK_ASAN" == "1" ]; then - echo @@@BUILD_STEP run asan-dynamic tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-asan-dynamic) || echo @@@STEP_FAILURE@@@ -fi - -if [ "$PLATFORM" == "Linux" -a "$CHECK_MSAN" == "1" ]; then - echo @@@BUILD_STEP run msan unit tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-msan) || echo @@@STEP_FAILURE@@@ -fi - -if [ "$PLATFORM" == "Linux" -a "$CHECK_TSAN" == "1" ]; then - echo @@@BUILD_STEP run 64-bit tsan unit tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-tsan) || echo @@@STEP_FAILURE@@@ -fi - -if [ "$PLATFORM" == "Linux" -a "$CHECK_LSAN" == "1" ]; then - echo @@@BUILD_STEP run 64-bit lsan unit tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-lsan) || echo @@@STEP_FAILURE@@@ -fi - -if [ "$PLATFORM" == "Linux" -a "$CHECK_DFSAN" == "1" ]; then - echo @@@BUILD_STEP run 64-bit dfsan unit tests@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-dfsan) || echo @@@STEP_FAILURE@@@ -fi +check_64bit() { + CONDITION=$1 + SANITIZER=$2 + if ["$CONDITION" == "1" ]; then + echo @@@BUILD_STEP 64-bit check-$SANITIZER@@@ + (cd llvm_build64 && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + fi +} -if [ "$PLATFORM" == "Linux" -a "$CHECK_CFI" == "1" ]; then - echo @@@BUILD_STEP run 64-bit cfi unit tests@@@ - # FIXME: Make these true dependencies of check-cfi-and-supported when - # compiler-rt is configured as an external project. - (cd llvm_build64 && make -j$MAKE_JOBS LLVMgold opt sanstats && make -j$MAKE_JOBS check-cfi-and-supported) || echo @@@STEP_FAILURE@@@ +check_64bit 1 sanitizer +check_64bit $CHECK_ASAN asan +if [ "$PLATFORM" == "Linux" ]; then + check_64bit $CHECK_ASAN asan-dynamic + check_64bit $CHECK_CFI cfi-and-supported + check_64bit $CHECK_DFSAN dfsan + check_64bit $CHECK_LSAN lsan + check_64bit $CHECK_MSAN msan + check_64bit $CHECK_TSAN tsan + check_64bit $CHECK_UBSAN ubsan fi echo @@@BUILD_STEP run sanitizer_common tests@@@ @@ -261,40 +235,24 @@ ${CMAKE_NINJA_OPTIONS} $LLVM_CHECKOUT) ln -sf llvm_build_ninja/compile_commands.json $LLVM_CHECKOUT (cd llvm_build_ninja && ninja) || echo @@@STEP_FAILURE@@@ - echo @@@BUILD_STEP ninja check-sanitizer@@@ - (cd llvm_build_ninja && ninja check-sanitizer) || echo @@@STEP_FAILURE@@@ - if [ "$CHECK_ASAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-asan@@@ - (cd llvm_build_ninja && ninja check-asan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_UBSAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-ubsan@@@ - (cd llvm_build_ninja && ninja check-ubsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_TSAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-tsan@@@ - (cd llvm_build_ninja && ninja check-tsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_SCUDO" == 1 ]; then - echo @@@BUILD_STEP ninja check-scudo@@@ - (cd llvm_build_ninja && ninja check-scudo) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_MSAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-msan@@@ - (cd llvm_build_ninja && ninja check-msan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_LSAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-lsan@@@ - (cd llvm_build_ninja && ninja check-lsan) || echo @@@STEP_FAILURE@@@ - fi - if [ "$CHECK_DFSAN" == 1 ]; then - echo @@@BUILD_STEP ninja check-dfsan@@@ - (cd llvm_build_ninja && ninja check-dfsan) || echo @@@STEP_WARNINGS@@@ - fi - if [ "$CHECK_CFI" == 1 ]; then - echo @@@BUILD_STEP ninja check-cfi-and-supported@@@ - (cd llvm_build_ninja && ninja check-cfi-and-supported) || echo @@@STEP_FAILURE@@@ - fi + + check_ninja() { + CONDITION=$1 + SANITIZER=$2 + if ["$CONDITION" == "1" ]; then + echo @@@BUILD_STEP ninja check-$SANITIZER@@@ + (cd llvm_build_ninja && ninja check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + fi + } + + check_ninja 1 sanitizer + check_ninja $CHECK_ASAN asan + check_ninja $CHECK_CFI cfi-and-supported + check_ninja $CHECK_DFSAN dfsan + check_ninja $CHECK_LSAN lsan + check_ninja $CHECK_MSAN msan + check_ninja $CHECK_TSAN tsan + check_ninja $CHECK_UBSAN ubsan fi if [ $BUILD_ANDROID == 1 ] ; then Index: zorg/buildbot/builders/sanitizers/buildbot_functions.sh =================================================================== --- zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -9,6 +9,7 @@ else svn co "${repo}" "${tree}" $rev_arg fi + svn revert -R "${tree}" } function buildbot_update { @@ -31,23 +32,24 @@ fi done - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/llvm/trunk llvm - if [ "$rev_arg" == "" ]; then - rev_arg="-r"$(svn info llvm | grep '^Revision:' | awk '{print $2}') + rev_arg="-r"$(svn info http://llvm.org/svn/llvm-project/llvm/trunk | grep '^Revision:' | awk '{print $2}') fi + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/llvm/trunk llvm & + # XXX: Keep this list in sync with the change filter in buildbot/osuosl/master/master.cfg. - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/cfe/trunk llvm/tools/clang - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/compiler-rt/trunk llvm/projects/compiler-rt + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/cfe/trunk llvm/tools/clang & + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/compiler-rt/trunk llvm/projects/compiler-rt & if [ "$CHECK_LIBCXX" != "0" ]; then - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libcxx/trunk llvm/projects/libcxx - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libcxxabi/trunk llvm/projects/libcxxabi - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libunwind/trunk llvm/projects/libunwind + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libcxx/trunk llvm/projects/libcxx & + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libcxxabi/trunk llvm/projects/libcxxabi & + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/libunwind/trunk llvm/projects/libunwind & fi if [ "$CHECK_LLD" != "0" ]; then - update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/lld/trunk llvm/tools/lld + update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/lld/trunk llvm/tools/lld & fi + wait } function set_chrome_suid_sandbox {