[zorg] Switch sanitizer-x86_64-linux-fast to use git

[vlad.tsyrklevich]

Any reason not to switch the other bots as well?

[eugenis]

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

181

Is USE_GIT initialized to 0 anywhere?

[vitalybuka]

CHECK_LLD

[vitalybuka]

In D66829#1647792, @vlad.tsyrklevich wrote:

Any reason not to switch the other bots as well?

I'll do in followup patches while this one is running on bot

[vitalybuka]

USE_GIT=0

zorg/buildbot/builders/sanitizers/buildbot_android.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 . ${HERE}/buildbot_android_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 LOCAL_IPS=`hostname -I`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 STAGE1_CLOBBER="llvm_build64 compiler_rt_build_android_* llvm_build_android_*"
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_PARALLEL_LINK_JOBS=20"
 
 if [ -e /usr/include/plugin-api.h ]; then

zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 STAGE2_ASAN_DIR=llvm_build_asan
 STAGE2_LIBCXX_ASAN_DIR=libcxx_build_asan
 STAGE3_ASAN_DIR=llvm_build2_asan
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=20"

zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 STAGE2_MSAN_DIR=llvm_build_msan
 STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan
 STAGE3_MSAN_DIR=llvm_build2_msan
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=20"

zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 STAGE2_UBSAN_DIR=llvm_build_ubsan
 STAGE2_LIBCXX_UBSAN_DIR=libcxx_build_ubsan
 STAGE3_UBSAN_DIR=llvm_build2_ubsan
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=20"

zorg/buildbot/builders/sanitizers/buildbot_cmake.sh

if [ "$PLATFORM" == "Darwin" ]; then
   CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DPYTHON_EXECUTABLE=/usr/bin/python"
   ENABLE_LIBCXX_FLAG="-DLLVM_ENABLE_LIBCXX=ON"
 fi
 
 if [ -e /usr/include/plugin-api.h ]; then
   CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_BINUTILS_INCDIR=/usr/include"
 fi
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_SYMBOLIZER=${CHECK_SYMBOLIZER:-$CHECK_LIBCXX}
 CHECK_LLD=${CHECK_LLD:-1}
 CHECK_ASAN=0
 CHECK_HWASAN=0
 CHECK_UBSAN=0
 CHECK_MSAN=0
 CHECK_TSAN=0

zorg/buildbot/builders/sanitizers/buildbot_fast.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=1
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 STAGE2_ASAN_DIR=llvm_build_asan
 STAGE2_MSAN_DIR=llvm_build_msan
 STAGE2_UBSAN_DIR=llvm_build_ubsan
 STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan
 STAGE2_LIBCXX_ASAN_DIR=libcxx_build_asan
 STAGE2_LIBCXX_UBSAN_DIR=libcxx_build_ubsan
 STAGE1_CLOBBER="${STAGE2_LIBCXX_MSAN_DIR} ${STAGE2_LIBCXX_ASAN_DIR} ${STAGE2_LIBCXX_UBSAN_DIR} ${STAGE2_MSAN_DIR} ${STAGE2_ASAN_DIR} ${STAGE2_UBSAN_DIR}"
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_PARALLEL_LINK_JOBS=20"
 
 if [ "$BUILDBOT_CLOBBER" != "" ]; then
   echo @@@BUILD_STEP clobber@@@
   rm -rf llvm
+  rm -rf llvm-project
   rm -rf ${STAGE1_DIR}
 fi
 
 # Stage 1
 
 build_stage1_clang_at_revison
 
 CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_ENABLE_ASSERTIONS=ON"
 
 echo @@@BUILD_STEP update@@@
-buildbot_update
+buildbot_update_git
 
 # Stage 2 / Memory Sanitizer
 
 build_stage2_msan
 
 check_stage2_msan
 
 # Stage 2 / AddressSanitizer

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

if [ -d "$BUILDBOT_MONO_REPO_PATH" ]; then
     return
   fi
 
   update_or_checkout "$rev_arg" http://llvm.org/svn/llvm-project/$repo_name/trunk $tree &
 }
 
 
 function buildbot_update {
+    if [[ "$USE_GIT" != "0" ]]; then
+      buildbot_update_git
+      return
+    fi
     if [[ -d "$BUILDBOT_MONO_REPO_PATH" ]]; then
       BUILDBOT_REVISION="-"
     else
       if [[ "$BUILDBOT_REVISION" == "-" ]]; then
         return
       fi
     fi
 
function gclient_runhooks {
   export CXX="$CLANG_BIN/clang++"
   
   gclient runhooks
   )
 }
 
 function build_stage1_clang {
   mkdir -p ${STAGE1_DIR}
-  cmake_stage1_options="${CMAKE_COMMON_OPTIONS}"
+  local cmake_stage1_options="${CMAKE_COMMON_OPTIONS}"
+  if [[ "$USE_GIT" != "0" ]]; then
+    cmake_stage1_options="${cmake_stage1_options} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'"
+  fi
   (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && \
     ninja clang lld compiler-rt llvm-symbolizer)
 }
 
 function build_stage1_clang_at_revison {
   local HOST_CLANG_REVISION=360832
 
   if  [ -r ${STAGE1_DIR}/host_clang_revision ] && \
       [ "$(cat ${STAGE1_DIR}/host_clang_revision)" == $HOST_CLANG_REVISION ]
   then
     echo @@@BUILD_STEP using pre-built stage1 clang at r$HOST_CLANG_REVISION@@@
   else
     echo @@@BUILD_STEP sync to r$HOST_CLANG_REVISION@@@
-    (BUILDBOT_REVISION=$HOST_CLANG_REVISION buildbot_update)
+    BUILDBOT_REVISION=$HOST_CLANG_REVISION buildbot_update
 
     echo @@@BUILD_STEP Clear ${STAGE1_DIR} ${STAGE1_CLOBBER}
     rm -rf ${STAGE1_DIR} ${STAGE1_CLOBBER}
 
     echo @@@BUILD_STEP build stage1 clang at r$HOST_CLANG_REVISION@@@
     build_stage1_clang
     echo $HOST_CLANG_REVISION > ${STAGE1_DIR}/host_clang_revision
   fi
function build_stage2 {
   local sanitizer_cflags=""
   local cmake_libcxx_flag="-DLLVM_ENABLE_LIBCXX=OFF"
 
   # Don't use libc++/libc++abi in UBSan builds (due to known bugs).
   if [ "$CHECK_LIBCXX" != "0" -a \
        "$sanitizer_name" != "ubsan" ]; then
     echo @@@BUILD_STEP build libcxx/$sanitizer_name@@@
     mkdir -p ${libcxx_build_dir}
+    local cmake_stage2_libcxx_options=
+    if [[ "$USE_GIT" != "0" ]]; then
+      cmake_stage2_libcxx_options="-DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi'"
+    fi
     (cd ${libcxx_build_dir} && \
       cmake \
         ${cmake_stage2_common_options} \
+        ${cmake_stage2_libcxx_options} \
         -DCMAKE_BUILD_TYPE=${build_type} \
         -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \
         $LLVM && \
       ninja cxx cxxabi) || echo $step_result
     sanitizer_ldflags="$sanitizer_ldflags -lc++abi -Wl,--rpath=${ROOT}/${libcxx_build_dir}/lib -L${ROOT}/${libcxx_build_dir}/lib"
     sanitizer_cflags="$sanitizer_cflags -nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1"
     cmake_libcxx_flag="-DLLVM_ENABLE_LIBCXX=ON"
   fi
 
   echo @@@BUILD_STEP build clang/$sanitizer_name@@@
 
   # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264
   local cmake_bug_workaround_cflags="$sanitizer_ldflags $fsanitize_flag -w"
   sanitizer_cflags="$sanitizer_cflags $cmake_bug_workaround_cflags"
 
   mkdir -p ${build_dir}
   local extra_dir
   if [ "$CHECK_LLD" != "0" ]; then
     extra_dir="lld"
   fi
+  local cmake_stage2_clang_options=
+  if [[ "$USE_GIT" != "0" ]]; then
+    local projects=clang
+    if [[ "$CHECK_LLD" != "0" ]]; then
+      projects="${projects};lld"
+    fi
+    cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='${projects}'"
+  fi
   (cd ${build_dir} && \
-   cmake ${cmake_stage2_common_options} \
+   cmake \
+     ${cmake_stage2_common_options} \
+     ${cmake_stage2_clang_options} \
      -DCMAKE_BUILD_TYPE=${build_type} \
      -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \
      ${cmake_libcxx_flag} \
      -DCMAKE_C_FLAGS="${sanitizer_cflags}" \
      -DCMAKE_CXX_FLAGS="${sanitizer_cflags}" \
      -DCMAKE_EXE_LINKER_FLAGS="${sanitizer_ldflags}" \
      $LLVM && \
    ninja clang ${extra_dir}) || echo $step_result

zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-0}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)}
 LLVM=$ROOT/llvm
 LIBFUZZER=$LLVM/lib/Fuzzer
 # No assertions. Need to clean up the existing assertion failures first.
 # Also, the Fuzzer does not provide reproducers on assertion failures yet.

zorg/buildbot/builders/sanitizers/buildbot_mte.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 STAGE1_DIR=llvm_build0
 LLVM=$ROOT/llvm
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_COMPILE_JOBS=100 -DLLVM_PARALLEL_LINK_JOBS=20"
 
 if [ "$BUILDBOT_CLOBBER" != "" ]; then
   echo @@@BUILD_STEP clobber@@@

zorg/buildbot/builders/sanitizers/buildbot_perf_asan.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+USE_GIT=0
+
 LLVM=$ROOT/llvm
 CLANG_BUILD=$ROOT/clang_build
 SPEC_DIRNAME=SPEC_CPU2006v1.2
 SPEC_SRC="${ROOT}/${SPEC_DIRNAME}"
 SPEC_RUNNER=./run_spec_clang_asan.sh
 #SPEC_TESTS='perlbench bzip2'
 SPEC_TESTS='perlbench bzip2 gcc mcf gobmk hmmer sjeng libquantum h264ref omnetpp astar xalancbmk'
 

zorg/buildbot/builders/sanitizers/buildbot_standard.sh

if [ "$BUILDBOT_CLOBBER" != "" ]; then
   rm -rf $TSAN_DEBUG_BUILD_DIR
   rm -rf $TSAN_FULL_DEBUG_BUILD_DIR
   rm -rf $TSAN_RELEASE_BUILD_DIR
 fi
 
 ROOT=`pwd`
 PLATFORM=`uname`
 MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)}
+
+USE_GIT=0
+
 CHECK_LIBCXX=${CHECK_LIBCXX:-1}
 CHECK_LLD=${CHECK_LLD:-1}
 
 LLVM=${ROOT}/llvm
 CMAKE_COMMON_OPTIONS="-DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON"
 
 function build_tsan {
   local build_dir=$1