Refactor sanitizer build scripts.

[earthdok]

Summary

• Move common code from buildbot_bootstrap.sh and buildbot_fast.sh to

buildbot_functions.sh.

• Do not capitalize variable names needlessly.
• Use mkdir -p where appropriate instead of "mdkir if directory does not exist".

[eugenis]

zorg/buildbot/builders/sanitizers/buildbot_fast.sh

65

I think it would be better if stuff in buildbot_functions.sh did not rely on the STAGE2_MSAN_DIR and similar settings, and instead got them through function arguments. That would make it more reusable.
Ex.

build_stage2_msan "step name" <target directory> <previous stage compiler directory>

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

99

these are debug leftovers, please remove

101

the comment is out of date, please remove it

123

local

[earthdok]

• address nits

zorg/buildbot/builders/sanitizers/buildbot_fast.sh

65

As discussed offline, this is not worth the trobule

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

99

Done.

101

Done.

123

done

[earthdok]

Committed r224520

zorg/buildbot/builders/sanitizers/buildbot_bootstrap.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
+STAGE1_DIR=llvm_build0
+STAGE2_ASAN_DIR=llvm_build_asan
+STAGE2_MSAN_DIR=llvm_build_msan
+STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan
+STAGE2_UBSAN_DIR=llvm_build_ubsan
+STAGE3_ASAN_DIR=llvm_build2_asan
+STAGE3_MSAN_DIR=llvm_build2_msan
+MAKE_JOBS=${MAX_MAKE_JOBS:-16}
+LLVM=$ROOT/llvm
+CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3"
+
 if [ "$BUILDBOT_CLOBBER" != "" ]; then
   echo @@@BUILD_STEP clobber@@@
   rm -rf llvm
-  rm -rf llvm_build0
-fi
-
-MEMORY_SANITIZER_KIND="Memory"
-BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-}
-if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then
-    MEMORY_SANITIZER_KIND="MemoryWithOrigins"
+  rm -rf ${STAGE1_DIR}
 fi
 
 # CMake does not notice that the compiler itself has changed.
 # Anyway, incremental builds of stage2 and stage3 compilers don't make sense.
 # Clobber the build trees.
-rm -rf libcxx_build_msan
-rm -rf llvm_build_msan
-rm -rf llvm_build2_msan
-rm -rf llvm_build_asan
-rm -rf llvm_build2_asan
-rm -rf llvm_build_ubsan
-
-MAKE_JOBS=${MAX_MAKE_JOBS:-16}
-LLVM=$ROOT/llvm
-
-type -a gcc
-type -a g++
-CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3"
-CMAKE_STAGE1_OPTIONS="${CMAKE_COMMON_OPTIONS}"
+rm -rf ${STAGE2_LIBCXX_MSAN_DIR}
+rm -rf ${STAGE2_MSAN_DIR}
+rm -rf ${STAGE3_MSAN_DIR}
+rm -rf ${STAGE2_ASAN_DIR}
+rm -rf ${STAGE3_ASAN_DIR}
+rm -rf ${STAGE2_UBSAN_DIR}
 
 echo @@@BUILD_STEP update@@@
 buildbot_update
 
 # Stage 1
 
 echo @@@BUILD_STEP build stage1 clang@@@
-if [ ! -d llvm_build0 ]; then
-  mkdir llvm_build0
-fi
-# Build cxx/cxxabi to fool the compiler check in MSan stage 2.
-(cd llvm_build0 && cmake ${CMAKE_STAGE1_OPTIONS} $LLVM && \
-  ninja clang compiler-rt llvm-symbolizer) || \
-  echo @@@STEP_FAILURE@@@
 
-CLANG_PATH=$ROOT/llvm_build0/bin
-CMAKE_STAGE2_COMMON_OPTIONS="\
-  ${CMAKE_COMMON_OPTIONS} \
-  -DCMAKE_C_COMPILER=${CLANG_PATH}/clang \
-  -DCMAKE_CXX_COMPILER=${CLANG_PATH}/clang++ \
-  "
-LLVM_SYMBOLIZER_PATH=${CLANG_PATH}/llvm-symbolizer
-export ASAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH}
-export MSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH}
+build_stage1_clang
 
 # Stage 2 / Memory Sanitizer
 
-echo @@@BUILD_STEP build libcxx/msan@@@
-if [ ! -d libcxx_build_msan ]; then
-  mkdir libcxx_build_msan
-fi
-
-(cd libcxx_build_msan && \
-  cmake \
-    ${CMAKE_STAGE2_COMMON_OPTIONS} \
-    -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \
-    $LLVM && \
-  ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP build clang/msan@@@
-if [ ! -d llvm_build_msan ]; then
-  mkdir llvm_build_msan
-fi
-
-MSAN_LDFLAGS="-lc++abi -Wl,--rpath=${ROOT}/libcxx_build_msan/lib -L${ROOT}/libcxx_build_msan/lib"
-# See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264
-CMAKE_BUG_WORKAROUND_CFLAGS="$MSAN_LDFLAGS -fsanitize=memory -w"
-MSAN_CFLAGS="-I${ROOT}/libcxx_build_msan/include -I${ROOT}/libcxx_build_msan/include/c++/v1 $CMAKE_BUG_WORKAROUND_CFLAGS"
-
-(cd llvm_build_msan && \
- cmake ${CMAKE_STAGE2_COMMON_OPTIONS} \
-   -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \
-   -DLLVM_ENABLE_LIBCXX=ON \
-   -DCMAKE_C_FLAGS="${MSAN_CFLAGS}" \
-   -DCMAKE_CXX_FLAGS="${MSAN_CFLAGS}" \
-   -DCMAKE_EXE_LINKER_FLAGS="${MSAN_LDFLAGS}" \
-   $LLVM && \
- ninja clang) || echo @@@STEP_FAILURE@@@
-
-echo @@@BUILD_STEP check-llvm msan@@@
-
-(cd llvm_build_msan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@
-
-echo @@@BUILD_STEP check-clang msan@@@
-
-(cd llvm_build_msan && ninja check-clang) || echo @@@STEP_FAILURE@@@
+build_stage2_msan
 
+check_stage2_msan
 
 # Stage 3 / MemorySanitizer
 
 echo @@@BUILD_STEP build stage3/msan clang@@@
 
-if [ ! -d llvm_build2_msan ]; then
-  mkdir llvm_build2_msan
-fi
+mkdir -p ${STAGE3_MSAN_DIR}
 
-CLANG_MSAN_PATH=$ROOT/llvm_build_msan/bin
-CMAKE_STAGE3_MSAN_OPTIONS="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${CLANG_MSAN_PATH}/clang -DCMAKE_CXX_COMPILER=${CLANG_MSAN_PATH}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=15"
+clang_msan_path=$ROOT/${STAGE2_MSAN_DIR}/bin
+cmake_stage3_msan_options="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${clang_msan_path}/clang -DCMAKE_CXX_COMPILER=${clang_msan_path}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=15"
 
-(cd llvm_build2_msan && cmake ${CMAKE_STAGE3_MSAN_OPTIONS} $LLVM && ninja) || \
+(cd ${STAGE3_MSAN_DIR} && cmake ${cmake_stage3_msan_options} $LLVM && ninja) || \
   echo @@@STEP_FAILURE@@@
 
 
 echo @@@BUILD_STEP check-all stage3/msan@@@
 
-(cd llvm_build2_msan && ninja check-all) || echo @@@STEP_FAILURE@@@
+(cd ${STAGE3_MSAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@
 
 
 # Stage 2 / AddressSanitizer
 
-echo @@@BUILD_STEP build clang/asan@@@
-
-# Turn on init-order checker as ASan runtime option.
-export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1"
-CMAKE_ASAN_OPTIONS=" \
-  ${CMAKE_STAGE2_COMMON_OPTIONS} \
-  -DLLVM_USE_SANITIZER=Address \
-  "
-
-if [ ! -d llvm_build_asan ]; then
-  mkdir llvm_build_asan
-fi
-
-(cd llvm_build_asan && \
- cmake ${CMAKE_ASAN_OPTIONS} $LLVM && \
- ninja clang) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP check-llvm asan@@@
-
-(cd llvm_build_asan && ninja check-llvm) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP check-clang asan@@@
-
-(cd llvm_build_asan && ninja check-clang) || echo @@@STEP_FAILURE@@@
+build_stage2_asan
 
+check_stage2_asan
 
 # Stage 3 / AddressSanitizer
 
 echo @@@BUILD_STEP build stage3/asan clang@@@
 
-if [ ! -d llvm_build2_asan ]; then
-  mkdir llvm_build2_asan
-fi
+mkdir -p ${STAGE3_ASAN_DIR}
 
-CLANG_ASAN_PATH=$ROOT/llvm_build_asan/bin
-CMAKE_STAGE3_ASAN_OPTIONS="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${CLANG_ASAN_PATH}/clang -DCMAKE_CXX_COMPILER=${CLANG_ASAN_PATH}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=10"
+clang_asan_path=$ROOT/${STAGE2_ASAN_DIR}/bin
+cmake_stage3_asan_options="${CMAKE_COMMON_OPTIONS} -DCMAKE_C_COMPILER=${clang_asan_path}/clang -DCMAKE_CXX_COMPILER=${clang_asan_path}/clang++ -DLLVM_PARALLEL_COMPILE_JOBS=10"
 
-(cd llvm_build2_asan && cmake ${CMAKE_STAGE3_ASAN_OPTIONS} $LLVM && ninja) || \
+(cd ${STAGE3_ASAN_DIR} && cmake ${cmake_stage3_asan_options} $LLVM && ninja) || \
   echo @@@STEP_FAILURE@@@
 
 
 echo @@@BUILD_STEP check-all stage3/asan@@@
 
 export ASAN_OPTIONS="check_initialization_order=true:detect_leaks=1"
 
-(cd llvm_build2_asan && ninja check-all) || echo @@@STEP_FAILURE@@@
+(cd ${STAGE3_ASAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@
 
 echo @@@BUILD_STEP check-all stage3/asan-uar@@@
 
 export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1"
 
-(cd llvm_build2_asan && ninja check-all) || echo @@@STEP_FAILURE@@@
+(cd ${STAGE3_ASAN_DIR} && ninja check-all) || echo @@@STEP_FAILURE@@@
 
 # Stage 2 / UndefinedBehaviorSanitizer
-echo @@@BUILD_STEP build clang/ubsan@@@
-
-export UBSAN_OPTIONS="external_symbolizer_path=${LLVM_SYMBOLIZER_PATH}:print_stacktrace=1"
-CMAKE_UBSAN_OPTIONS=" \
-  ${CMAKE_STAGE2_COMMON_OPTIONS} \
-  -DCMAKE_BUILD_TYPE=Debug \
-  -DLLVM_USE_SANITIZER=Undefined \
-  "
-
-if [ ! -d llvm_build_ubsan ]; then
-  mkdir llvm_build_ubsan
-fi
-
-(cd llvm_build_ubsan &&
-  cmake ${CMAKE_UBSAN_OPTIONS} $LLVM && \
-  ninja clang) || echo @@@STEP_FAILURE@@@
 
-echo @@@BUILD_STEP check-llvm ubsan@@@
-(cd llvm_build_ubsan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@
+build_stage2_ubsan
 
-echo @@@BUILD_STEP check-clang ubsan@@@
-(cd llvm_build_ubsan && ninja check-clang) || echo @@@STEP_WARNINGS@@@
+check_stage2_ubsan

zorg/buildbot/builders/sanitizers/buildbot_fast.sh

 #!/usr/bin/env bash
 
 set -x
 set -e
 set -u
 
 HERE="$(cd $(dirname $0) && pwd)"
 . ${HERE}/buildbot_functions.sh
 
 ROOT=`pwd`
 PLATFORM=`uname`
 export PATH="/usr/local/bin:$PATH"
 
-if [ "$BUILDBOT_CLOBBER" != "" ]; then
-  echo @@@BUILD_STEP clobber@@@
-  rm -rf llvm
-  rm -rf llvm_build0
-fi
-
+STAGE1_DIR=llvm_build0
+STAGE2_ASAN_DIR=llvm_build_asan
+STAGE2_MSAN_DIR=llvm_build_msan
+STAGE2_LIBCXX_MSAN_DIR=libcxx_build_msan
 HOST_CLANG_REVISION=223108
-
-MEMORY_SANITIZER_KIND="Memory"
-BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-}
-if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then
-    MEMORY_SANITIZER_KIND="MemoryWithOrigins"
-fi
-
 MAKE_JOBS=${MAX_MAKE_JOBS:-16}
 LLVM=$ROOT/llvm
-
-type -a gcc
-type -a g++
 CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=3"
-CMAKE_STAGE1_OPTIONS="${CMAKE_COMMON_OPTIONS}"
+
+if [ "$BUILDBOT_CLOBBER" != "" ]; then
+  echo @@@BUILD_STEP clobber@@@
+  rm -rf llvm
+  rm -rf ${STAGE1_DIR}
+fi
 
 # Stage 1
 
 if  [ -r host_clang_revision ] && \
     [ "$(cat host_clang_revision)" == $HOST_CLANG_REVISION ]
 then
   # Do nothing.
   echo @@@BUILD_STEP using pre-built stage1 clang at r$HOST_CLANG_REVISION@@@
 else
   echo @@@BUILD_STEP sync to r$HOST_CLANG_REVISION@@@
-  REAL_BUILDBOT_REVISION=$BUILDBOT_REVISION
+  local real_buildbot_revision=$BUILDBOT_REVISION
   BUILDBOT_REVISION=$HOST_CLANG_REVISION
   buildbot_update
 
   echo @@@BUILD_STEP build stage1 clang at r$HOST_CLANG_REVISION@@@
 
   rm -rf host_clang_revision
 
   # CMake does not notice that the compiler itself has changed. Anyway,
   # incremental builds of stage2 don't make sense if stage1 compiler has
   # changed. Clobber the build trees.
-  rm -rf libcxx_build_msan
-  rm -rf llvm_build_msan
-  rm -rf llvm_build_asan
+  rm -rf ${STAGE2_LIBCXX_MSAN_DIR}
+  rm -rf ${STAGE2_MSAN_DIR}
+  rm -rf ${STAGE2_ASAN_DIR}
+
+  build_stage1_clang
 
-  if [ ! -d llvm_build0 ]; then
-    mkdir llvm_build0
-  fi
-  # Build cxx/cxxabi to fool the compiler check in MSan stage 2.
-  (cd llvm_build0 && cmake ${CMAKE_STAGE1_OPTIONS} $LLVM && \
-    ninja clang compiler-rt llvm-symbolizer)
   echo $HOST_CLANG_REVISION > host_clang_revision
 
-  BUILDBOT_REVISION=$REAL_BUILDBOT_REVISION
+  BUILDBOT_REVISION=$real_buildbot_revision
 fi
 
 echo @@@BUILD_STEP update@@@
 buildbot_update
 
-CLANG_PATH=$ROOT/llvm_build0/bin
-CMAKE_STAGE2_COMMON_OPTIONS="\
-  ${CMAKE_COMMON_OPTIONS} \
-  -DCMAKE_C_COMPILER=${CLANG_PATH}/clang \
-  -DCMAKE_CXX_COMPILER=${CLANG_PATH}/clang++ \
-  "
-LLVM_SYMBOLIZER_PATH=${CLANG_PATH}/llvm-symbolizer
-export ASAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH}
-export MSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER_PATH}
-
-
 # Stage 2 / Memory Sanitizer
 
-echo @@@BUILD_STEP build libcxx/msan@@@
-if [ ! -d libcxx_build_msan ]; then
-  mkdir libcxx_build_msan
-fi
-
-(cd libcxx_build_msan && \
-  cmake \
-    ${CMAKE_STAGE2_COMMON_OPTIONS} \
-    -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \
-    $LLVM && \
-  ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP build clang/msan@@@
-if [ ! -d llvm_build_msan ]; then
-  mkdir llvm_build_msan
-fi
-
-MSAN_LDFLAGS="-lc++abi -Wl,--rpath=${ROOT}/libcxx_build_msan/lib -L${ROOT}/libcxx_build_msan/lib"
-# See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264
-CMAKE_BUG_WORKAROUND_CFLAGS="$MSAN_LDFLAGS -fsanitize=memory -w"
-MSAN_CFLAGS="-I${ROOT}/libcxx_build_msan/include -I${ROOT}/libcxx_build_msan/include/c++/v1 $CMAKE_BUG_WORKAROUND_CFLAGS"
-
-(cd llvm_build_msan && \
- cmake ${CMAKE_STAGE2_COMMON_OPTIONS} \
-   -DLLVM_USE_SANITIZER=${MEMORY_SANITIZER_KIND} \
-   -DLLVM_ENABLE_LIBCXX=ON \
-   -DCMAKE_C_FLAGS="${MSAN_CFLAGS}" \
-   -DCMAKE_CXX_FLAGS="${MSAN_CFLAGS}" \
-   -DCMAKE_EXE_LINKER_FLAGS="${MSAN_LDFLAGS}" \
-   $LLVM && \
- ninja clang) || echo @@@STEP_FAILURE@@@
-
-echo @@@BUILD_STEP check-llvm msan@@@
-
-(cd llvm_build_msan && ninja check-llvm) || echo @@@STEP_WARNINGS@@@
-
-echo @@@BUILD_STEP check-clang msan@@@
-
-(cd llvm_build_msan && ninja check-clang) || echo @@@STEP_FAILURE@@@
+build_stage2_msan
 
+check_stage2_msan
 
 # Stage 2 / AddressSanitizer
 
-echo @@@BUILD_STEP build clang/asan@@@
-
-# Turn on init-order checker as ASan runtime option.
-export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1"
-CMAKE_ASAN_OPTIONS=" \
-  ${CMAKE_STAGE2_COMMON_OPTIONS} \
-  -DLLVM_USE_SANITIZER=Address \
-  "
-
-if [ ! -d llvm_build_asan ]; then
-  mkdir llvm_build_asan
-fi
-
-(cd llvm_build_asan && \
- cmake ${CMAKE_ASAN_OPTIONS} $LLVM && \
- ninja clang) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP check-llvm asan@@@
-
-(cd llvm_build_asan && ninja check-llvm) || echo @@@STEP_FAILURE@@@
-
-
-echo @@@BUILD_STEP check-clang asan@@@
+build_stage2_asan
 
-(cd llvm_build_asan && ninja check-clang) || echo @@@STEP_FAILURE@@@
+check_stage2_asan

zorg/buildbot/builders/sanitizers/buildbot_functions.sh

function gclient_runhooks {
   export GYP_GENERATORS=ninja
   export CLANG_BIN=$CLANG_BUILD/bin
   export CC="$CLANG_BIN/clang"
   export CXX="$CLANG_BIN/clang++"
   
   gclient runhooks
   )
 }
+
+function build_stage1_clang {
+  type -a gcc
+  type -a g++
+  mkdir -p ${STAGE1_DIR}
+  # Build cxx/cxxabi to fool the compiler check in MSan stage 2.
+  cmake_stage1_options="${CMAKE_COMMON_OPTIONS}"
+  (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && \
+    ninja clang compiler-rt llvm-symbolizer)
+}
+
+function common_stage2_variables {
+  local stage1_clang_path=$ROOT/${STAGE1_DIR}/bin
+  cmake_stage2_common_options="\
+    ${CMAKE_COMMON_OPTIONS} \
+    -DCMAKE_C_COMPILER=${stage1_clang_path}/clang \
+    -DCMAKE_CXX_COMPILER=${stage1_clang_path}/clang++ \
+    "
+  llvm_symbolizer_path=${stage1_clang_path}/llvm-symbolizer
+}
+
+function build_stage2_msan {
+  echo @@@BUILD_STEP build libcxx/msan@@@
+  
+  common_stage2_variables
+  export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}"
+  
+  memory_sanitizer_kind="Memory"
+  BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-}
+  if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then
+      memory_sanitizer_kind="MemoryWithOrigins"
+  fi
+
+  mkdir -p ${STAGE2_LIBCXX_MSAN_DIR}
+  (cd ${STAGE2_LIBCXX_MSAN_DIR} && \
+    cmake \
+      ${cmake_stage2_common_options} \
+      -DLLVM_USE_SANITIZER=${memory_sanitizer_kind} \
+      $LLVM && \
+    ninja cxx cxxabi) || echo @@@STEP_FAILURE@@@
+
+  echo @@@BUILD_STEP build clang/msan@@@
+
+  local msan_ldflags="-lc++abi -Wl,--rpath=${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/lib -L${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/lib"
+  # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264
+  local cmake_bug_workaround_cflags="$msan_ldflags -fsanitize=memory -w"
+  local msan_cflags="-I${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/include -I${ROOT}/${STAGE2_LIBCXX_MSAN_DIR}/include/c++/v1 $cmake_bug_workaround_cflags"
+  mkdir -p ${STAGE2_MSAN_DIR}
+  (cd ${STAGE2_MSAN_DIR} && \
+   cmake ${cmake_stage2_common_options} \
+     -DLLVM_USE_SANITIZER=${memory_sanitizer_kind} \
+     -DLLVM_ENABLE_LIBCXX=ON \
+     -DCMAKE_C_FLAGS="${msan_cflags}" \
+     -DCMAKE_CXX_FLAGS="${msan_cflags}" \
+     -DCMAKE_EXE_LINKER_FLAGS="${msan_ldflags}" \
+     $LLVM && \
+   ninja clang) || echo @@@STEP_FAILURE@@@
+}
+
+function build_stage2_asan {
+  echo @@@BUILD_STEP build clang/asan@@@
+
+  common_stage2_variables
+  # Turn on init-order checker as ASan runtime option.
+  export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}"
+  export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1"
+  local cmake_asan_options=" \
+    ${cmake_stage2_common_options} \
+    -DLLVM_USE_SANITIZER=Address \
+    "
+  mkdir -p ${STAGE2_ASAN_DIR}
+  (cd ${STAGE2_ASAN_DIR} && \
+   cmake ${cmake_asan_options} $LLVM && \
+   ninja clang) || echo @@@STEP_FAILURE@@@
+}
+
+function build_stage2_ubsan {
+  echo @@@BUILD_STEP build clang/ubsan@@@
+
+  common_stage2_variables
+  export UBSAN_OPTIONS="external_symbolizer_path=${llvm_symbolizer_path}:print_stacktrace=1"
+  local cmake_ubsan_options=" \
+    ${cmake_stage2_common_options} \
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DLLVM_USE_SANITIZER=Undefined \
+    "
+  mkdir -p ${STAGE2_UBSAN_DIR}
+  (cd ${STAGE2_UBSAN_DIR} &&
+    cmake ${cmake_ubsan_options} $LLVM && \
+    ninja clang) || echo @@@STEP_FAILURE@@@
+}
+
+function check_stage2 {
+  local sanitizer_name=$1
+  local build_dir=$2
+  echo @@@BUILD_STEP check-llvm ${sanitizer_name}@@@
+
+  (cd ${build_dir} && ninja check-llvm) || echo @@@STEP_WARNINGS@@@
+
+  echo @@@BUILD_STEP check-clang ${sanitizer_name}@@@
+
+  (cd ${build_dir} && ninja check-clang) || echo @@@STEP_FAILURE@@@
+}
+
+function check_stage2_msan {
+  check_stage2 msan "${STAGE2_MSAN_DIR}"
+}
+
+function check_stage2_asan {
+  check_stage2 asan "${STAGE2_ASAN_DIR}"
+}
+
+function check_stage2_ubsan {
+  check_stage2 ubsan "${STAGE2_UBSAN_DIR}"
+}