[sanitizer] Test HWASan + LAM via QEMU.

[morehouse]

Summary

Uses LAM-enabled full system QEMU emulation to test HWASan's LAM
support.

Depends on:
https://github.com/google/sanitizers/pull/1408
https://github.com/google/sanitizers/pull/1410

[morehouse]

Tests pass locally.

Tested manually on a fresh Debian 10 GCE instance, and things are mostly working. The check-hwasan-lam step fails to find lld.

I'll look into the lld issue next week.

[vitalybuka]

zorg/buildbot/builders/sanitizers/buildbot_qemu.sh

112

I also need patched qemu now.
Please rebase and add similar revision comparison to make bot rebuild qemu if remote changed.

115

why you don't fetch submodules?

120

-e does not work in "() ||" case
so you need && or || exit

123

can we make a single function
build_qemu URL <revision> to build both?
The one above is only need --enable-linux-user, "--disable-system --static" can be removed.

Maybe later, not now we can consider to have a single repo/branch with all patches LAM and MTE?

137

need remote revison check

178

Is this available from build_stage1_clang already?
or this is about DLLVM_STATIC_LINK_CXX_STDLIB?

probably stage one can be build with DLLVM_STATIC_LINK_CXX_STDLIB

194

same with revision check
probably we need a function

202

it's going to be huge commandline in lit error reports
maybe make sure that scripts is looking for fixed default locations?

[morehouse]

• Ensure LLD is built prior to testing HWASan.
• Refactor git clone functionality into a helper function.
• Use single function for both QEMU builds.
• Shorten emulator prefix to single env var.

[morehouse]

zorg/buildbot/builders/sanitizers/buildbot_qemu.sh

178

This step imitates the Android bot. I don't think we can get this from stage 1 clang.

IIRC we need both LLVM_BUILD_RUNTIME=OFF and LLVM_STATIC_LINK_CXX_STDLIB=ON to avoid dynamic loader errors inside QEMU.

202

Simplified this down to a single environment variable. I think we still need to specify the ROOT dir since check-hwasan-lam may run the script from a different directory than the ROOT.

Depends on https://github.com/google/sanitizers/commit/709edc4b4d5324e520963a486c2e8d19fbd16a50.

[vitalybuka]

zorg/buildbot/builders/sanitizers/buildbot_qemu.sh

221

would it be more convenient to have as a script argument?

[morehouse]

zorg/buildbot/builders/sanitizers/buildbot_qemu.sh

221

I don't see much difference in convenience either way.

With the current setup environment variables point to prerequisites that are expected to be the same with each run, while arguments change depending on the %run line.

zorg/buildbot/builders/sanitizers/buildbot_qemu.sh

 clobber
 
 buildbot_update
 
 build_stage1_clang
 
 COMPILER_BIN_DIR=$(readlink -f ${STAGE1_DIR})/bin
 
-function build_qemu {
-  echo "@@@BUILD_STEP build qemu@@@"
+function git_clone_at_revision {
+  local src_dir_name="${1}"
+  local git_url="${2}"
+  local revision="${3}"
+  local build_dir="${4}"
   (
-    local qemu_url=https://github.com/vitalybuka/qemu.git
-    cd $ROOT
-    [[ -d qemu ]] || git clone $qemu_url || exit 1
-    cd $ROOT/qemu
-    git remote set-url origin $qemu_url
+    cd ${ROOT}
+    [[ -d ${src_dir_name} ]] || git clone ${git_url} ${src_dir_name} || exit 1
+    cd ${ROOT}/${src_dir_name}
+    git remote set-url origin ${git_url}
     git fetch origin
-    [[ "$(git rev-parse HEAD)" == "$(git rev-parse $1)" ]] && $ROOT/qemu_build/qemu-x86_64 --version && exit 0
 
-    rm -rf $ROOT/qemu_build &&
-    mkdir -p $ROOT/qemu_build &&
-    git reset --hard $1 &&
-    git submodule update --init --recursive &&
-    cd $ROOT/qemu_build &&
-    ../qemu/configure --disable-system --enable-linux-user --static &&
+    # Short circuit if we already have a build at the correct revision.
+    [[ "$(git rev-parse HEAD)" == "$(git rev-parse ${revision})" ]] && exit 0
+
+    # Delete the build dir to ensure a rebuild happens at the new revision.
+    rm -rf ${build_dir}
+
+    git reset --hard ${revision}
+    git submodule update --init --recursive
+  )
+}
+
+function build_qemu {
+  local qemu_name="${1}"
+  local qemu_url="${2}"
+  local qemu_revision="${3}"
+
+  local build_dir="${ROOT}/${qemu_name}_build"
+
+  echo "@@@BUILD_STEP build ${qemu_name}@@@"
+  (
+    git_clone_at_revision ${qemu_name} ${qemu_url} ${qemu_revision} \
+      ${build_dir} || exit 1
+
+    ${build_dir}/qemu-x86_64 --version &&
+      ${build_dir}/qemu-system-x86_64 --version &&
+      ${build_dir}/qemu-img --version &&
+      exit 0
+
+    rm -rf ${build_dir} &&
+    mkdir -p ${build_dir} &&
+    cd ${build_dir} &&
+    ${ROOT}/${qemu_name}/configure --enable-linux-user &&
     ninja &&
-    $ROOT/qemu_build/qemu-x86_64 --version
+    ${build_dir}/qemu-x86_64 --version &&
+    ${build_dir}/qemu-system-x86_64 --version &&
+    ${build_dir}/qemu-img --version
   ) || (
     echo "@@@STEP_EXCEPTION@@@"
     exit 2
   )
 }
 
-build_qemu origin/sanitizer_bot
+function build_lam_linux {
+  local build_dir="${ROOT}/lam_linux_build"
+
+  echo "@@@BUILD_STEP build lam linux@@@"
+  (
+    git_clone_at_revision lam_linux https://github.com/morehouse/linux.git \
+      origin/lam ${build_dir} || exit 1
 
-BUILDS=
+    ls ${build_dir}/arch/x86_64/boot/bzImage && exit 0
+
+    rm -rf ${build_dir} &&
+    mkdir -p ${build_dir} &&
+    cd ${ROOT}/lam_linux &&
+    make O=${build_dir} LD=ld.bfd defconfig &&
+    make O=${build_dir} LD=ld.bfd -j $(nproc) &&
+    ls ${build_dir}/arch/x86_64/boot/bzImage
+  ) || (
+    echo "@@@STEP_EXCEPTION@@@"
+    exit 2
+  )
+}
 
-function configure_compiler_rt {
+build_qemu qemu https://github.com/vitalybuka/qemu.git origin/sanitizer_bot
+build_qemu lam_qemu https://github.com/morehouse/qemu.git origin/lam
+build_lam_linux
+
+SCUDO_BUILDS=
+
+function configure_scudo_compiler_rt {
   local arch=$1
   local target="${arch}-linux-gnu${2:-}"
 
   local name=""
   if [[ "$DBG" == "ON" ]] ; then
     name=debug_
   fi
   name+="${arch}"
   
   local qemu_cmd=""
   if [[ "${QEMU:-}" != "0" ]] ; then
     name+="_qemu"
     local qemu_arch=${arch}
     [[ "${arch}" == "powerpc64le" ]] && qemu_arch="ppc64le"
     qemu_cmd="$ROOT/qemu_build/qemu-${qemu_arch} -L /usr/${target}"
     if [[ ! -z "${QEMU_CPU:-}" ]] ; then
       qemu_cmd+=" -cpu ${QEMU_CPU}"
       name+="_${QEMU_CPU}"
     fi
   fi
 
   local out_dir=llvm_build2_${name}
   rm -rf ${out_dir}
   mkdir -p ${out_dir}
 
-  BUILDS+=" ${name}"
+  SCUDO_BUILDS+=" ${name}"
 
   (
     cd ${out_dir}
 
     LINKER_FLAGS=${LINKER_FLAGS:-}
 
     (
       cmake \
         ${CMAKE_COMMON_OPTIONS} \
         -DCOMPILER_RT_DEBUG=$DBG \
         -DLLVM_CONFIG_PATH=${COMPILER_BIN_DIR}/llvm-config \
(
         -DCMAKE_C_COMPILER_TARGET=${target} \
         -DCMAKE_CXX_COMPILER_TARGET=${target} \
         -DCOMPILER_RT_EMULATOR="${qemu_cmd:-}" \
         $LLVM/../compiler-rt
      ) >& configure.log
   ) &
 }
 
-function run_tests {
+readonly LLVM_SYMBOLIZER_DIR="${ROOT}/llvm_build2_x86_64_symbolizer"
+
+function configure_llvm_symbolizer {
+  rm -rf ${LLVM_SYMBOLIZER_DIR}
+  mkdir -p ${LLVM_SYMBOLIZER_DIR}
+
+  (
+    cd ${LLVM_SYMBOLIZER_DIR}
+
+    (
+      cmake \
+        ${CMAKE_COMMON_OPTIONS} \
+        -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \
+        -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \
+        -DLLVM_BUILD_RUNTIME=OFF \
+        -DLLVM_STATIC_LINK_CXX_STDLIB=ON \
+        $LLVM
+     ) >& configure.log
+  ) &
+}
+
+function configure_hwasan_lam {
+  git_clone_at_revision sanitizers https://github.com/google/sanitizers.git \
+    origin/master ""
+  local script="${ROOT}/sanitizers/hwaddress-sanitizer/run_in_qemu_with_lam.sh"
+  ls ${script}
+
+  # Ensure the buildbot start script created a QEMU image for us.
+  ls /b/qemu_image/debian.img
+  ls /b/qemu_image/debian.id_rsa
+
+  # Link to the QEMU image from ROOT, as the QEMU script expects it there.
+  ln -sf /b/qemu_image ${ROOT}/qemu_image
+
+  local out_dir=llvm_build2_x86_64_lam_qemu
+  rm -rf ${out_dir}
+  mkdir -p ${out_dir}
+
+  (
+    cd ${out_dir}
+
+    (
+      cmake \
+        ${CMAKE_COMMON_OPTIONS} \
+        -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld;libcxx;libcxxabi" \
+        -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \
+        -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \
+        -DLLVM_ENABLE_LLD=ON \
+        -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+        -DLLVM_LIT_ARGS="-v --time-tests" \
+        -DCOMPILER_RT_EMULATOR="env ROOT=${ROOT} ${script}" \
+        $LLVM
+     ) >& configure.log
+  ) &
+}
+
+function run_scudo_tests {
   local name="${1}"
   local out_dir=llvm_build2_${name}
 
   echo "@@@BUILD_STEP scudo $name@@@"
 
   (
     cd ${out_dir}
 
     cat configure.log
 
     # Copy into clang resource dir to make -fsanitize= work in lit tests.
     ninja install-scudo_standalone
 
     ninja check-scudo_standalone || exit 3
   ) || echo "@@@STEP_FAILURE@@@"
 }
 
+function build_llvm_symbolizer {
+  echo "@@@BUILD_STEP llvm-symbolizer x86_64@@@"
+
+  (
+    cd ${LLVM_SYMBOLIZER_DIR}
+
+    cat configure.log
+
+    ninja llvm-symbolizer || exit 3
+  ) || echo "@@@STEP_FAILURE@@@"
+}
+
+function run_hwasan_lam_tests {
+  local name="x86_64_lam_qemu"
+  local out_dir=llvm_build2_${name}
+
+  echo "@@@BUILD_STEP hwasan ${name}@@@"
+
+  (
+    cd ${out_dir}
+
+    cat configure.log
+
+    # LLD must be built first since HWASan tests use -fuse-ld=lld and the
+    # buildbots don't have LLD preinstalled.
+    ninja lld || exit 3
+
+    ninja check-hwasan-lam || exit 3
+  ) || echo "@@@STEP_FAILURE@@@"
+}
+
 echo "@@@BUILD_STEP configure@@@"
 
 for DBG in OFF ON ; do
-  QEMU=0 configure_compiler_rt x86_64
-  configure_compiler_rt x86_64
-  configure_compiler_rt arm eabihf
-  configure_compiler_rt aarch64
-  QEMU_CPU="cortex-a72" configure_compiler_rt aarch64
+  QEMU=0 configure_scudo_compiler_rt x86_64
+  configure_scudo_compiler_rt x86_64
+  configure_scudo_compiler_rt arm eabihf
+  configure_scudo_compiler_rt aarch64
+  QEMU_CPU="cortex-a72" configure_scudo_compiler_rt aarch64
   (
     LINKER_FLAGS="-latomic -Wl,-z,notext -Wno-unused-command-line-argument"
-    configure_compiler_rt mips
-    configure_compiler_rt mipsel
-    configure_compiler_rt mips64 abi64
-    configure_compiler_rt mips64el abi64
+    configure_scudo_compiler_rt mips
+    configure_scudo_compiler_rt mipsel
+    configure_scudo_compiler_rt mips64 abi64
+    configure_scudo_compiler_rt mips64el abi64
   )
-  
-  configure_compiler_rt powerpc64le
+  configure_scudo_compiler_rt powerpc64le
 done
+configure_llvm_symbolizer
+configure_hwasan_lam
 
 wait
 
-for B in $BUILDS ; do
-  run_tests $B
-done 
-
+for B in $SCUDO_BUILDS ; do
+  run_scudo_tests $B
+done
+build_llvm_symbolizer
+run_hwasan_lam_tests