[gn build] Add support for check-asan

[aeubanks]

Summary

With these changes, check-asan passes on Linux and Windows.

There are a couple libraries we need to add support for, asan_static, asan_preinit, and the shared library version of asan proper.
Since we need to build the asan proper sources twice, once with -DASAN_DYNAMIC and once without, those sources are no longer in a source_set.
Much of the check-asan target is taken from the existing check-hwasan target.

[aeubanks]

update

[aeubanks]

update

[aeubanks]

update

[thakis]

Nice!

llvm/utils/gn/build/BUILD.gn

config("clang_code") {
   include_dirs = [
     "//clang/include",
     "$root_gen_dir/clang/include",
   ]
 }
 
 config("crt_code") {
   include_dirs = [ "//compiler-rt/lib" ]
-  cflags = [
+  cflags = [ "-fno-builtin" ]
+  if (current_os != "win") {
+    cflags += [
       "-fPIC",
       "-funwind-tables",
-    "-gline-tables-only",
       "-fvisibility=hidden",
-    "-fno-builtin",
     ]
+  } else {
+    cflags += [
+      # Disable thread safe initialization for static locals. ASan shouldn't need it.
+      # Thread safe initialization assumes that the CRT has already been initialized, but ASan initializes before the CRT.
+      "/Zc:threadSafeInit-",
+      "-gline-tables-only",
+    ]
+  }
   if (is_clang) {
     cflags += [
       "-Werror=thread-safety",
       "-Werror=thread-safety-reference",
       "-Werror=thread-safety-beta",
     ]
   }
 }

llvm/utils/gn/secondary/BUILD.gn

deps = [
     "//llvm/test",
   ]
   if (current_os == "linux") {
     deps += [
       "//libcxx",
       "//libcxxabi",
     ]
   }
+  if (current_os == "linux" || current_os == "win") {
+    deps += [ "//compiler-rt/test/asan" ]
+  }
   if (current_os == "linux" || current_os == "android") {
     deps += [ "//compiler-rt/test/hwasan" ]
   }
   if (current_os == "linux" || current_os == "mac") {
     deps += [ "//libunwind" ]
   }
 
   testonly = true

llvm/utils/gn/secondary/compiler-rt/lib/BUILD.gn

 group("lib") {
   deps = [
+    "//compiler-rt/lib/asan:ignorelist($host_toolchain)",
     "//compiler-rt/lib/cfi:ignorelist($host_toolchain)",
     "//compiler-rt/lib/profile",
   ]
   if (current_os == "linux") {
     deps += [ "//compiler-rt/lib/msan" ]
   }
   if (current_os == "linux" || current_os == "android") {
     deps += [ "//compiler-rt/lib/ubsan_minimal" ]

llvm/utils/gn/secondary/compiler-rt/lib/asan/BUILD.gn

 import("//compiler-rt/target.gni")
 
-source_set("cxx_sources") {
-  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
-  configs += [ "//llvm/utils/gn/build:crt_code" ]
-  sources = [ "asan_new_delete.cpp" ]
-}
-
-if (current_os == "mac") {
-  asan_target_type = "shared_library"
-} else {
-  asan_target_type = "static_library"
-}
-
-target(asan_target_type, "asan") {
-  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
-  configs += [ "//llvm/utils/gn/build:crt_code" ]
-
-  output_dir = crt_current_out_dir
-  if (current_os == "mac") {
-    output_name = "clang_rt.asan_osx_dynamic"
-  } else {
-    assert(current_os != "win", "FIXME")
-    output_name = "clang_rt.asan$crt_current_target_suffix"
+if (current_toolchain == host_toolchain) {
+  copy("ignorelist") {
+    sources = [ "asan_ignorelist.txt" ]
+    outputs = [ "$clang_resource_dir/share/{{source_target_relative}}" ]
   }
-
-  deps = [
-    "//compiler-rt/lib/interception:sources",
-    "//compiler-rt/lib/lsan:common_sources",
-    "//compiler-rt/lib/sanitizer_common:sources",
-    "//compiler-rt/lib/ubsan:sources",
-  ]
-
-  if (asan_target_type == "static_library") {
-    complete_static_lib = true
-    configs -= [ "//llvm/utils/gn/build:thin_archive" ]
-    deps += [ ":asan_cxx" ]
-  } else {
-    deps += [
-      ":cxx_sources",
-      "//compiler-rt/lib/ubsan:cxx_sources",
-    ]
-    defines = [ "ASAN_DYNAMIC" ]
-  }
-
-  sources = [
+  asan_sources = [
     "asan_activation.cpp",
     "asan_activation.h",
     "asan_activation_flags.inc",
     "asan_allocator.cpp",
     "asan_allocator.h",
     "asan_debugging.cpp",
     "asan_descriptions.cpp",
     "asan_descriptions.h",
asan_sources = [
     "asan_stats.h",
     "asan_suppressions.cpp",
     "asan_suppressions.h",
     "asan_thread.cpp",
     "asan_thread.h",
     "asan_win.cpp",
   ]
   if (target_os != "mac" && target_os != "win") {
-    sources += [ "asan_interceptors_vfork.S" ]
+    asan_sources += [ "asan_interceptors_vfork.S" ]
   }
-
+  config("asan_config") {
     cflags = []
     if (target_os != "win") {
       cflags += [ "-ftls-model=initial-exec" ]
     }
 
     # FIXME: link rt dl m pthread log
     # FIXME: dep on libcxx-headers?
     # FIXME: add_sanitizer_rt_version_list (cf hwasan)
     # FIXME: need libclang_rt.asan*.a.syms?
-  # FIXME: windows flags (-Zl -nodefaultlibs)
-  # FIXME: asan_ignorelist.txt
 
     if (target_os == "android") {
       ldflags = [ "-Wl,-z,global" ]
     }
 
     if (target_os == "mac") {
       # The -U flags below correspond to the add_weak_symbols() calls in CMake.
       ldflags = [
         "-lc++",
         "-lc++abi",
 
         # asan
         "-Wl,-U,___asan_default_options",
         "-Wl,-U,___asan_default_suppressions",
         "-Wl,-U,___asan_on_error",
         "-Wl,-U,___asan_set_shadow_00",
         "-Wl,-U,___asan_set_shadow_f1",
         "-Wl,-U,___asan_set_shadow_f2",
         "-Wl,-U,___asan_set_shadow_f3",
         "-Wl,-U,___asan_set_shadow_f4",
         "-Wl,-U,___asan_set_shadow_f5",
         "-Wl,-U,___asan_set_shadow_f6",
         "-Wl,-U,___asan_set_shadow_f7",
         "-Wl,-U,___asan_set_shadow_f8",
 
         # lsan
         "-Wl,-U,___lsan_default_options",
         "-Wl,-U,___lsan_default_suppressions",
         "-Wl,-U,___lsan_is_turned_off",
 
         # ubsan
         "-Wl,-U,___ubsan_default_options",
 
         # sanitizer_common
         "-Wl,-U,___sanitizer_free_hook",
         "-Wl,-U,___sanitizer_malloc_hook",
         "-Wl,-U,___sanitizer_report_error_summary",
         "-Wl,-U,___sanitizer_sandbox_on_notify",
         "-Wl,-U,___sanitizer_symbolize_code",
         "-Wl,-U,___sanitizer_symbolize_data",
         "-Wl,-U,___sanitizer_symbolize_demangle",
         "-Wl,-U,___sanitizer_symbolize_flush",
         "-Wl,-U,___sanitizer_symbolize_set_demangle",
         "-Wl,-U,___sanitizer_symbolize_set_inline_frames",
 
         # xray
         "-Wl,-U,___start_xray_fn_idx",
         "-Wl,-U,___start_xray_instr_map",
         "-Wl,-U,___stop_xray_fn_idx",
         "-Wl,-U,___stop_xray_instr_map",
 
         # FIXME: better
         "-Wl,-install_name,@rpath/libclang_rt.asan_osx_dynamic.dylib",
       ]
       # FIXME: -Wl,-rpath
       # FIXME: codesign (??)
     }
   }
 
-if (asan_target_type == "static_library") {
+  source_set("cxx_sources") {
+    configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+    configs += [ "//llvm/utils/gn/build:crt_code" ]
+    sources = [ "asan_new_delete.cpp" ]
+  }
+
+  source_set("static_sources") {
+    configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+    configs += [ "//llvm/utils/gn/build:crt_code" ]
+    sources = [ "asan_rtl_static.cpp" ]
+    if (target_os != "mac" && target_os != "win") {
+      sources += [ "asan_rtl_x86_64.S" ]
+    }
+  }
+
+  source_set("preinit_sources") {
+    configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+    configs += [ "//llvm/utils/gn/build:crt_code" ]
+    sources = [ "asan_preinit.cpp" ]
+  }
+
+  shared_library("asan_shared_library") {
+    output_dir = crt_current_out_dir
+    output_name = "clang_rt.asan$crt_current_target_suffix"
+    if (current_os == "win") {
+      output_name = "clang_rt.asan_dynamic$crt_current_target_suffix"
+    }
+    configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+    configs += [ "//llvm/utils/gn/build:crt_code" ]
+    configs += [ ":asan_config" ]
+    sources = asan_sources
+    deps = [
+      ":cxx_sources",
+      "//compiler-rt/lib/interception:sources",
+      "//compiler-rt/lib/lsan:common_sources",
+      "//compiler-rt/lib/sanitizer_common:sources",
+      "//compiler-rt/lib/ubsan:cxx_sources",
+      "//compiler-rt/lib/ubsan:sources",
+    ]
+    defines = [ "ASAN_DYNAMIC" ]
+    if (current_os == "win") {
+      defines += [ "INTERCEPTION_DYNAMIC_CRT" ]
+    }
+  }
+
+  if (current_os != "mac") {
+    static_library("asan_static_library") {
+      output_dir = crt_current_out_dir
+      output_name = "clang_rt.asan$crt_current_target_suffix"
+      complete_static_lib = true
+      configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+      configs += [ "//llvm/utils/gn/build:crt_code" ]
+      configs += [ ":asan_config" ]
+      configs -= [ "//llvm/utils/gn/build:thin_archive" ]
+      sources = asan_sources
+      deps = [
+        ":preinit_sources",
+        "//compiler-rt/lib/interception:sources",
+        "//compiler-rt/lib/lsan:common_sources",
+        "//compiler-rt/lib/sanitizer_common:sources",
+        "//compiler-rt/lib/ubsan:sources",
+      ]
+    }
+
     static_library("asan_cxx") {
-    assert(current_os != "win", "FIXME")
       output_dir = crt_current_out_dir
       output_name = "clang_rt.asan_cxx$crt_current_target_suffix"
       complete_static_lib = true
       configs -= [ "//llvm/utils/gn/build:thin_archive" ]
       deps = [
         ":cxx_sources",
         "//compiler-rt/lib/ubsan:cxx_sources",
       ]
     }
+
+    static_library("asan_static") {
+      output_dir = crt_current_out_dir
+      output_name = "clang_rt.asan_static$crt_current_target_suffix"
+      complete_static_lib = true
+      configs -= [ "//llvm/utils/gn/build:thin_archive" ]
+      deps = [ ":static_sources" ]
     }
 
-# FIXME: Move these to real targets.
-source_set("unused") {
+    static_library("asan_preinit") {
+      output_dir = crt_current_out_dir
+      output_name = "clang_rt.asan-preinit$crt_current_target_suffix"
+      complete_static_lib = true
+      configs -= [ "//llvm/utils/gn/build:thin_archive" ]
+      deps = [ ":preinit_sources" ]
+    }
+  }
+  if (current_os == "win") {
+    static_library("asan_dll_thunk") {
+      output_dir = crt_current_out_dir
+      output_name = "clang_rt.asan_dll_thunk$crt_current_target_suffix"
+      configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+      configs += [ "//llvm/utils/gn/build:crt_code" ]
+      complete_static_lib = true
+      configs -= [ "//llvm/utils/gn/build:thin_archive" ]
       sources = [
-    "asan_preinit.cpp",
+        "asan_globals_win.cpp",
         "asan_win_dll_thunk.cpp",
+      ]
+      defines = [ "SANITIZER_DLL_THUNK" ]
+      deps = [
+        "//compiler-rt/lib/interception:sources",
+        "//compiler-rt/lib/sanitizer_common:dll_thunk",
+        "//compiler-rt/lib/ubsan:dll_thunk",
+      ]
+    }
+    static_library("asan_dynamic_runtime_thunk") {
+      output_dir = crt_current_out_dir
+      output_name =
+          "clang_rt.asan_dynamic_runtime_thunk$crt_current_target_suffix"
+      configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+      configs += [ "//llvm/utils/gn/build:crt_code" ]
+      complete_static_lib = true
+      configs -= [ "//llvm/utils/gn/build:thin_archive" ]
+      sources = [
+        "asan_globals_win.cpp",
         "asan_win_dynamic_runtime_thunk.cpp",
-
-    # FIXME: These are in clang_rt.asan_static in the CMake build.
-    "asan_rtl_static.cpp",
-    "asan_rtl_x86_64.S",
       ]
+      defines = [ "SANITIZER_DYNAMIC_RUNTIME_THUNK" ]
+      deps = [
+        "//compiler-rt/lib/sanitizer_common:dynamic_runtime_thunk",
+        "//compiler-rt/lib/ubsan:dynamic_runtime_thunk",
+      ]
+      cflags = [ "-Zl" ]
+    }
+  }
+  group("asan") {
+    deps = [ ":asan_shared_library" ]
+    if (current_os == "win") {
+      deps += [
+        ":asan_dll_thunk",
+        ":asan_dynamic_runtime_thunk",
+      ]
+    }
+    if (current_os != "mac") {
+      deps += [
+        ":asan_cxx",
+        ":asan_preinit",
+        ":asan_static",
+        ":asan_static_library",
+      ]
+    }
+  }
 }

llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn

sources = [
     "sanitizer_win.cpp",
     "sanitizer_win.h",
     "sanitizer_win_defs.h",
     "sanitizer_win_dll_thunk.h",
     "sanitizer_win_weak_interception.h",
   ]
 }
 
+source_set("dll_thunk") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  sources = [
+    "sanitizer_coverage_win_dll_thunk.cpp",
+    "sanitizer_coverage_win_sections.cpp",
+    "sanitizer_win_dll_thunk.cpp",
+  ]
+  defines = [ "SANITIZER_DLL_THUNK" ]
+}
+
+source_set("dynamic_runtime_thunk") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  sources = [
+    "sanitizer_coverage_win_dynamic_runtime_thunk.cpp",
+    "sanitizer_coverage_win_sections.cpp",
+    "sanitizer_win_dynamic_runtime_thunk.cpp",
+  ]
+  defines = [ "SANITIZER_DYNAMIC_RUNTIME_THUNK" ]
+  cflags = []
+  if (current_os == "win") {
+    cflags += [ "-Zl" ]
+  }
+}
+
 # Libc functions stubs. These sources should be linked instead of
 # SANITIZER_LIBCDEP_SOURCES when sanitizer_common library must not depend on
 # libc. This is currently unreferenced and exists to make
 # sync_source_lists_from_cmake happy.
 source_set("nolibc_sources") {
   configs -= [ "//llvm/utils/gn/build:llvm_code" ]
   configs += [ "//llvm/utils/gn/build:crt_code" ]
   deps = [ "//llvm/utils/gn/build/libs/pthread" ]
   sources = [ "sanitizer_common_nolibc.cpp" ]
 }

llvm/utils/gn/secondary/compiler-rt/lib/ubsan/BUILD.gn

sources = [
     "ubsan_platform.h",
     "ubsan_signals_standalone.h",
     "ubsan_type_hash.h",
     "ubsan_value.cpp",
     "ubsan_value.h",
   ]
 }
 
+source_set("dll_thunk") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  sources = [ "ubsan_win_dll_thunk.cpp" ]
+  defines = [ "SANITIZER_DLL_THUNK" ]
+}
+
+source_set("dynamic_runtime_thunk") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  defines = [ "SANITIZER_DYNAMIC_RUNTIME_THUNK" ]
+  sources = [ "ubsan_win_dynamic_runtime_thunk.cpp" ]
+}
+
 # Unreferenced; at the moment exists to make sync_source_lists_from_cmake happy.
 source_set("standalone_sources") {
   configs -= [ "//llvm/utils/gn/build:llvm_code" ]
   configs -= [ "//llvm/utils/gn/build:no_rtti" ]
   configs += [ "//llvm/utils/gn/build:crt_code" ]
   defines = [ "UBSAN_CAN_USE_CXXABI" ]
   sources = [
     "ubsan_diag_standalone.cpp",

llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn

 import("//compiler-rt/target.gni")
 import("//compiler-rt/test/test.gni")
+import("//llvm/lib/DebugInfo/PDB/enable_dia.gni")
 import("//llvm/triples.gni")
 import("//llvm/utils/gn/build/buildflags.gni")
 import("//llvm/utils/gn/build/libs/zlib/enable.gni")
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 import("//llvm/utils/gn/build/write_cmake_config.gni")
 import("//llvm/version.gni")
 
+ext = ""
+if (host_os == "win") {
+  ext = ".exe"
+}
+
 write_cmake_config("lit_common_configured") {
   input = "lit.common.configured.in"
   output = "$target_gen_dir/lit.common.configured"
 
   values = [
     "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
 
     "COMPILER_RT_DEFAULT_TARGET_TRIPLE=$llvm_current_triple",
values = [
     "LLVM_TOOLS_DIR=" + rebase_path("$root_build_dir/bin"),
     "LLVM_LIBRARY_OUTPUT_INTDIR=" + rebase_path("$root_build_dir/lib"),
 
     "LLVM_LIBCXX_USED=0",
 
     "GOLD_EXECUTABLE=ld",
     "GNU_LD_EXECUTABLE=ld",
     "COMPILER_RT_RESOLVED_TEST_COMPILER=" +
-        rebase_path("$root_build_dir/bin/clang"),
+        rebase_path("$root_build_dir/bin/clang") + ext,
     "COMPILER_RT_TEST_COMPILER_ID=Clang",
     "Python3_EXECUTABLE=$python_path",
     "COMPILER_RT_DEBUG_PYBOOL=False",
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
     "COMPILER_RT_EMULATOR=",
     "COMPILER_RT_ASAN_SHADOW_SCALE=",

llvm/utils/gn/secondary/compiler-rt/test/asan/BUILD.gn

+import("//compiler-rt/target.gni")
+import("//compiler-rt/test/test.gni")
+import("//llvm/lib/Target/targets.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+write_cmake_config("lit_site_cfg") {
+  input = "lit.site.cfg.py.in"
+  output = "$target_gen_dir/lit.site.cfg.py"
+
+  values = [
+    "ASAN_LIT_SOURCE_DIR=" + rebase_path("."),
+    "ASAN_TEST_CONFIG_SUFFIX=$crt_current_target_suffix",
+    "ASAN_TEST_TARGET_CFLAGS=$target_flags_string",
+    "ASAN_TEST_TARGET_CC=",
+    "ASAN_TEST_BITS=64",
+    "ASAN_TEST_APPLE_PLATFORM=",
+    "ASAN_TEST_MIN_DEPLOYMENT_TARGET_FLAG=",
+    "ASAN_TEST_DYNAMIC=0",
+    "ASAN_TEST_TARGET_ARCH=$crt_current_target_arch",
+    "COMPILER_RT_ARM_THUMB=",
+    "COMPILER_RT_BINARY_DIR=" + rebase_path("$root_gen_dir/compiler-rt"),
+    "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+  ]
+}
+
+if (current_toolchain != host_toolchain) {
+  group("asan_toolchain") {
+    deps = [
+      ":lit_site_cfg",
+      "//compiler-rt/include($host_toolchain)",
+      "//compiler-rt/lib/asan",
+      "//compiler-rt/lib/asan:ignorelist($host_toolchain)",
+      "//compiler-rt/lib/profile",
+      "//compiler-rt/test:lit_common_configured",
+      "//llvm/tools/llvm-readobj($host_toolchain)",
+      "//llvm/tools/llvm-symbolizer($host_toolchain)",
+      "//llvm/utils/FileCheck($host_toolchain)",
+      "//llvm/utils/llvm-lit($host_toolchain)",
+      "//llvm/utils/not($host_toolchain)",
+    ]
+  }
+}
+
+supported_toolchains = []
+if (host_os == "linux") {
+  supported_toolchains += [ "//llvm/utils/gn/build/toolchain:stage2_unix" ]
+} else if (host_os == "win") {
+  supported_toolchains += [ "//llvm/utils/gn/build/toolchain:stage2_win" ]
+}
+group("asan") {
+  deps = []
+  foreach(toolchain, supported_toolchains) {
+    deps += [ ":asan_toolchain($toolchain)" ]
+  }
+}
+
+if (supported_toolchains != []) {
+  action("check-asan") {
+    script = "$root_build_dir/bin/llvm-lit"
+    if (host_os == "win") {
+      script += ".py"
+    }
+    args = [ "-sv" ]
+    foreach(toolchain, supported_toolchains) {
+      args += [ rebase_path(
+              get_label_info(":lit_site_cfg($toolchain)", "target_gen_dir"),
+              root_build_dir) ]
+    }
+    outputs = [ "$target_gen_dir/run-lit" ]  # Non-existing, so that ninja runs
+                                             # it each time.
+
+    # Since check-asan is always dirty, //:default doesn't depend on it so
+    # that it's not part of the default ninja target.  Hence, check-asan
+    # shouldn't have any deps except :asan. so that the default target is
+    # sure to build all the deps.
+    deps = [ ":asan" ]
+    testonly = true
+
+    pool = "//:console"
+  }
+}