[HWAsan][NewPM] Handle hwasan like other sanitizers

[aeubanks]

Summary

Move it as an EP callback (-O[123]) or in addSanitizersAtO0.

This makes it not run in ThinLTO pre-link (like the other sanitizers),
so don't check LTO runs in hwasan-new-pm.c. Changing its position also
seems to change the generated IR. I think we just need to make sure the
pass runs.

[leonardchan]

To what extent does the IR change? My only concern is the right codegen is still produced.

[aeubanks]

diff for hwasan + -O1:

14a15

@__hwasan_shadow = external global [0 x i8]

72a74,109

declare void @__hwasan_loadN(i64, i64)

declare void @__hwasan_load1(i64)

declare void @__hwasan_load2(i64)

declare void @__hwasan_load4(i64)

declare void @__hwasan_load8(i64)

declare void @__hwasan_load16(i64)

declare void @__hwasan_storeN(i64, i64)

declare void @__hwasan_store1(i64)

declare void @__hwasan_store2(i64)

declare void @__hwasan_store4(i64)

declare void @__hwasan_store8(i64)

declare void @__hwasan_store16(i64)

declare void @__hwasan_tag_memory(i8*, i8, i64)

declare i8 @__hwasan_generate_tag()

declare i8* @__hwasan_memmove(i8*, i8*, i64)

declare i8* @__hwasan_memcpy(i8*, i8*, i64)

declare i8* @__hwasan_memset(i8*, i32, i64)

declare void @__hwasan_handle_vfork(i64)

diff for khwasan + -O1:

7a8

@__hwasan_shadow = external global [0 x i8]

34,35c35,36
< call void asm sideeffect "int3\0Anopl 66(%rax)", "{rdi}"(i64 %4)

< unreachable

call void asm sideeffect "int3\0Anopl 98(%rax)", "{rdi}"(i64 %4)
br label %27

51c52

< 27: ; preds = %22

27: ; preds = %16, %22

60a62,97

declare void @__hwasan_loadN_noabort(i64, i64)

declare void @__hwasan_load1_noabort(i64)

declare void @__hwasan_load2_noabort(i64)

declare void @__hwasan_load4_noabort(i64)

declare void @__hwasan_load8_noabort(i64)

declare void @__hwasan_load16_noabort(i64)

declare void @__hwasan_storeN_noabort(i64, i64)

declare void @__hwasan_store1_noabort(i64)

declare void @__hwasan_store2_noabort(i64)

declare void @__hwasan_store4_noabort(i64)

declare void @__hwasan_store8_noabort(i64)

declare void @__hwasan_store16_noabort(i64)

declare void @__hwasan_tag_memory(i8*, i8, i64)

declare i8 @__hwasan_generate_tag()

declare i8* @memmove(i8*, i8*, i64)

declare i8* @memcpy(i8*, i8*, i64)

declare i8* @memset(i8*, i32, i64)

Looks like the IR is the (mostly) same, just some declarations cleaned up.

[leonardchan]

I see. LGTM

clang/lib/CodeGen/BackendUtil.cpp

static void addSanitizersAtO0(ModulePassManager &MPM,
   if (LangOpts.Sanitize.has(SanitizerKind::Address)) {
     ASanPass(SanitizerKind::Address, /*CompileKernel=*/false);
   }
 
   if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) {
     ASanPass(SanitizerKind::KernelAddress, /*CompileKernel=*/true);
   }
 
+  if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) {
+    bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress);
+    MPM.addPass(HWAddressSanitizerPass(
+        /*CompileKernel=*/false, Recover));
+  }
+  if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) {
+    MPM.addPass(HWAddressSanitizerPass(
+        /*CompileKernel=*/true, /*Recover=*/true));
+  }
+
   if (LangOpts.Sanitize.has(SanitizerKind::Memory)) {
     bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::Memory);
     int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins;
     MPM.addPass(MemorySanitizerPass({TrackOrigins, Recover, false}));
     MPM.addPass(createModuleToFunctionPassAdaptor(
         MemorySanitizerPass({TrackOrigins, Recover, false})));
   }
 
if (CodeGenOpts.OptimizationLevel == 0) {
               MPM.addPass(ModuleAddressSanitizerPass(
                   /*CompileKernel=*/false, Recover, ModuleUseAfterScope,
                   UseOdrIndicator));
               MPM.addPass(
                   createModuleToFunctionPassAdaptor(AddressSanitizerPass(
                       /*CompileKernel=*/false, Recover, UseAfterScope)));
             });
       }
+
+      if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) {
+        bool Recover =
+            CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress);
+        PB.registerOptimizerLastEPCallback(
+            [Recover](ModulePassManager &MPM,
+                      PassBuilder::OptimizationLevel Level) {
+              MPM.addPass(HWAddressSanitizerPass(
+                  /*CompileKernel=*/false, Recover));
+            });
+      }
+      if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) {
+        bool Recover =
+            CodeGenOpts.SanitizeRecover.has(SanitizerKind::KernelHWAddress);
+        PB.registerOptimizerLastEPCallback(
+            [Recover](ModulePassManager &MPM,
+                      PassBuilder::OptimizationLevel Level) {
+              MPM.addPass(HWAddressSanitizerPass(
+                  /*CompileKernel=*/true, Recover));
+            });
+      }
+
       if (Optional<GCOVOptions> Options = getGCOVOptions(CodeGenOpts, LangOpts))
         PB.registerPipelineStartEPCallback([Options](ModulePassManager &MPM) {
           MPM.addPass(GCOVProfilerPass(*Options));
         });
       if (Optional<InstrProfOptions> Options =
               getInstrProfOptions(CodeGenOpts, LangOpts))
         PB.registerPipelineStartEPCallback([Options](ModulePassManager &MPM) {
           MPM.addPass(InstrProfiling(*Options, false));
if (!CodeGenOpts.DisableLLVMPasses) {
     if (CodeGenOpts.UniqueInternalLinkageNames)
       MPM.addPass(UniqueInternalLinkageNamesPass());
 
     if (CodeGenOpts.MemProf) {
       MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
       MPM.addPass(ModuleMemProfilerPass());
     }
 
-    if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) {
-      bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress);
-      MPM.addPass(HWAddressSanitizerPass(
-          /*CompileKernel=*/false, Recover));
-    }
-    if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) {
-      MPM.addPass(HWAddressSanitizerPass(
-          /*CompileKernel=*/true, /*Recover=*/true));
-    }
-
     if (CodeGenOpts.OptimizationLevel == 0) {
       // FIXME: the backends do not handle matrix intrinsics currently. Make
       // sure they are also lowered in O0. A lightweight version of the pass
       // should run in the backend pipeline on demand.
       if (LangOpts.MatrixTypes)
         MPM.addPass(
             createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass()));
 

clang/test/CodeGen/hwasan-new-pm.c

 // Test that HWASan and KHWASan runs with the new pass manager.
-// We run them under different optimizations and LTOs to ensure the IR is still
+// We run them under different optimizations to ensure the IR is still
 // being instrumented properly.
 
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=hwaddress %s | FileCheck %s --check-prefixes=CHECK,HWASAN,HWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=hwaddress -flto %s | FileCheck %s --check-prefixes=CHECK,HWASAN,HWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=hwaddress -flto=thin %s | FileCheck %s --check-prefixes=CHECK,HWASAN,HWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=hwaddress %s | FileCheck %s --check-prefixes=CHECK,HWASAN
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=hwaddress -flto %s | FileCheck %s --check-prefixes=CHECK,HWASAN
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=hwaddress -flto=thin %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=hwaddress %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=hwaddress %s | FileCheck %s
 
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress %s | FileCheck %s --check-prefixes=CHECK,KHWASAN,KHWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress -flto %s | FileCheck %s --check-prefixes=CHECK,KHWASAN,KHWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress -flto=thin %s | FileCheck %s --check-prefixes=CHECK,KHWASAN,KHWASAN-NOOPT
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress %s | FileCheck %s --check-prefixes=CHECK,KHWASAN
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress -flto %s | FileCheck %s --check-prefixes=CHECK,KHWASAN
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress -flto=thin %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -S -emit-llvm -o - -O1 -fexperimental-new-pass-manager -fsanitize=kernel-hwaddress %s | FileCheck %s
 
 int foo(int *a) { return *a; }
 
 // All the cases above mark the function with sanitize_hwaddress.
-// CHECK-DAG: sanitize_hwaddress
-
-// Both sanitizers produce %hwasan.shadow without both thinlto and optimizations.
-// HWASAN-DAG: %hwasan.shadow
-// KHWASAN-DAG: %hwasan.shadow
-
-// Both sanitizers produce __hwasan_tls without both thinlto and optimizations.
-// HWASAN-DAG: __hwasan_tls
-// KHWASAN-DAG: __hwasan_tls
-
-// For unoptimized cases, both sanitizers produce different load functions.
-// HWASAN-NOOPT-DAG: __hwasan_loadN
-// KHWASAN-NOOPT-DAG: __hwasan_loadN_noabort
+// CHECK: sanitize_hwaddress