[hwasan] support hwasan-match-all-tag flag for callback memory access instrumentation

[Enna1]

Summary

Currently, hwasan-match-all-tag flag is supported in inline memory access instrumentation and outline memory access instrumentation, but not supported in callback memory access instrumentation.

• For inline memory access instrumentation: a hwasan-match-all-tag check is added following the tag-mismtach check, if tag from pointer is mismatched with tag from shadow memory and tag from pointer is not equal with hwasan-match-all-tag, then a tag-mismatch will be report.
• For outline memory acess instrumentation: MatchAllTag is encoded in AccessInfo, when emit HWASAN memaccess symbols, asm-printer emits assembly instructions to check if tag from pointer is equal with hwasan-match-all-tag.
• For callback memory access instrumentation: hwasan-match-all-tag check is not implemented in __hwasan_load/__hwasan_store.

This patch implements a set of callback functions: __hwasan_[load|store][1|2|4|8|16|n]_match_all and __hwasan_load[load|store][1|2|4|8|16|n]_match_all_noabort, making hwasan-match-all-tag flag working for callback memory access instrumentation.

[Enna1]

What do you think about the two approaches. If the first approach is prefered, I'm happy to update this patch using the first approach :)

[vitalybuka]

Do we need this feature outside kernel?

In D149580#4309534, @Enna1 wrote:

What do you think about the two approaches. If the first approach is prefered, I'm happy to update this patch using the first approach :)

It's better to handler them in hwasan_load/hwasan_store
However we don't want additional argument when we don't need them. So we need a new hwasan_load_match_all/hwasan_store_match_all used with the feature.

[Enna1]

implement __hwasan_load_match_all/__hwasan_store_match_all and handle hwasan-match-all-tag flag in them.

[Enna1]

update

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

999

Int8ptrTy and one below

[Enna1]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

999

Sorry, I don't get it.
Here we call ConstantInt::get(IntptrTy, 8) to create a constant int value 8, O.TypeStoreSize represents size in bits, and O.TypeStoreSize divided by 8 represents size in bytes.
We should not use Int8ptrTy here.

[Enna1]

do not use match-all callback for kernel mode HWASAN

[Enna1]

update

compiler-rt/lib/hwasan/hwasan.cpp

 }
 void __hwasan_load8_noabort(uptr p) {
   CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
 }
 void __hwasan_load16_noabort(uptr p) {
   CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
 }
 
+void __hwasan_loadN_match_all(uptr p, uptr sz, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
+}
+
+void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
+}
+
 void __hwasan_storeN(uptr p, uptr sz) {
   CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
 }
 void __hwasan_store1(uptr p) {
   CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
 }
 void __hwasan_store2(uptr p) {
   CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);

 }
 void __hwasan_store8_noabort(uptr p) {
   CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
 }
 void __hwasan_store16_noabort(uptr p) {
   CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
 }
 
+void __hwasan_storeN_match_all(uptr p, uptr sz, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_match_all(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
+}
+
+void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_match_all_noabort(uptr p, u8 match_all_tag) {
+  if (GetTagFromPointer(p) != match_all_tag)
+    CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
+}
+
 void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
   TagMemoryAligned(UntagAddr(p), sz, tag);
 }
 
 uptr __hwasan_tag_pointer(uptr p, u8 tag) {
   return AddTagToPointer(p, tag);
 }
 

compiler-rt/lib/hwasan/hwasan_interface_internal.h

 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load4_noabort(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load8_noabort(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load16_noabort(uptr);
 
 SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN_match_all(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_match_all(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN_match_all_noabort(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_match_all_noabort(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_storeN(uptr, uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store1(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store2(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store4(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE

 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store4_noabort(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store8_noabort(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store16_noabort(uptr);
 
 SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN_match_all(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_match_all(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN_match_all_noabort(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_match_all_noabort(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_tag_memory(uptr p, u8 tag, uptr sz);
 
 SANITIZER_INTERFACE_ATTRIBUTE
 uptr __hwasan_tag_pointer(uptr p, u8 tag);
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_tag_mismatch(uptr addr, u8 ts);
 

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

private:
   bool Recover;
   bool OutlinedChecks;
   bool UseShortGranules;
   bool InstrumentLandingPads;
   bool InstrumentWithCalls;
   bool InstrumentStack;
   bool DetectUseAfterScope;
   bool UsePageAliases;
+  bool UseMatchAllCallback;
 
   std::optional<uint8_t> MatchAllTag;
 
   unsigned PointerTagShift;
   uint64_t TagMaskByte;
 
   Function *HwasanCtorFunction;
 
void HWAddressSanitizer::initializeModule() {
 
   if (ClMatchAllTag.getNumOccurrences()) {
     if (ClMatchAllTag != -1) {
       MatchAllTag = ClMatchAllTag & 0xFF;
     }
   } else if (CompileKernel) {
     MatchAllTag = 0xFF;
   }
+  UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
 
   // If we don't have personality function support, fall back to landing pads.
   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
                               ? ClInstrumentLandingPads
                               : !NewRuntime;
 
   if (!CompileKernel) {
     createHwasanCtorComdat();
void HWAddressSanitizer::initializeModule() {
   }
 }
 
 void HWAddressSanitizer::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
     const std::string TypeStr = AccessIsWrite ? "store" : "load";
     const std::string EndingStr = Recover ? "_noabort" : "";
+    const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
+
+    FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
+        *HwasanMemoryAccessCallbackFnTy;
+    if (UseMatchAllCallback) {
+      HwasanMemoryAccessCallbackSizedFnTy =
+          FunctionType::get(VoidTy, {IntptrTy, IntptrTy, Int8Ty}, false);
+      HwasanMemoryAccessCallbackFnTy =
+          FunctionType::get(VoidTy, {IntptrTy, Int8Ty}, false);
+    } else {
+      HwasanMemoryAccessCallbackSizedFnTy =
+          FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false);
+      HwasanMemoryAccessCallbackFnTy =
+          FunctionType::get(VoidTy, {IntptrTy}, false);
+    }
 
     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
-        ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
-        FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false));
+        ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
+        HwasanMemoryAccessCallbackSizedFnTy);
 
     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
          AccessSizeIndex++) {
       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
           M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr +
-                                    itostr(1ULL << AccessSizeIndex) + EndingStr,
-                                FunctionType::get(VoidTy, {IntptrTy}, false));
+                                    itostr(1ULL << AccessSizeIndex) +
+                                    MatchAllStr + EndingStr,
+                                HwasanMemoryAccessCallbackFnTy);
     }
   }
 
   HwasanTagMemoryFunc = M.getOrInsertFunction("__hwasan_tag_memory", VoidTy,
                                               Int8PtrTy, Int8Ty, IntptrTy);
   HwasanGenerateTagFunc =
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
 
bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
 
   IRBuilder<> IRB(O.getInsn());
   if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(O.TypeStoreSize) &&
       (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
       (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
        *O.Alignment >= O.TypeStoreSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize);
     if (InstrumentWithCalls) {
+      if (UseMatchAllCallback) {
+        IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
+                       {IRB.CreatePointerCast(Addr, IntptrTy),
+                        ConstantInt::get(Int8Ty, *MatchAllTag)});
+      } else {
         IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
                        IRB.CreatePointerCast(Addr, IntptrTy));
+      }
     } else if (OutlinedChecks) {
       instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
     } else {
       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
     }
   } else {
-    IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
-                   {IRB.CreatePointerCast(Addr, IntptrTy),
-                    IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy,
-                                                      O.TypeStoreSize),
-                                   ConstantInt::get(IntptrTy, 8))});
+    if (UseMatchAllCallback) {
+      IRB.CreateCall(
+          HwasanMemoryAccessCallbackSized[O.IsWrite],
+          {IRB.CreatePointerCast(Addr, IntptrTy),
+           IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
+                          ConstantInt::get(IntptrTy, 8)),
+           ConstantInt::get(Int8Ty, *MatchAllTag)});
+    } else {
+      IRB.CreateCall(
+          HwasanMemoryAccessCallbackSized[O.IsWrite],
+           IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
     }
+  }
   untagPointerOperand(O.getInsn(), Addr);
 
   return true;
 }
 
 void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
                                    size_t Size) {
   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());

llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll

 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT
 ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-ABORT
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_load1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load1_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load1_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i8, ptr %a
 ; CHECK: ret i8 %[[B]]
 
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_load2(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load2_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load2_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i16, ptr %a
 ; CHECK: ret i16 %[[B]]
 
 entry:
   %b = load i16, ptr %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_load4(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load4_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load4_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i32, ptr %a
 ; CHECK: ret i32 %[[B]]
 
 entry:
   %b = load i32, ptr %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_load8(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load8_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load8_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i64, ptr %a
 ; CHECK: ret i64 %[[B]]
 
 entry:
   %b = load i64, ptr %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_load16(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load16_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load16_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i128, ptr %a
 ; CHECK: ret i128 %[[B]]
 
 entry:
   %b = load i128, ptr %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_loadN_match_all(i64 %[[A]], i64 5, i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_loadN_match_all_noabort(i64 %[[A]], i64 5, i8 0)
 ; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, ptr %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_store1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store1_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store1_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: store i8 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i8 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_store2(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store2_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store2_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: store i16 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i16 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_store4(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store4_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store4_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: store i32 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i32 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_store8(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store8_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store8_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: store i64 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i64 %b, ptr %a, align 8
   ret void
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_store16(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store16_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store16_match_all_noabort(i64 %[[A]], i8 0)
 ; CHECK: store i128 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i128 %b, ptr %a, align 16
   ret void
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_storeN_match_all(i64 %[[A]], i64 5, i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_storeN_match_all_noabort(i64 %[[A]], i64 5, i8 0)
 ; CHECK: store i40 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i40 %b, ptr %a, align 4
   ret void
 }