[hwasan] support hwasan-match-all-tag flag for callback memory access instrumentation

[Enna1]

Summary

Currently, hwasan-match-all-tag flag is supported in inline memory access instrumentation and outline memory access instrumentation, but not supported in callback memory access instrumentation.

• For inline memory access instrumentation: a hwasan-match-all-tag check is added following the tag-mismtach check, if tag from pointer is mismatched with tag from shadow memory and tag from pointer is not equal with hwasan-match-all-tag, then a tag-mismatch will be report.
• For outline memory acess instrumentation: MatchAllTag is encoded in AccessInfo, when emit HWASAN memaccess symbols, asm-printer emits assembly instructions to check if tag from pointer is equal with hwasan-match-all-tag.
• For callback memory access instrumentation: hwasan-match-all-tag check is not implemented in __hwasan_load/__hwasan_store.

This patch implements a set of callback functions: __hwasan_[load|store][1|2|4|8|16|n]_match_all and __hwasan_load[load|store][1|2|4|8|16|n]_match_all_noabort, making hwasan-match-all-tag flag working for callback memory access instrumentation.

[Enna1]

What do you think about the two approaches. If the first approach is prefered, I'm happy to update this patch using the first approach :)

[vitalybuka]

Do we need this feature outside kernel?

In D149580#4309534, @Enna1 wrote:

What do you think about the two approaches. If the first approach is prefered, I'm happy to update this patch using the first approach :)

It's better to handler them in hwasan_load/hwasan_store
However we don't want additional argument when we don't need them. So we need a new hwasan_load_match_all/hwasan_store_match_all used with the feature.

[Enna1]

implement __hwasan_load_match_all/__hwasan_store_match_all and handle hwasan-match-all-tag flag in them.

[Enna1]

update

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

995

Int8ptrTy and one below

[Enna1]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

995

Sorry, I don't get it.
Here we call ConstantInt::get(IntptrTy, 8) to create a constant int value 8, O.TypeStoreSize represents size in bits, and O.TypeStoreSize divided by 8 represents size in bytes.
We should not use Int8ptrTy here.

[Enna1]

do not use match-all callback for kernel mode HWASAN

[Enna1]

update

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
   Value *Addr = O.getPtr();
 
   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
 
   if (O.MaybeMask)
     return false; // FIXME
 
   IRBuilder<> IRB(O.getInsn());
+
+  auto checkMatchAllTagForCallback = [&](Value *PtrLong) {
+    if (MatchAllTag.has_value()) {
+      Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, PointerTagShift),
+                                      IRB.getInt8Ty());
+      Value *TagIgnored = IRB.CreateICmpNE(
+          PtrTag, ConstantInt::get(PtrTag->getType(), *MatchAllTag));
+      Instruction *NotTagIgnoredTerm = SplitBlockAndInsertIfThen(
+          TagIgnored, O.getInsn(), false,
+          MDBuilder(*C).createBranchWeights(1, 100000));
+      IRB.SetInsertPoint(NotTagIgnoredTerm);
+    }
+  };
+
   if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(O.TypeStoreSize) &&
       (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
       (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
        *O.Alignment >= O.TypeStoreSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize);
     if (InstrumentWithCalls) {
+      Value *PtrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+      checkMatchAllTagForCallback(PtrLong);
       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
-                     IRB.CreatePointerCast(Addr, IntptrTy));
+                     PtrLong);
     } else if (OutlinedChecks) {
       instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
     } else {
       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
     }
   } else {
-    IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
-                   {IRB.CreatePointerCast(Addr, IntptrTy),
-                    IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy,
-                                                      O.TypeStoreSize),
-                                   ConstantInt::get(IntptrTy, 8))});
+    Value *PtrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+    checkMatchAllTagForCallback(PtrLong);
+    IRB.CreateCall(
+        HwasanMemoryAccessCallbackSized[O.IsWrite],
+        {PtrLong, IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
   }
   untagPointerOperand(O.getInsn(), Addr);
 
   return true;
 }
 
 void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
                                    size_t Size) {

llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll

 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT
 ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,MATCH-ALL-TAG
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,MATCH-ALL-TAG
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_load1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i8, ptr %a
 ; CHECK: ret i8 %[[B]]
 
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_load2(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i16, ptr %a
 ; CHECK: ret i16 %[[B]]
 
 entry:
   %b = load i16, ptr %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_load4(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i32, ptr %a
 ; CHECK: ret i32 %[[B]]
 
 entry:
   %b = load i32, ptr %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_load8(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i64, ptr %a
 ; CHECK: ret i64 %[[B]]
 
 entry:
   %b = load i64, ptr %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_load16(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i128, ptr %a
 ; CHECK: ret i128 %[[B]]
 
 entry:
   %b = load i128, ptr %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
 ; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, ptr %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_store1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 ; CHECK: store i8 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i8 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_store2(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]])
 ; CHECK: store i16 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i16 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_store4(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]])
 ; CHECK: store i32 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i32 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_store8(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]])
 ; CHECK: store i64 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i64 %b, ptr %a, align 8
   ret void
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_store16(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]])
 ; CHECK: store i128 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i128 %b, ptr %a, align 16
   ret void
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0
+; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}}
 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
 ; CHECK: store i40 %b, ptr %a
 ; CHECK: ret void
 
 entry:
   store i40 %b, ptr %a, align 4
   ret void