[hwasan] support hwasan-match-all-tag flag for hwasan meminstrinsic calls

[Enna1]

Summary

This patch implements __hwasan_memset_match_all, __hwasan_memcpy_match_all and __hwasan_memmove_match_all, making hwasan-match-all-tag flag working for hwasan versions of memset, memcpy and memmove.

[Enna1]

D149580 and this patch implement hwasan-match-all-tag flag support for hwasan callback memaccess instrumentation and memset, memcpy, memmove meminstrinsic calls.
And with D149580 and this patch, if hwasan-match-all-tag is set, calls to __hwasan_load, __hwasan_store, ____hwasan_memset, __hwasan_memcpy and __hwasan_memmove will be replaced with their match_all versions: __hwasan_load_match_all, __hwasan_store_match_all, __hwasan_memset_match_all, __hwasan_memcpy_match_all and __hwasan_memmove_match_all.
When HWAddressSanitizer compiling in kernel mode, hwasan-match-all-tag is implicitly set to 0xFF.
I'm not familiar with kernel mode HWASAN.
But from https://github.com/torvalds/linux/blob/master/mm/kasan/sw_tags.c#L88, I see pointers tagged with 0xff is already ignored in __hwasan_load, __hwasan_store.
So should we distinguish kernel mode HWASAN and user-space HWASAN for hwasan-match-all-tag flag ? :

• for user-space HWASAN, when hwasan-match-all-tag flag is set, we emit calls to __hwasan_load_match_all, __hwasan_memset_match_all, etc.
• for kernel mode HWASAN, hwasan-match-all-tag is implicitly set to 0xFF, we emit calls to __hwasan_load, __hwasan_memset, etc.

Or if hwasan-match-all-tag is set, no matter user-space HWASAN or kernel mode HWASAN, we always emit calls to __hwasan_load_match_all,__hwasan_memset_match_all, and implement __hwasan_load_match_all, __hwasan_memset_match_all in kernel HWASAN runtime ?

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

690–691

Introduced in https://reviews.llvm.org/D122724, for supporting HWAddressSanitizer compiling in kernel mode

llvm/test/Instrumentation/HWAddressSanitizer/mem-intrinsics.ll

18–21

-hwasan-kernel implicit set match-all-tag to 0xff.
With this change, opt -S -passes=hwasan -hwasan-kernel will emit:

call ptr @__hwasan_memset_match_all
call ptr @memset_match_all

Just happen to pass these checks.

[Enna1]

gentle ping :)

[Enna1]

do not use match-all memset/memcpy/memmove for kernel mode HWASAN

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

696

Can you make use of this constructor

template <size_t N>
    /*implicit*/ constexpr ArrayRef(const std::array<T, N> &Arr)

FunctionType *HWAsanMemTransferFnTy = FunctionType::get(
      Int8PtrTy,       ArrayRef(HWAsanMemTransferArgTys), false);

696

Can you make use of this constructor

template <size_t N>
    /*implicit*/ constexpr ArrayRef(const std::array<T, N> &Arr)

sorry, copied wrong one

template <size_t N>
    /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {}

FunctionType *HWAsanMemTransferFnTy = FunctionType::get(
      Int8PtrTy,       ArrayRef(HWAsanMemTransferArgTys), false);

[Enna1]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

696

Sorry, I'm not sure if this is exactly what you meant:

FunctionType *HWAsanMemTransferFnTy;
if (UseMatchAllCallback) {
  HWAsanMemTransferFnTy = FunctionType::get(
    Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy, Int8Ty}, false);
} else {
  HWAsanMemTransferFnTy = FunctionType::get(
    Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy}, false);
}

[Enna1]

gentle ping

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

696

yes, this also looks good

if (UseMatchAllCallback) {
     HWAsanMemTransferFnTy = FunctionType::get(
        Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy, Int8Ty}, false);
     HWAsanMemsetFnTy = FunctionType::get(
        Int8PtrTy, {Int8PtrTy, Int32Ty, IntptrTy, Int8Ty}, false);
  } else {

  }

[Enna1]

update from comments

[Enna1]

update

compiler-rt/lib/hwasan/hwasan_interface_internal.h

 SANITIZER_INTERFACE_ATTRIBUTE
 void *__hwasan_memcpy(void *dst, const void *src, uptr size);
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__hwasan_memset(void *s, int c, uptr n);
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__hwasan_memmove(void *dest, const void *src, uptr n);
 
 SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memcpy_match_all(void *dst, const void *src, uptr size, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memset_match_all(void *s, int c, uptr n, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_memmove_match_all(void *dest, const void *src, uptr n, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_set_error_report_callback(void (*callback)(const char *));
 }  // extern "C"
 
 #endif  // HWASAN_INTERFACE_INTERNAL_H

compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp

 
 void *__hwasan_memmove(void *to, const void *from, uptr size) {
   CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
       reinterpret_cast<uptr>(to), size);
   CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
       reinterpret_cast<uptr>(from), size);
   return memmove(to, from, size);
 }
+
+void *__hwasan_memset_match_all(void *block, int c, uptr size,
+                                u8 match_all_tag) {
+  if (GetTagFromPointer(reinterpret_cast<uptr>(block)) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+        reinterpret_cast<uptr>(block), size);
+  return memset(block, c, size);
+}
+
+void *__hwasan_memcpy_match_all(void *to, const void *from, uptr size,
+                                u8 match_all_tag) {
+  if (GetTagFromPointer(reinterpret_cast<uptr>(to)) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+        reinterpret_cast<uptr>(to), size);
+  if (GetTagFromPointer(reinterpret_cast<uptr>(from)) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
+        reinterpret_cast<uptr>(from), size);
+  return memcpy(to, from, size);
+}
+
+void *__hwasan_memmove_match_all(void *to, const void *from, uptr size,
+                                 u8 match_all_tag) {
+  if (GetTagFromPointer(reinterpret_cast<uptr>(to)) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
+        reinterpret_cast<uptr>(to), size);
+  if (GetTagFromPointer(reinterpret_cast<uptr>(from)) != match_all_tag)
+    CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
+        reinterpret_cast<uptr>(from), size);
+  return memmove(to, from, size);
+}

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
       return GV;
     });
     ThreadPtrGlobal = cast<GlobalVariable>(C);
   }
 }
 
 void HWAddressSanitizer::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
-  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
-    const std::string TypeStr = AccessIsWrite ? "store" : "load";
-    const std::string EndingStr = Recover ? "_noabort" : "";
-    const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
-
-    FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
-        *HwasanMemoryAccessCallbackFnTy;
-    if (UseMatchAllCallback) {
-      HwasanMemoryAccessCallbackSizedFnTy =
-          FunctionType::get(VoidTy, {IntptrTy, IntptrTy, Int8Ty}, false);
-      HwasanMemoryAccessCallbackFnTy =
-          FunctionType::get(VoidTy, {IntptrTy, Int8Ty}, false);
-    } else {
-      HwasanMemoryAccessCallbackSizedFnTy =
-          FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false);
-      HwasanMemoryAccessCallbackFnTy =
-          FunctionType::get(VoidTy, {IntptrTy}, false);
-    }
+      *HwasanMemoryAccessCallbackFnTy, *HWAsanMemTransferFnTy,
+      *HWAsanMemsetFnTy;
+    HWAsanMemTransferFnTy = FunctionType::get(
+        Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy, Int8Ty}, false);
+    HWAsanMemsetFnTy = FunctionType::get(
+        Int8PtrTy, {Int8PtrTy, Int32Ty, IntptrTy, Int8Ty}, false);
+    HWAsanMemTransferFnTy =
+        FunctionType::get(Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy}, false);
+    HWAsanMemsetFnTy =
+        FunctionType::get(Int8PtrTy, {Int8PtrTy, Int32Ty, IntptrTy}, false);
 
+  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
+    const std::string TypeStr = AccessIsWrite ? "store" : "load";
+    const std::string EndingStr = Recover ? "_noabort" : "";
+
     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
         ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
         HwasanMemoryAccessCallbackSizedFnTy);
 
     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
          AccessSizeIndex++) {
       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
           M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr +
                                     itostr(1ULL << AccessSizeIndex) +
                                     MatchAllStr + EndingStr,
                                 HwasanMemoryAccessCallbackFnTy);
     }
   }
 
+  const std::string MemIntrinCallbackPrefix =
+      (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
+          ? std::string("")
+          : ClMemoryAccessCallbackPrefix;
+
+  HWAsanMemmove = M.getOrInsertFunction(
+      MemIntrinCallbackPrefix + "memmove" + MatchAllStr, HWAsanMemTransferFnTy);
+  HWAsanMemcpy = M.getOrInsertFunction(
+      MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, HWAsanMemTransferFnTy);
+  HWAsanMemset = M.getOrInsertFunction(
+      MemIntrinCallbackPrefix + "memset" + MatchAllStr, HWAsanMemsetFnTy);
+
   HwasanTagMemoryFunc = M.getOrInsertFunction("__hwasan_tag_memory", VoidTy,
                                               Int8PtrTy, Int8Ty, IntptrTy);
   HwasanGenerateTagFunc =
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
 
   HwasanRecordFrameRecordFunc =
       M.getOrInsertFunction("__hwasan_add_frame_record", VoidTy, Int64Ty);
 
   ShadowGlobal =
       M.getOrInsertGlobal("__hwasan_shadow", ArrayType::get(Int8Ty, 0));
 
-  const std::string MemIntrinCallbackPrefix =
-      (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
-          ? std::string("")
-          : ClMemoryAccessCallbackPrefix;
-  HWAsanMemmove =
-      M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove", Int8PtrTy,
-                            Int8PtrTy, Int8PtrTy, IntptrTy);
-  HWAsanMemcpy =
-      M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy", Int8PtrTy,
-                            Int8PtrTy, Int8PtrTy, IntptrTy);
-  HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
-                                       Int8PtrTy, Int8PtrTy, Int32Ty, IntptrTy);
-
   HWAsanHandleVfork =
       M.getOrInsertFunction("__hwasan_handle_vfork", VoidTy, IntptrTy);
 }
 
 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
   // An empty inline asm with input reg == output reg.
   // An opaque no-op cast, basically.
   // This prevents code bloat as a result of rematerializing trivial definitions
   // such as constants or global addresses at every load and store.
   InlineAsm *Asm =
       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
                      StringRef(""), StringRef("=r,0"),
                      /*hasSideEffects=*/false);
   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) {
   if (isa<MemSetInst>(MI))
     return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest());
   return false;
 }
 
 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
   IRBuilder<> IRB(MI);
   if (isa<MemTransferInst>(MI)) {
-    IRB.CreateCall(isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
-                   {IRB.CreatePointerCast(MI->getOperand(0), Int8PtrTy),
-                    IRB.CreatePointerCast(MI->getOperand(1), Int8PtrTy),
-                    IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+    if (UseMatchAllCallback) {
+      IRB.CreateCall(
+          isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
+          {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+           IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
+           IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false),
+           ConstantInt::get(Int8Ty, *MatchAllTag)});
+    } else {
+      IRB.CreateCall(
+          isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
+          {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+           IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
+    }
   } else if (isa<MemSetInst>(MI)) {
-    IRB.CreateCall(HWAsanMemset,
-                   {IRB.CreatePointerCast(MI->getOperand(0), Int8PtrTy),
-                    IRB.CreateIntCast(MI->getOperand(1), Int32Ty, false),
-                    IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+    if (UseMatchAllCallback) {
+      IRB.CreateCall(
+          HWAsanMemset,
+          {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+           IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+           IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false),
+           ConstantInt::get(Int8Ty, *MatchAllTag)});
+    } else {
+      IRB.CreateCall(
+          HWAsanMemset,
+          {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+           IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
     }
+  }
   MI->eraseFromParent();
 }
 
 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
   Value *Addr = O.getPtr();
 
   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
 

llvm/test/Instrumentation/HWAddressSanitizer/mem-intrinsics.ll

 ; RUN: opt -S -passes=hwasan -hwasan-use-stack-safety=0 %s | FileCheck --check-prefixes=CHECK,CHECK-PREFIX %s
 ; RUN: opt -S -passes=hwasan -hwasan-kernel -hwasan-use-stack-safety=0 %s | FileCheck --check-prefixes=CHECK,CHECK-NOPREFIX %s
 ; RUN: opt -S -passes=hwasan -hwasan-kernel -hwasan-kernel-mem-intrinsic-prefix -hwasan-use-stack-safety=0 %s | FileCheck --check-prefixes=CHECK,CHECK-PREFIX %s
+; RUN: opt -S -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-match-all-tag=0 %s | FileCheck --check-prefixes=CHECK,CHECK-MATCH-ALL-TAG %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noinline nounwind optnone uwtable
 define dso_local i32 @main() sanitize_hwaddress {
 ; CHECK-LABEL: main
 entry:
   %retval = alloca i32, align 4
   %Q = alloca [10 x i8], align 1
   %P = alloca [10 x i8], align 1
   store i32 0, ptr %retval, align 4
 
   call void @llvm.memset.p0.i64(ptr align 1 %Q, i8 0, i64 10, i1 false)
-; CHECK-PREFIX: call ptr @__hwasan_memset
-; CHECK-NOPREFIX: call ptr @memset
+; CHECK-PREFIX: call ptr @__hwasan_memset(
+; CHECK-NOPREFIX: call ptr @memset(
+; CHECK-MATCH-ALL-TAG: call ptr @__hwasan_memset_match_all(ptr %Q.hwasan, i32 0, i64 10, i8 0)
 
   %add.ptr = getelementptr inbounds i8, ptr %Q, i64 5
 
   call void @llvm.memmove.p0.p0.i64(ptr align 1 %Q, ptr align 1 %add.ptr, i64 5, i1 false)
-; CHECK-PREFIX: call ptr @__hwasan_memmove
-; CHECK-NOPREFIX: call ptr @memmove
+; CHECK-PREFIX: call ptr @__hwasan_memmove(
+; CHECK-NOPREFIX: call ptr @memmove(
+; CHECK-MATCH-ALL-TAG: call ptr @__hwasan_memmove_match_all(ptr %Q.hwasan, ptr %add.ptr, i64 5, i8 0)
 
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 1 %P, ptr align 1 %Q, i64 10, i1 false)
-; CHECK-PREFIX: call ptr @__hwasan_memcpy
-; CHECK-NOPREFIX: call ptr @memcpy
+; CHECK-PREFIX: call ptr @__hwasan_memcpy(
+; CHECK-NOPREFIX: call ptr @memcpy(
+; CHECK-MATCH-ALL-TAG: call ptr @__hwasan_memcpy_match_all(ptr %P.hwasan, ptr %Q.hwasan, i64 10, i8 0)
   ret i32 0
 }
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1) #1
 
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1) #1