diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -530,6 +530,56 @@ CheckAddress(p); } +void __hwasan_loadN_match_all(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_load1_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load2_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load4_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load8_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load16_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + +void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_load1_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load2_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load4_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load8_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load16_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + void __hwasan_storeN(uptr p, uptr sz) { CheckAddressSized(p, sz); } @@ -568,6 +618,56 @@ CheckAddress(p); } +void __hwasan_storeN_match_all(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_store1_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store2_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store4_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store8_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store16_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + +void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_store1_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store2_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store4_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store8_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store16_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(UntagAddr(p), sz, tag); } diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h --- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -76,6 +76,32 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_load16_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_loadN_match_all(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16_match_all(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_loadN_match_all_noabort(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16_match_all_noabort(uptr, u8); + SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_storeN(uptr, uptr); SANITIZER_INTERFACE_ATTRIBUTE @@ -102,6 +128,32 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store16_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_storeN_match_all(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16_match_all(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_storeN_match_all_noabort(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16_match_all_noabort(uptr, u8); + SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -380,6 +380,7 @@ bool InstrumentStack; bool DetectUseAfterScope; bool UsePageAliases; + bool UseMatchAllCallback; std::optional MatchAllTag; @@ -584,6 +585,7 @@ } else if (CompileKernel) { MatchAllTag = 0xFF; } + UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value(); // If we don't have personality function support, fall back to landing pads. InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences() @@ -624,17 +626,27 @@ for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { const std::string TypeStr = AccessIsWrite ? "store" : "load"; const std::string EndingStr = Recover ? "_noabort" : ""; + const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : ""; + const size_t CallbackSizedArgSize = UseMatchAllCallback ? 3 : 2; + llvm::Type *CallbackSizedArgTys[] = {IntptrTy, IntptrTy, Int8Ty}; HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, - FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false)); + ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr, + FunctionType::get( + VoidTy, ArrayRef(&CallbackSizedArgTys[0], CallbackSizedArgSize), + false)); for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; AccessSizeIndex++) { + const size_t CallbackArgSize = UseMatchAllCallback ? 2 : 1; + llvm::Type *CallbackArgTys[] = {IntptrTy, Int8Ty}; HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = - M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + - itostr(1ULL << AccessSizeIndex) + EndingStr, - FunctionType::get(VoidTy, {IntptrTy}, false)); + M.getOrInsertFunction( + ClMemoryAccessCallbackPrefix + TypeStr + + itostr(1ULL << AccessSizeIndex) + MatchAllStr + EndingStr, + FunctionType::get(VoidTy, + ArrayRef(&CallbackArgTys[0], CallbackArgSize), + false)); } } @@ -959,19 +971,34 @@ *O.Alignment >= O.TypeStoreSize / 8)) { size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize); if (InstrumentWithCalls) { - IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], - IRB.CreatePointerCast(Addr, IntptrTy)); + if (UseMatchAllCallback) { + IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], + {IRB.CreatePointerCast(Addr, IntptrTy), + ConstantInt::get(Int8Ty, *MatchAllTag)}); + } else { + IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], + IRB.CreatePointerCast(Addr, IntptrTy)); + } } else if (OutlinedChecks) { instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } else { instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } } else { - IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite], - {IRB.CreatePointerCast(Addr, IntptrTy), - IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, - O.TypeStoreSize), - ConstantInt::get(IntptrTy, 8))}); + if (UseMatchAllCallback) { + IRB.CreateCall( + HwasanMemoryAccessCallbackSized[O.IsWrite], + {IRB.CreatePointerCast(Addr, IntptrTy), + IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize), + ConstantInt::get(IntptrTy, 8)), + ConstantInt::get(Int8Ty, *MatchAllTag)}); + } else { + IRB.CreateCall( + HwasanMemoryAccessCallbackSized[O.IsWrite], + {IRB.CreatePointerCast(Addr, IntptrTy), + IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize), + ConstantInt::get(IntptrTy, 8))}); + } } untagPointerOperand(O.getInsn(), Addr); diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll --- a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll @@ -2,6 +2,8 @@ ; ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-ABORT +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-RECOVER target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-android" @@ -11,6 +13,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load1(i64 %[[A]]) ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load1_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load1_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i8, ptr %a ; CHECK: ret i8 %[[B]] @@ -24,6 +28,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load2(i64 %[[A]]) ; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load2_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load2_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i16, ptr %a ; CHECK: ret i16 %[[B]] @@ -37,6 +43,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load4(i64 %[[A]]) ; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load4_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load4_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i32, ptr %a ; CHECK: ret i32 %[[B]] @@ -50,6 +58,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load8(i64 %[[A]]) ; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load8_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load8_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i64, ptr %a ; CHECK: ret i64 %[[B]] @@ -63,6 +73,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load16(i64 %[[A]]) ; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load16_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load16_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i128, ptr %a ; CHECK: ret i128 %[[B]] @@ -76,6 +88,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_loadN_match_all(i64 %[[A]], i64 5, i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_loadN_match_all_noabort(i64 %[[A]], i64 5, i8 0) ; CHECK: %[[B:[^ ]*]] = load i40, ptr %a ; CHECK: ret i40 %[[B]] @@ -89,6 +103,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store1(i64 %[[A]]) ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store1_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store1_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i8 %b, ptr %a ; CHECK: ret void @@ -102,6 +118,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store2(i64 %[[A]]) ; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store2_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store2_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i16 %b, ptr %a ; CHECK: ret void @@ -115,6 +133,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store4(i64 %[[A]]) ; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store4_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store4_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i32 %b, ptr %a ; CHECK: ret void @@ -128,6 +148,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store8(i64 %[[A]]) ; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store8_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store8_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i64 %b, ptr %a ; CHECK: ret void @@ -141,6 +163,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store16(i64 %[[A]]) ; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store16_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store16_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i128 %b, ptr %a ; CHECK: ret void @@ -154,6 +178,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_storeN_match_all(i64 %[[A]], i64 5, i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_storeN_match_all_noabort(i64 %[[A]], i64 5, i8 0) ; CHECK: store i40 %b, ptr %a ; CHECK: ret void