diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -530,6 +530,56 @@ CheckAddress(p); } +void __hwasan_loadN_match_all(uptr p, uptr sz, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_load1_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load2_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load4_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load8_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load16_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + +void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_load1_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load2_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load4_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load8_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_load16_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + void __hwasan_storeN(uptr p, uptr sz) { CheckAddressSized(p, sz); } @@ -568,6 +618,56 @@ CheckAddress(p); } +void __hwasan_storeN_match_all(uptr p, uptr sz, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_store1_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store2_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store4_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store8_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store16_match_all(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + +void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized(p, sz); +} +void __hwasan_store1_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store2_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store4_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store8_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} +void __hwasan_store16_match_all_noabort(uptr p, tag_t match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress(p); +} + void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(UntagAddr(p), sz, tag); } diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -624,18 +624,27 @@ for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { const std::string TypeStr = AccessIsWrite ? "store" : "load"; const std::string EndingStr = Recover ? "_noabort" : ""; + const std::string MatchAllStr = MatchAllTag.has_value() ? "_match_all" : ""; + const size_t CallbackSizedArgSize = MatchAllTag.has_value() ? 3 : 2; + llvm::Type *CallbackSizedArgTys[] = {IntptrTy, IntptrTy, Int8Ty}; HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, - FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)); + ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr, + FunctionType::get( + IRB.getVoidTy(), + ArrayRef(&CallbackSizedArgTys[0], CallbackSizedArgSize), false)); for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; AccessSizeIndex++) { + const size_t CallbackArgSize = MatchAllTag.has_value() ? 2 : 1; + llvm::Type *CallbackArgTys[] = {IntptrTy, Int8Ty}; HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = M.getOrInsertFunction( ClMemoryAccessCallbackPrefix + TypeStr + - itostr(1ULL << AccessSizeIndex) + EndingStr, - FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)); + itostr(1ULL << AccessSizeIndex) + MatchAllStr + EndingStr, + FunctionType::get(IRB.getVoidTy(), + ArrayRef(&CallbackArgTys[0], CallbackArgSize), + false)); } } @@ -963,19 +972,34 @@ *O.Alignment >= O.TypeStoreSize / 8)) { size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize); if (InstrumentWithCalls) { - IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], - IRB.CreatePointerCast(Addr, IntptrTy)); + if (MatchAllTag.has_value()) { + IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], + {IRB.CreatePointerCast(Addr, IntptrTy), + ConstantInt::get(Int8Ty, *MatchAllTag)}); + } else { + IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], + IRB.CreatePointerCast(Addr, IntptrTy)); + } } else if (OutlinedChecks) { instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } else { instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } } else { - IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite], - {IRB.CreatePointerCast(Addr, IntptrTy), - IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, - O.TypeStoreSize), - ConstantInt::get(IntptrTy, 8))}); + if (MatchAllTag.has_value()) { + IRB.CreateCall( + HwasanMemoryAccessCallbackSized[O.IsWrite], + {IRB.CreatePointerCast(Addr, IntptrTy), + IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize), + ConstantInt::get(IntptrTy, 8)), + ConstantInt::get(Int8Ty, *MatchAllTag)}); + } else { + IRB.CreateCall( + HwasanMemoryAccessCallbackSized[O.IsWrite], + {IRB.CreatePointerCast(Addr, IntptrTy), + IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize), + ConstantInt::get(IntptrTy, 8))}); + } } untagPointerOperand(O.getInsn(), Addr); diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll --- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll @@ -15,8 +15,8 @@ ; CHECK-LABEL: @test_load( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 -; ABORT: call void @__hwasan_load1(i64 %[[A]]) -; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]]) +; ABORT: call void @__hwasan_load1_match_all(i64 %[[A]], i8 -1) +; RECOVER: call void @__hwasan_load1_match_all_noabort(i64 %[[A]], i8 -1) ; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4 ; CHECK: ret i8 %[[G]] diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll --- a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll @@ -2,6 +2,8 @@ ; ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-ABORT +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-RECOVER target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-android" @@ -11,6 +13,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load1(i64 %[[A]]) ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load1_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load1_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i8, ptr %a ; CHECK: ret i8 %[[B]] @@ -24,6 +28,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load2(i64 %[[A]]) ; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load2_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load2_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i16, ptr %a ; CHECK: ret i16 %[[B]] @@ -37,6 +43,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load4(i64 %[[A]]) ; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load4_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load4_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i32, ptr %a ; CHECK: ret i32 %[[B]] @@ -50,6 +58,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load8(i64 %[[A]]) ; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load8_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load8_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i64, ptr %a ; CHECK: ret i64 %[[B]] @@ -63,6 +73,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_load16(i64 %[[A]]) ; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_load16_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load16_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: %[[B:[^ ]*]] = load i128, ptr %a ; CHECK: ret i128 %[[B]] @@ -76,6 +88,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_loadN_match_all(i64 %[[A]], i64 5, i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_loadN_match_all_noabort(i64 %[[A]], i64 5, i8 0) ; CHECK: %[[B:[^ ]*]] = load i40, ptr %a ; CHECK: ret i40 %[[B]] @@ -89,6 +103,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store1(i64 %[[A]]) ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store1_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store1_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i8 %b, ptr %a ; CHECK: ret void @@ -102,6 +118,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store2(i64 %[[A]]) ; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store2_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store2_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i16 %b, ptr %a ; CHECK: ret void @@ -115,6 +133,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store4(i64 %[[A]]) ; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store4_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store4_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i32 %b, ptr %a ; CHECK: ret void @@ -128,6 +148,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store8(i64 %[[A]]) ; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store8_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store8_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i64 %b, ptr %a ; CHECK: ret void @@ -141,6 +163,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_store16(i64 %[[A]]) ; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]]) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_store16_match_all(i64 %[[A]], i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store16_match_all_noabort(i64 %[[A]], i8 0) ; CHECK: store i128 %b, ptr %a ; CHECK: ret void @@ -154,6 +178,8 @@ ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5) +; MATCH-ALL-TAG-ABORT: call void @__hwasan_storeN_match_all(i64 %[[A]], i64 5, i8 0) +; MATCH-ALL-TAG-RECOVER: call void @__hwasan_storeN_match_all_noabort(i64 %[[A]], i64 5, i8 0) ; CHECK: store i40 %b, ptr %a ; CHECK: ret void