diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -957,25 +957,42 @@ return false; // FIXME IRBuilder<> IRB(O.getInsn()); + + auto checkMatchAllTagForCallback = [&](Value *PtrLong) { + if (MatchAllTag.has_value()) { + Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, PointerTagShift), + IRB.getInt8Ty()); + Value *TagIgnored = IRB.CreateICmpNE( + PtrTag, ConstantInt::get(PtrTag->getType(), *MatchAllTag)); + Instruction *NotTagIgnoredTerm = SplitBlockAndInsertIfThen( + TagIgnored, O.getInsn(), false, + MDBuilder(*C).createBranchWeights(1, 100000)); + IRB.SetInsertPoint(NotTagIgnoredTerm); + } + }; + if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(O.TypeStoreSize) && (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) && (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() || *O.Alignment >= O.TypeStoreSize / 8)) { size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize); if (InstrumentWithCalls) { + Value *PtrLong = IRB.CreatePointerCast(Addr, IntptrTy); + checkMatchAllTagForCallback(PtrLong); IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], - IRB.CreatePointerCast(Addr, IntptrTy)); + PtrLong); } else if (OutlinedChecks) { instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } else { instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn()); } } else { - IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite], - {IRB.CreatePointerCast(Addr, IntptrTy), - IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, - O.TypeStoreSize), - ConstantInt::get(IntptrTy, 8))}); + Value *PtrLong = IRB.CreatePointerCast(Addr, IntptrTy); + checkMatchAllTagForCallback(PtrLong); + IRB.CreateCall( + HwasanMemoryAccessCallbackSized[O.IsWrite], + {PtrLong, IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize), + ConstantInt::get(IntptrTy, 8))}); } untagPointerOperand(O.getInsn(), Addr); diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll --- a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll @@ -2,6 +2,8 @@ ; ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT ; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,MATCH-ALL-TAG +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,MATCH-ALL-TAG target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-android" @@ -9,6 +11,10 @@ define i8 @test_load8(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load8( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_load1(i64 %[[A]]) ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]]) ; CHECK: %[[B:[^ ]*]] = load i8, ptr %a @@ -22,6 +28,10 @@ define i16 @test_load16(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load16( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_load2(i64 %[[A]]) ; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]]) ; CHECK: %[[B:[^ ]*]] = load i16, ptr %a @@ -35,6 +45,10 @@ define i32 @test_load32(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load32( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_load4(i64 %[[A]]) ; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]]) ; CHECK: %[[B:[^ ]*]] = load i32, ptr %a @@ -48,6 +62,10 @@ define i64 @test_load64(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load64( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_load8(i64 %[[A]]) ; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]]) ; CHECK: %[[B:[^ ]*]] = load i64, ptr %a @@ -61,6 +79,10 @@ define i128 @test_load128(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load128( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_load16(i64 %[[A]]) ; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]]) ; CHECK: %[[B:[^ ]*]] = load i128, ptr %a @@ -74,6 +96,10 @@ define i40 @test_load40(ptr %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load40( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5) ; CHECK: %[[B:[^ ]*]] = load i40, ptr %a @@ -87,6 +113,10 @@ define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store8( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_store1(i64 %[[A]]) ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]]) ; CHECK: store i8 %b, ptr %a @@ -100,6 +130,10 @@ define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store16( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_store2(i64 %[[A]]) ; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]]) ; CHECK: store i16 %b, ptr %a @@ -113,6 +147,10 @@ define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store32( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_store4(i64 %[[A]]) ; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]]) ; CHECK: store i32 %b, ptr %a @@ -126,6 +164,10 @@ define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store64( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_store8(i64 %[[A]]) ; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]]) ; CHECK: store i64 %b, ptr %a @@ -139,6 +181,10 @@ define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store128( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_store16(i64 %[[A]]) ; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]]) ; CHECK: store i128 %b, ptr %a @@ -152,6 +198,10 @@ define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store40( ; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64 +; MATCH-ALL-TAG: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 +; MATCH-ALL-TAG: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 +; MATCH-ALL-TAG: %[[C:[^ ]*]] = icmp ne i8 %[[PTRTAG]], 0 +; MATCH-ALL-TAG: br i1 %[[C]], label {{.*}}, label {{.*}}, !prof {{.*}} ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5) ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5) ; CHECK: store i40 %b, ptr %a