diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -447,6 +447,12 @@ void __hwasan_load16(uptr p) { CheckAddress(p); } +void __hwasan_load32(uptr p) { + CheckAddress(p); +} +void __hwasan_load64(uptr p) { + CheckAddress(p); +} void __hwasan_loadN_noabort(uptr p, uptr sz) { CheckAddressSized(p, sz); @@ -466,6 +472,12 @@ void __hwasan_load16_noabort(uptr p) { CheckAddress(p); } +void __hwasan_load32_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_load64_noabort(uptr p) { + CheckAddress(p); +} void __hwasan_storeN(uptr p, uptr sz) { CheckAddressSized(p, sz); @@ -485,6 +497,12 @@ void __hwasan_store16(uptr p) { CheckAddress(p); } +void __hwasan_store32(uptr p) { + CheckAddress(p); +} +void __hwasan_store64(uptr p) { + CheckAddress(p); +} void __hwasan_storeN_noabort(uptr p, uptr sz) { CheckAddressSized(p, sz); @@ -504,6 +522,12 @@ void __hwasan_store16_noabort(uptr p) { CheckAddress(p); } +void __hwasan_store32_noabort(uptr p) { + CheckAddress(p); +} +void __hwasan_store64_noabort(uptr p) { + CheckAddress(p); +} void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(p, sz, tag); diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h --- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -62,6 +62,10 @@ void __hwasan_load8(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_load16(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load32(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load64(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_loadN_noabort(uptr, uptr); @@ -75,6 +79,10 @@ void __hwasan_load8_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_load16_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load32_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load64_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_storeN(uptr, uptr); @@ -88,6 +96,10 @@ void __hwasan_store8(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store16(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store32(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store64(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_storeN_noabort(uptr, uptr); @@ -101,6 +113,10 @@ void __hwasan_store8_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store16_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store32_noabort(uptr); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store64_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -69,8 +69,11 @@ const char kHwasanShadowMemoryDynamicAddress[] = "__hwasan_shadow_memory_dynamic_address"; -// Accesses sizes are powers of two: 1, 2, 4, 8, 16. -static const size_t kNumberOfAccessSizes = 5; +// Access-sizes that are supported in call-based instrumentation on old version +// of Android. +static const size_t kOldNumberOfAccessSizes = 5; +// Accesses sizes are powers of two: 1, 2, 4, 8, 16, 32, 64. +static const size_t kNumberOfAccessSizes = 7; static const size_t kDefaultShadowScale = 4; static const uint64_t kDynamicShadowSentinel = @@ -377,6 +380,7 @@ bool InstrumentStack; bool DetectUseAfterScope; bool UsePageAliases; + size_t UsableNumberOfAccessCallbacks; bool HasMatchAllTag = false; uint8_t MatchAllTag = 0; @@ -659,6 +663,11 @@ bool NewRuntime = !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30); + UsableNumberOfAccessCallbacks = + TargetTriple.isAndroid() && TargetTriple.isAndroidVersionLT(32) + ? kOldNumberOfAccessSizes + : kNumberOfAccessSizes; + UseShortGranules = ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime; OutlinedChecks = @@ -719,8 +728,8 @@ ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)); - for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; - AccessSizeIndex++) { + for (size_t AccessSizeIndex = 0; + AccessSizeIndex < UsableNumberOfAccessCallbacks; AccessSizeIndex++) { HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = M.getOrInsertFunction( ClMemoryAccessCallbackPrefix + TypeStr + @@ -1032,12 +1041,15 @@ return false; // FIXME IRBuilder<> IRB(O.getInsn()); + auto MaxAccessNumber = InstrumentWithCalls ? UsableNumberOfAccessCallbacks + : kNumberOfAccessSizes; if (isPowerOf2_64(O.TypeSize) && - (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) && + (O.TypeSize / 8 <= (1ULL << (MaxAccessNumber - 1))) && (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) || *O.Alignment >= O.TypeSize / 8)) { size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize); if (InstrumentWithCalls) { + assert(AccessSizeIndex < UsableNumberOfAccessCallbacks); IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex], IRB.CreatePointerCast(Addr, IntptrTy)); } else if (OutlinedChecks) { diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll --- a/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll @@ -1,7 +1,11 @@ ; Test basic address sanitizer instrumentation. ; -; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT -; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT,OLDRUNTIME-ABORT +; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,OLDRUNTIME-RECOVER +; RUN: opt < %s -mtriple aarch64--linux-gnu -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT,NEWRUNTIME-ABORT +; RUN: opt < %s -mtriple aarch64--linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,NEWRUNTIME-RECOVER +; RUN: opt < %s -mtriple aarch64--linux-android32 -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT,NEWRUNTIME-ABORT +; RUN: opt < %s -mtriple aarch64--linux-android32 -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,NEWRUNTIME-RECOVER target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-android" @@ -71,6 +75,36 @@ ret i128 %b } +define i256 @test_load256(i256* %a) sanitize_hwaddress { +; CHECK-LABEL: @test_load256( +; CHECK: %[[A:[^ ]*]] = ptrtoint i256* %a to i64 +; OLDRUNTIME-ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 32) +; OLDRUNTIME-RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 32) +; NEWRUNTIME-ABORT: call void @__hwasan_load32(i64 %[[A]]) +; NEWRUNTIME-RECOVER: call void @__hwasan_load32_noabort(i64 %[[A]]) +; CHECK: %[[B:[^ ]*]] = load i256, i256* %a +; CHECK: ret i256 %[[B]] + +entry: + %b = load i256, i256* %a, align 16 + ret i256 %b +} + +define i512 @test_load512(i512* %a) sanitize_hwaddress { +; CHECK-LABEL: @test_load512( +; CHECK: %[[A:[^ ]*]] = ptrtoint i512* %a to i64 +; OLDRUNTIME-ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 64) +; OLDRUNTIME-RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 64) +; NEWRUNTIME-ABORT: call void @__hwasan_load64(i64 %[[A]]) +; NEWRUNTIME-RECOVER: call void @__hwasan_load64_noabort(i64 %[[A]]) +; CHECK: %[[B:[^ ]*]] = load i512, i512* %a +; CHECK: ret i512 %[[B]] + +entry: + %b = load i512, i512* %a, align 16 + ret i512 %b +} + define i40 @test_load40(i40* %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load40( ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64 @@ -149,6 +183,36 @@ ret void } +define void @test_store256(i256* %a, i256 %b) sanitize_hwaddress { +; CHECK-LABEL: @test_store256( +; CHECK: %[[A:[^ ]*]] = ptrtoint i256* %a to i64 +; OLDRUNTIME-ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 32) +; OLDRUNTIME-RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 32) +; NEWRUNTIME-ABORT: call void @__hwasan_store32(i64 %[[A]]) +; NEWRUNTIME-RECOVER: call void @__hwasan_store32_noabort(i64 %[[A]]) +; CHECK: store i256 %b, i256* %a +; CHECK: ret void + +entry: + store i256 %b, i256* %a, align 16 + ret void +} + +define void @test_store512(i512* %a, i512 %b) sanitize_hwaddress { +; CHECK-LABEL: @test_store512( +; CHECK: %[[A:[^ ]*]] = ptrtoint i512* %a to i64 +; OLDRUNTIME-ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 64) +; OLDRUNTIME-RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 64) +; NEWRUNTIME-ABORT: call void @__hwasan_store64(i64 %[[A]]) +; NEWRUNTIME-RECOVER: call void @__hwasan_store64_noabort(i64 %[[A]]) +; CHECK: store i512 %b, i512* %a +; CHECK: ret void + +entry: + store i512 %b, i512* %a, align 16 + ret void +} + define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress { ; CHECK-LABEL: @test_store40( ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64