hwasan: Move fixed shadow behind opaque no-op cast as well.

[pcc]

Summary

This is a workaround for poor heuristics in the backend where we can
end up materializing the constant multiple times. This is particularly
bad when using outlined checks because we materialize it for every call
(because the backend considers it trivial to materialize).

As a result the field containing the shadow base value will always
be set so simplify the code taking that into account.

Depends on D90424

[hctim]

LGTM but will let eugenis make the approval as he's much better versed here.

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

588–593

should this be getShadowNonTls now (and getShadowIfunc() above)?

[eugenis]

LGTM

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

60

This seems suboptimal in the zero shadow case. But is it even used anywhere?

[pcc]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

588–593

I'll rename this one. It's still dynamic in the case where there is an ifunc so I left the other one as is.

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

60

I don't believe that it is. But I think that even with zero shadow we would want a single mov x20, xzr at the top of the function rather than repeating it for every memory access.

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

public:
   }
 
   bool sanitizeFunction(Function &F);
   void initializeModule();
   void createHwasanCtorComdat();
 
   void initializeCallbacks(Module &M);
 
+  Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
+
   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
-  Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
+  Value *getShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
   Value *shadowBase();
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
   void instrumentMemIntrinsic(MemIntrinsic *MI);
private:
   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
   FunctionCallee HwasanMemoryAccessCallbackSized[2];
 
   FunctionCallee HwasanTagMemoryFunc;
   FunctionCallee HwasanGenerateTagFunc;
 
   Constant *ShadowGlobal;
 
-  Value *LocalDynamicShadow = nullptr;
+  Value *ShadowBase = nullptr;
   Value *StackBaseTag = nullptr;
   GlobalValue *ThreadPtrGlobal = nullptr;
 };
 
 class HWAddressSanitizerLegacyPass : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
void HWAddressSanitizer::initializeCallbacks(Module &M) {
   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
                                        IRB.getInt32Ty(), IntptrTy);
 
   HWAsanHandleVfork =
       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
 }
 
-Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
   // An empty inline asm with input reg == output reg.
   // An opaque no-op cast, basically.
-  InlineAsm *Asm = InlineAsm::get(
-      FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
+  // This prevents code bloat as a result of rematerializing trivial definitions
+  // such as constants or global addresses at every load and store.
+  InlineAsm *Asm =
+      InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
                      StringRef(""), StringRef("=r,0"),
                      /*hasSideEffects=*/false);
-  return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+  return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
+}
+
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+  return getOpaqueNoopCast(IRB, ShadowGlobal);
 }
 
-Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
-  // Generate code only when dynamic addressing is needed.
+Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
   if (Mapping.Offset != kDynamicShadowSentinel)
-    return nullptr;
+    return getOpaqueNoopCast(
+        IRB, ConstantExpr::getIntToPtr(
+                 ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
 
   if (Mapping.InGlobal) {
     return getDynamicShadowIfunc(IRB);
   } else {
     Value *GlobalDynamicAddress =
         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);

 
 void HWAddressSanitizer::getInterestingMemoryOperands(
     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
   // Skip memory accesses inserted by another instrumentation.
   if (I->hasMetadata("nosanitize"))
     return;
 
   // Do not instrument the load fetching the dynamic shadow address.
-  if (LocalDynamicShadow == I)
+  if (ShadowBase == I)
     return;
 
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
       return;
     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
                              LI->getType(), LI->getAlign());
   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
 
   IRBuilder<> IRB(I);
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   Value *UntaggedPtr =
       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 }
 
-Value *HWAddressSanitizer::shadowBase() {
-  if (LocalDynamicShadow)
-    return LocalDynamicShadow;
-  return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
-                                   Int8PtrTy);
-}
-
 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
   // Mem >> Scale
   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
   if (Mapping.Offset == 0)
     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
   // (Mem >> Scale) + Offset
-  return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
+  return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
 }
 
 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
   const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
   IRBuilder<> IRB(InsertBefore);
 
   if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
       TargetTriple.isOSBinFormatELF() && !Recover) {
     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
     IRB.CreateCall(Intrinsic::getDeclaration(
                        M, UseShortGranules
                               ? Intrinsic::hwasan_check_memaccess_shortgranules
                               : Intrinsic::hwasan_check_memaccess),
-                   {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
+                   {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
     return;
   }
 
   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
                                   IRB.getInt8Ty());
   Value *AddrLong = untagPointer(IRB, PtrLong);
   Value *Shadow = memToShadow(AddrLong, IRB);
if (ThreadPtrGlobal)
     return ThreadPtrGlobal;
 
 
   return nullptr;
 }
 
 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
   if (!Mapping.InTls) {
-    LocalDynamicShadow = getDynamicShadowNonTls(IRB);
+    ShadowBase = getShadowNonTls(IRB);
     return;
   }
 
   if (!WithFrameRecord && TargetTriple.isAndroid()) {
-    LocalDynamicShadow = getDynamicShadowIfunc(IRB);
+    ShadowBase = getDynamicShadowIfunc(IRB);
     return;
   }
 
   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
   assert(SlotPtr);
 
   Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
if (WithFrameRecord) {
     Value *ThreadLongNew = IRB.CreateAnd(
         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
     IRB.CreateStore(ThreadLongNew, SlotPtr);
   }
 
   // Get shadow base address by aligning RecordPtr up.
   // Note: this is not correct if the pointer is already aligned.
   // Runtime library will make sure this never happens.
-  LocalDynamicShadow = IRB.CreateAdd(
+  ShadowBase = IRB.CreateAdd(
       IRB.CreateOr(
           ThreadLongMaybeUntagged,
           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
-  LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy);
+  ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
 }
 
 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   Function *ReadRegister =
       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
   Value *Args[] = {MetadataAsValue::get(*C, MD)};
if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
     F.setPersonalityFn(nullptr);
     Changed = true;
   }
 
   if (AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
       IntrinToInstrument.empty())
     return Changed;
 
-  assert(!LocalDynamicShadow);
+  assert(!ShadowBase);
 
   Instruction *InsertPt = &*F.getEntryBlock().begin();
   IRBuilder<> EntryIRB(InsertPt);
   emitPrologue(EntryIRB,
                /*WithFrameRecord*/ ClRecordStackHistory &&
                    !AllocasToInstrument.empty());
 
   if (!AllocasToInstrument.empty()) {
bool HWAddressSanitizer::sanitizeFunction(Function &F) {
   for (auto &Operand : OperandsToInstrument)
     instrumentMemAccess(Operand);
 
   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
     for (auto Inst : IntrinToInstrument)
       instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
   }
 
-  LocalDynamicShadow = nullptr;
+  ShadowBase = nullptr;
   StackBaseTag = nullptr;
 
   return true;
 }
 
 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
   Constant *Initializer = GV->getInitializer();
   uint64_t SizeInBytes =

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
 ; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
 
 ; RECOVER: [[CONT1]]:
 ; RECOVER: br label %[[CONT]]
 
 ; RECOVER: [[CONT]]:
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 0)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0)
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }

 ; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
 
 ; RECOVER: [[CONT1]]:
 ; RECOVER: br label %[[CONT]]
 
 ; RECOVER: [[CONT]]:
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 1)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 2)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 3)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 4)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
 
 entry:
   %b = load i128, i128* %a, align 16
   ret i128 %b
 }

 ; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 16)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16)
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 17)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 18)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 19)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 8
   ret void
 }

 ; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 20)
+; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
 
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
 entry:
   store i128 %b, i128* %a, align 16
   ret void
 }

llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll

 ; Test KHWASan instrumentation.
 ;
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,MATCH-ALL
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=CHECK,OFFSET,MATCH-ALL
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-match-all-tag=-1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,NO-MATCH-ALL
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load(
+; OFFSET: %[[SHADOW:[^ ]*]] = call i8* asm "", "=r,0"(i8* inttoptr (i64 12345678 to i8*))
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = or i64 %[[A]], -72057594037927936
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 
 ; NOOFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 
-; OFFSET: %[[E:[^ ]*]] = getelementptr i8, i8* inttoptr (i64 12345678 to i8*), i64 %[[D]]
+; OFFSET: %[[E:[^ ]*]] = getelementptr i8, i8* %[[SHADOW]], i64 %[[D]]
 
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 
 ; MATCH-ALL: %[[G:[^ ]*]] = icmp ne i8 %[[PTRTAG]], -1
 ; MATCH-ALL: %[[H:[^ ]*]] = and i1 %[[F]], %[[G]]
 ; MATCH-ALL: br i1 %[[H]], label {{.*}}, label {{.*}}, !prof {{.*}}