[HWASan] Introduce non-zero based and dynamic shadow memory (LLVM).

[alekseyshl]

Summary

Support the dynamic shadow memory offset (the default case for user
space now) and static non-zero shadow memory offset
(-hwasan-mapping-offset option). Keeping the the latter case around
for functionality and performance comparison tests (and mostly for
-hwasan-mapping-offset=0 case).

The implementation is stripped down ASan one, picking only the relevant
parts in the following assumptions: shadow scale is fixed, the shadow
memory is dynamic, it is accessed via ifunc global, shadow memory address
rematerialization is suppressed.

Keep zero-based shadow memory for kernel (-hwasan-kernel option) and
calls instreumented case (-hwasan-instrument-with-calls option), which
essentially means that the generated code is not changed in these cases.

[kcc]

plz don't forget to update the documentation (the asm snippet) at clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html

[eugenis]

LGTM

lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

 
 using namespace llvm;
 
 #define DEBUG_TYPE "hwasan"
 
 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
 static const char *const kHwasanInitName = "__hwasan_init";
 
+static const char *const kHwasanShadowMemoryDynamicAddress =
+    "__hwasan_shadow_memory_dynamic_address";
+
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
 
-static const size_t kShadowScale = 4;
-static const unsigned kAllocaAlignment = 1U << kShadowScale;
+static const size_t kDefaultShadowScale = 4;
+static const uint64_t kDynamicShadowSentinel =
+    std::numeric_limits<uint64_t>::max();
 static const unsigned kPointerTagShift = 56;
 
 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
     "hwasan-memory-access-callback-prefix",
     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
     cl::init("__hwasan_"));
 
 static cl::opt<bool>
static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
                                        cl::desc("instrument stack (allocas)"),
                                        cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClGenerateTagsWithCalls(
     "hwasan-generate-tags-with-calls",
     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
     cl::init(false));
 
-static cl::opt<unsigned long long> ClMappingOffset(
-    "hwasan-mapping-offset",
-    cl::desc("offset of hwasan shadow mapping [EXPERIMENTAL]"), cl::Hidden,
-    cl::init(0));
-
 static cl::opt<int> ClMatchAllTag(
     "hwasan-match-all-tag",
-    cl::desc("don't report bad accesses via pointers with this tag"), cl::Hidden,
-    cl::init(-1));
+    cl::desc("don't report bad accesses via pointers with this tag"),
+    cl::Hidden, cl::init(-1));
 
 static cl::opt<bool> ClEnableKhwasan(
-    "hwasan-kernel", cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
+    "hwasan-kernel",
+    cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
     cl::Hidden, cl::init(false));
 
+// These flags allow to change the shadow mapping and control how shadow memory
+// is accessed. The shadow mapping looks like:
+//    Shadow = (Mem >> scale) + offset
+
+static cl::opt<unsigned long long> ClMappingOffset(
+    "hwasan-mapping-offset",
+    cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden,
+    cl::init(0));
+
 namespace {
 
 /// \brief An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
 
   HWAddressSanitizer(bool Recover = false)
       : FunctionPass(ID), Recover(Recover || ClRecover) {}
 
   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 
   bool runOnFunction(Function &F) override;
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
+
+  void maybeInsertDynamicShadowAtFunctionEntry(Function &F);
+
   void untagPointerOperand(Instruction *I, Value *Addr);
+  Value *memToShadow(Value *Shadow, Type *Ty, IRBuilder<> &IRB);
   void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
                                    Value **MaybeMask);
 
   bool isInterestingAlloca(const AllocaInst &AI);
   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
   bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
                        SmallVectorImpl<Instruction *> &RetVec);
   Value *getNextTagWithCall(IRBuilder<> &IRB);
   Value *getStackBaseTag(IRBuilder<> &IRB);
   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
                      unsigned AllocaNo);
   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 
 private:
   LLVMContext *C;
   Triple TargetTriple;
 
+  /// This struct defines the shadow mapping using the rule:
+  ///   shadow = (mem >> Scale) + Offset.
+  /// If InGlobal is true, then
+  ///   extern char __hwasan_shadow[];
+  ///   shadow = (mem >> Scale) + &__hwasan_shadow
+  struct ShadowMapping {
+    int Scale;
+    uint64_t Offset;
+    bool InGlobal;
+
+    void init(Triple &TargetTriple);
+    unsigned getAllocaAlignment() const { return 1U << Scale; }
+  };
+  ShadowMapping Mapping;
+
   Type *IntptrTy;
   Type *Int8Ty;
 
   bool Recover;
 
   Function *HwasanCtorFunction;
 
   Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
   Function *HwasanMemoryAccessCallbackSized[2];
 
   Function *HwasanTagMemoryFunc;
   Function *HwasanGenerateTagFunc;
+
+  Constant *ShadowGlobal;
+
+  Value *LocalDynamicShadow = nullptr;
 };
 
 } // end anonymous namespace
 
 char HWAddressSanitizer::ID = 0;
 
 INITIALIZE_PASS_BEGIN(
     HWAddressSanitizer, "hwasan",
-    "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
+    "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
+    false)
 INITIALIZE_PASS_END(
     HWAddressSanitizer, "hwasan",
-    "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
+    "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
+    false)
 
 FunctionPass *llvm::createHWAddressSanitizerPass(bool Recover) {
   return new HWAddressSanitizer(Recover);
 }
 
 /// \brief Module-level initialization.
 ///
 /// inserts a call to __hwasan_init to the module's constructor list.
 bool HWAddressSanitizer::doInitialization(Module &M) {
   DEBUG(dbgs() << "Init " << M.getName() << "\n");
   auto &DL = M.getDataLayout();
 
   TargetTriple = Triple(M.getTargetTriple());
 
+  Mapping.init(TargetTriple);
+
   C = &(M.getContext());
   IRBuilder<> IRB(*C);
   IntptrTy = IRB.getIntPtrTy(DL);
   Int8Ty = IRB.getInt8Ty();
 
   HwasanCtorFunction = nullptr;
   if (!ClEnableKhwasan) {
     std::tie(HwasanCtorFunction, std::ignore) =
for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)));
     }
   }
 
   HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       "__hwasan_tag_memory", IRB.getVoidTy(), IntptrTy, Int8Ty, IntptrTy));
   HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
+
+  if (Mapping.InGlobal)
+    ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
+                                       ArrayType::get(IRB.getInt8Ty(), 0));
+}
+
+void HWAddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
+  // Generate code only when dynamic addressing is needed.
+  if (Mapping.Offset != kDynamicShadowSentinel)
+    return;
+
+  IRBuilder<> IRB(&F.front().front());
+  if (Mapping.InGlobal) {
+    // An empty inline asm with input reg == output reg.
+    // An opaque pointer-to-int cast, basically.
+    InlineAsm *Asm = InlineAsm::get(
+        FunctionType::get(IntptrTy, {ShadowGlobal->getType()}, false),
+        StringRef(""), StringRef("=r,0"),
+        /*hasSideEffects=*/false);
+    LocalDynamicShadow = IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+  } else {
+    Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
+        kHwasanShadowMemoryDynamicAddress, IntptrTy);
+    LocalDynamicShadow = IRB.CreateLoad(GlobalDynamicAddress);
+  }
 }
 
 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
                                                      bool *IsWrite,
                                                      uint64_t *TypeSize,
                                                      unsigned *Alignment,
                                                      Value **MaybeMask) {
   // Skip memory accesses inserted by another instrumentation.
   if (I->getMetadata("nosanitize")) return nullptr;
 
+  // Do not instrument the load fetching the dynamic shadow address.
+  if (LocalDynamicShadow == I)
+    return nullptr;
+
   Value *PtrOperand = nullptr;
   const DataLayout &DL = I->getModule()->getDataLayout();
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     if (!ClInstrumentReads) return nullptr;
     *IsWrite = false;
     *TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
     *Alignment = LI->getAlignment();
     PtrOperand = LI->getPointerOperand();
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     if (!ClInstrumentAtomics) return nullptr;
     *IsWrite = true;
     *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
     *Alignment = 0;
     PtrOperand = XCHG->getPointerOperand();
   }
 
   if (PtrOperand) {
-    // Do not instrument acesses from different address spaces; we cannot deal
+    // Do not instrument accesses from different address spaces; we cannot deal
     // with them.
     Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
     if (PtrTy->getPointerAddressSpace() != 0)
       return nullptr;
 
     // Ignore swifterror addresses.
     // swifterror memory addresses are mem2reg promoted by instruction
     // selection. As such they cannot have regular uses like an instrumentation
void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
 
   IRBuilder<> IRB(I);
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   Value *UntaggedPtr =
       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 }
 
+Value *HWAddressSanitizer::memToShadow(Value *Mem, Type *Ty, IRBuilder<> &IRB) {
+  // Mem >> Scale
+  Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
+  if (Mapping.Offset == 0)
+    return Shadow;
+  // (Mem >> Scale) + Offset
+  Value *ShadowBase;
+  if (LocalDynamicShadow)
+    ShadowBase = LocalDynamicShadow;
+  else
+    ShadowBase = ConstantInt::get(Ty, Mapping.Offset);
+  return IRB.CreateAdd(Shadow, ShadowBase);
+}
+
 void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
   IRBuilder<> IRB(InsertBefore);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
                                   IRB.getInt8Ty());
   Value *AddrLong = untagPointer(IRB, PtrLong);
-  Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
-  if (ClMappingOffset)
-    ShadowLong = IRB.CreateAdd(
-        ShadowLong, ConstantInt::get(PtrLong->getType(), ClMappingOffset,
-                                     /*isSigned=*/false));
+  Value *ShadowLong = memToShadow(AddrLong, PtrLong->getType(), IRB);
   Value *MemTag =
       IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 
   if (ClMatchAllTag != -1) {
     Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
         ConstantInt::get(PtrTag->getType(), ClMatchAllTag));
     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
 
   if (MaybeMask)
     return false; //FIXME
 
   IRBuilder<> IRB(I);
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   if (isPowerOf2_64(TypeSize) &&
       (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
-      (Alignment >= (1UL << kShadowScale) || Alignment == 0 ||
+      (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
        Alignment >= TypeSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
     if (ClInstrumentWithCalls) {
       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
                      AddrLong);
     } else {
       instrumentMemAccessInline(AddrLong, IsWrite, AccessSizeIndex, I);
     }
static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
   }
   Type *Ty = AI.getAllocatedType();
   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
   return SizeInBytes * ArraySize;
 }
 
 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
                                    Value *Tag) {
-  size_t Size = (getAllocaSizeInBytes(*AI) + kAllocaAlignment - 1) &
-                ~(kAllocaAlignment - 1);
+  size_t Size = (getAllocaSizeInBytes(*AI) + Mapping.getAllocaAlignment() - 1) &
+                ~(Mapping.getAllocaAlignment() - 1);
 
   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
   if (ClInstrumentWithCalls) {
     IRB.CreateCall(HwasanTagMemoryFunc,
                    {IRB.CreatePointerCast(AI, IntptrTy), JustTag,
                     ConstantInt::get(IntptrTy, Size)});
   } else {
-    size_t ShadowSize = Size >> kShadowScale;
+    size_t ShadowSize = Size >> Mapping.Scale;
     Value *ShadowPtr = IRB.CreateIntToPtr(
-        IRB.CreateLShr(IRB.CreatePointerCast(AI, IntptrTy), kShadowScale),
+        memToShadow(IRB.CreatePointerCast(AI, IntptrTy), AI->getType(), IRB),
         IRB.getInt8PtrTy());
     // If this memset is not inlined, it will be intercepted in the hwasan
     // runtime library. That's OK, because the interceptor skips the checks if
     // the address is in the shadow region.
     // FIXME: the interceptor is not as fast as real memset. Consider lowering
     // llvm.memset right here into either a sequence of stores, or a call to
     // hwasan_tag_memory.
     IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);

 
 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
   if (ClGenerateTagsWithCalls)
     return getNextTagWithCall(IRB);
   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
 }
 
 // Add a tag to an address.
-Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong,
-                                      Value *Tag) {
+Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
+                                      Value *PtrLong, Value *Tag) {
   Value *TaggedPtrLong;
   if (ClEnableKhwasan) {
     // Kernel addresses have 0xFF in the most significant byte.
     Value *ShiftedTag = IRB.CreateOr(
         IRB.CreateShl(Tag, kPointerTagShift),
         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
   } else {
bool HWAddressSanitizer::runOnFunction(Function &F) {
 
   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
     return false;
 
   DEBUG(dbgs() << "Function: " << F.getName() << "\n");
 
   initializeCallbacks(*F.getParent());
 
+  assert(!LocalDynamicShadow);
+  maybeInsertDynamicShadowAtFunctionEntry(F);
+
   bool Changed = false;
   SmallVector<Instruction*, 16> ToInstrument;
   SmallVector<AllocaInst*, 8> AllocasToInstrument;
   SmallVector<Instruction*, 8> RetVec;
   for (auto &BB : F) {
     for (auto &Inst : BB) {
       if (ClInstrumentStack)
         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
           // Realign all allocas. We don't want small uninteresting allocas to
           // hide in instrumented alloca's padding.
-          if (AI->getAlignment() < kAllocaAlignment)
-            AI->setAlignment(kAllocaAlignment);
+          if (AI->getAlignment() < Mapping.getAllocaAlignment())
+            AI->setAlignment(Mapping.getAllocaAlignment());
           // Instrument some of them.
           if (isInterestingAlloca(*AI))
             AllocasToInstrument.push_back(AI);
           continue;
         }
 
-      if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) || isa<CleanupReturnInst>(Inst))
+      if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
+          isa<CleanupReturnInst>(Inst))
         RetVec.push_back(&Inst);
 
       Value *MaybeMask = nullptr;
       bool IsWrite;
       unsigned Alignment;
       uint64_t TypeSize;
       Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize,
                                               &Alignment, &MaybeMask);
       if (Addr || isa<MemIntrinsic>(Inst))
         ToInstrument.push_back(&Inst);
     }
   }
 
   if (!AllocasToInstrument.empty())
     Changed |= instrumentStack(AllocasToInstrument, RetVec);
 
   for (auto Inst : ToInstrument)
     Changed |= instrumentMemAccess(Inst);
 
+  LocalDynamicShadow = nullptr;
+
   return Changed;
 }
+
+void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
+  const bool IsAndroid = TargetTriple.isAndroid();
+  const bool IsLinux = TargetTriple.isOSLinux();
+  const bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
+  const bool IsAndroidWithIfuncSupport =
+      IsAndroid && !TargetTriple.isAndroidVersionLT(21);
+
+  Scale = kDefaultShadowScale;
+
+  if (ClEnableKhwasan || ClInstrumentWithCalls)
+    Offset = 0;
+  else
+    Offset = kDynamicShadowSentinel;
+  if (ClMappingOffset.getNumOccurrences() > 0)
+    Offset = ClMappingOffset;
+
+  InGlobal = (IsX86_64 && IsLinux) || IsAndroidWithIfuncSupport;
+}

test/Instrumentation/HWAddressSanitizer/alloca.ll

 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -S | FileCheck %s
+; RUN: opt < %s -hwasan -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW
+; RUN: opt < %s -hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ZERO-BASED-SHADOW
 ; RUN: opt < %s -hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s --check-prefix=WITH-CALLS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 declare void @use32(i32*)
 
 define void @test_alloca() sanitize_hwaddress {
 ; CHECK-LABEL: @test_alloca(
 ; CHECK: %[[FP:[^ ]*]] = call i8* @llvm.frameaddress(i32 0)
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %[[FP]] to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
 ; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
 
 ; CHECK: %[[X:[^ ]*]] = alloca i32, align 16
 ; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
 ; CHECK: %[[X1:[^ ]*]] = ptrtoint i32* %[[X]] to i64
 ; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
 ; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
 ; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to i32*
 
 ; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
 ; CHECK: %[[E:[^ ]*]] = ptrtoint i32* %[[X]] to i64
 ; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; CHECK: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to i8*
+; DYNAMIC-SHADOW: %[[F_DYN:[^ ]*]] = add i64 %[[F]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to i8*
 ; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %[[X_SHADOW]], i8 %[[X_TAG2]], i64 1, i1 false)
 ; CHECK: call void @use32(i32* nonnull %[[X_HWASAN]])
 
 ; CHECK: %[[X_TAG_UAR:[^ ]*]] = xor i64 %[[BASE_TAG]], 255
 ; CHECK: %[[X_TAG_UAR2:[^ ]*]] = trunc i64 %[[X_TAG_UAR]] to i8
 ; CHECK: %[[E2:[^ ]*]] = ptrtoint i32* %[[X]] to i64
 ; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; CHECK: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to i8*
+; DYNAMIC-SHADOW: %[[F2_DYN:[^ ]*]] = add i64 %[[F2]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to i8*
 ; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR2]], i64 1, i1 false)
 ; CHECK: ret void
 
 
 entry:
   %x = alloca i32, align 4
   call void @use32(i32* nonnull %x)
   ret void
 }
 
 ; WITH-CALLS-LABEL: @test_alloca(
 ; WITH-CALLS: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
 ; WITH-CALLS: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
 ; WITH-CALLS: %[[B:[^ ]*]] = ptrtoint i32* %x to i64
 ; WITH-CALLS: %[[C:[^ ]*]] = shl i64 %[[A]], 56
 ; WITH-CALLS: or i64 %[[B]], %[[C]]

test/Instrumentation/HWAddressSanitizer/basic.ll

 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -hwasan -hwasan-recover=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,DYNAMIC-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,DYNAMIC-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,ZERO-BASED-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,ZERO-BASED-SHADOW
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2305", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2306", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2307", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2308", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
 ; RECOVER: br label

 
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2320", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2321", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2322", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
+; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
+; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
 ; ABORT: call void asm sideeffect "brk #2324", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
 ; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
 ; RECOVER: br label

 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(i8 addrspace(256)* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load_addrspace(
 ; CHECK-NEXT: entry:
+; DYNAMIC-SHADOW: %.hwasan.shadow = call i64 asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
 ; CHECK-NEXT: %[[B:[^ ]*]] = load i8, i8 addrspace(256)* %a
 ; CHECK-NEXT: ret i8 %[[B]]
 
 entry:
   %b = load i8, i8 addrspace(256)* %a, align 4
   ret i8 %b
 }
 
 ; CHECK: declare void @__hwasan_init()
 
 ; CHECK:      define internal void @hwasan.module_ctor() {
 ; CHECK-NEXT:   call void @__hwasan_init()
 ; CHECK-NEXT:   ret void
 ; CHECK-NEXT: }