[test][HWASAN] Precommit -hwasan-inline-fast-path-checks tests

[vitalybuka]

update

[fmayer]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

587–592

I'm not a huge fan of different defaults per platform, wouldn't it be easier to just specify this for the platforms where you want this enabled?

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

587–592

I think it should be default, and I am just giving android time to investigate.

How about I land as is and soon after we remove exceptions, so it would be easy to bisect/revert if it's not good enough for android.

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

 static cl::opt<bool> ClInstrumentPersonalityFunctions(
     "hwasan-instrument-personality-functions",
     cl::desc("instrument personality functions"), cl::Hidden);
 
 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
                                        cl::desc("inline all checks"),
                                        cl::Hidden, cl::init(false));
 
+static cl::opt<bool> ClInlineFastPathChecks("hwasan-inline-fast-path-checks",
+                                            cl::desc("inline all checks"),
+                                            cl::Hidden, cl::init(false));
+
 // Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
 static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
                                       cl::desc("Use page aliasing in HWASan"),
                                       cl::Hidden, cl::init(false));
 
 namespace {
 
 bool shouldUsePageAliases(const Triple &TargetTriple) {
private:
   Type *Int8PtrTy;
   Type *Int8Ty;
   Type *Int32Ty;
   Type *Int64Ty = Type::getInt64Ty(M.getContext());
 
   bool CompileKernel;
   bool Recover;
   bool OutlinedChecks;
+  bool InlineFastPath;
   bool UseShortGranules;
   bool InstrumentLandingPads;
   bool InstrumentWithCalls;
   bool InstrumentStack;
   bool DetectUseAfterScope;
   bool UsePageAliases;
   bool UseMatchAllCallback;
 
void HWAddressSanitizer::initializeModule() {
 
   UseShortGranules =
       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
   OutlinedChecks =
       (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
       TargetTriple.isOSBinFormatELF() &&
       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
 
+  InlineFastPath =
+      (ClInlineFastPathChecks.getNumOccurrences() ? ClInlineFastPathChecks
+                                                  : !TargetTriple.isAndroid());
+
   if (ClMatchAllTag.getNumOccurrences()) {
     if (ClMatchAllTag != -1) {
       MatchAllTag = ClMatchAllTag & 0xFF;
     }
   } else if (CompileKernel) {
     MatchAllTag = 0xFF;
   }
   UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
 
   // If we don't have personality function support, fall back to landing pads.

 }
 
 void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
                                                     unsigned AccessSizeIndex,
                                                     Instruction *InsertBefore) {
   assert(!UsePageAliases);
   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
   IRBuilder<> IRB(InsertBefore);
+
+  if (InlineFastPath) {
+    // TODO.
+  }
+
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
   IRB.CreateCall(Intrinsic::getDeclaration(
                      M, UseShortGranules
                             ? Intrinsic::hwasan_check_memaccess_shortgranules
                             : Intrinsic::hwasan_check_memaccess),
                  {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
 }

llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=0 -S | FileCheck %s --check-prefixes=NOFASTPATH
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=1 -S | FileCheck %s --check-prefixes=FASTPATH
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i8 @test_load8
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; NOFASTPATH-LABEL: define i8 @test_load8
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load8
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
 ;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i16 @test_load16
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
+; CHECK-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; CHECK-NEXT:    ret i16 [[B]]
+;
+; NOFASTPATH-LABEL: define i16 @test_load16
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i16 [[B]]
+;
+; FASTPATH-LABEL: define i16 @test_load16
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
+; FASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i16 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
 ;
 entry:
   %b = load i16, ptr %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i32 @test_load32
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[B]]
+;
+; NOFASTPATH-LABEL: define i32 @test_load32
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i32 [[B]]
+;
+; FASTPATH-LABEL: define i32 @test_load32
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
+; FASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i32 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
 ;
 entry:
   %b = load i32, ptr %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i64 @test_load64
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
+; CHECK-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[B]]
+;
+; NOFASTPATH-LABEL: define i64 @test_load64
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; NOFASTPATH-NEXT:    ret i64 [[B]]
+;
+; FASTPATH-LABEL: define i64 @test_load64
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
+; FASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; FASTPATH-NEXT:    ret i64 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
 ;
 entry:
   %b = load i64, ptr %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i128 @test_load128
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
+; CHECK-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; CHECK-NEXT:    ret i128 [[B]]
+;
+; NOFASTPATH-LABEL: define i128 @test_load128
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; NOFASTPATH-NEXT:    ret i128 [[B]]
+;
+; FASTPATH-LABEL: define i128 @test_load128
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
+; FASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; FASTPATH-NEXT:    ret i128 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
 ;
 entry:
   %b = load i128, ptr %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i40 @test_load40
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP4]], i64 5)
+; CHECK-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; CHECK-NEXT:    ret i40 [[B]]
+;
+; NOFASTPATH-LABEL: define i40 @test_load40
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP4]], i64 5)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i40 [[B]]
+;
+; FASTPATH-LABEL: define i40 @test_load40
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP4]], i64 5)
+; FASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i40 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
 ;
 entry:
   %b = load i40, ptr %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store8
+; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
+; CHECK-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store8
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
+; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store8
+; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
+; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i8 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store16
+; CHECK-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
+; CHECK-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store16
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
+; NOFASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store16
+; FASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
+; FASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i16 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store32
+; CHECK-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
+; CHECK-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store32
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
+; NOFASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store32
+; FASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
+; FASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i32 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store64
+; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
+; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store64
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
+; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store64
+; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
+; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i64 %b, ptr %a, align 8
   ret void
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store128
+; CHECK-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
+; CHECK-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store128
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
+; NOFASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store128
+; FASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
+; FASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i128 %b, ptr %a, align 16
   ret void
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store40
+; CHECK-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 5)
+; CHECK-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store40
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 5)
+; NOFASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store40
+; FASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 5)
+; FASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i40 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store_unaligned
+; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 8)
+; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store_unaligned
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOFASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 8)
+; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store_unaligned
+; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 8)
+; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void

 
 define i8 @test_load_noattr(ptr %a) {
 ; CHECK-LABEL: define i8 @test_load_noattr
 ; CHECK-SAME: (ptr [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_noattr
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_noattr
+; FASTPATH-SAME: (ptr [[A:%.*]]) {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {

 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
 ; CHECK-LABEL: define i8 @test_load_notmyattr
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_notmyattr
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_notmyattr
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {

 ;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
 ; CHECK-LABEL: define i8 @test_load_addrspace
-; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_addrspace
+; NOFASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_addrspace
+; FASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {

llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ; Generic code is covered by ../basic.ll, only the x86_64 specific code is
 ; tested here.
 ;
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=0 -S | FileCheck %s --check-prefixes=NOFASTPATH
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=1 -S | FileCheck %s --check-prefixes=FASTPATH
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=ABORT
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=RECOVER
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-instrument-with-calls=0 -S | FileCheck %s  --check-prefixes=ABORT-INLINE
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-instrument-with-calls=0 -S | FileCheck %s  --check-prefixes=RECOVER-INLINE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i8 @test_load8
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; NOFASTPATH-LABEL: define i8 @test_load8
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load8
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-LABEL: define i8 @test_load8
 ; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-NEXT:  entry:
 ; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
 ; ABORT-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-NEXT:    ret i8 [[B]]

 ; RECOVER-INLINE-NEXT:    ret i8 [[B]]
 ;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i40 @test_load40
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; CHECK-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; CHECK-NEXT:    ret i40 [[B]]
+;
+; NOFASTPATH-LABEL: define i40 @test_load40
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i40 [[B]]
+;
+; FASTPATH-LABEL: define i40 @test_load40
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; FASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i40 [[B]]
+;
 ; ABORT-LABEL: define i40 @test_load40
 ; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-NEXT:  entry:
 ; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
 ; ABORT-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
 ; ABORT-NEXT:    ret i40 [[B]]

 ; RECOVER-INLINE-NEXT:    ret i40 [[B]]
 ;
 entry:
   %b = load i40, ptr %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store8
+; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
+; CHECK-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store8
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
+; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store8
+; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
+; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-LABEL: define void @test_store8
 ; ABORT-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-NEXT:  entry:
 ; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
 ; ABORT-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; ABORT-NEXT:    ret void

 ; RECOVER-INLINE-NEXT:    [[TMP24:%.*]] = icmp ne i8 [[TMP6]], [[TMP23]]
 ; RECOVER-INLINE-NEXT:    br i1 [[TMP24]], label [[TMP14]], label [[TMP25]], !prof [[PROF1]]
 ; RECOVER-INLINE:       25:
 ; RECOVER-INLINE-NEXT:    br label [[TMP26]]
 ; RECOVER-INLINE:       26:
 ; RECOVER-INLINE-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; RECOVER-INLINE-NEXT:    ret void
 ;
+entry:
+  store i8 %b, ptr %a, align 4
+  ret void
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store40
+; CHECK-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; CHECK-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store40
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; NOFASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store40
+; FASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; FASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-LABEL: define void @test_store40
 ; ABORT-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-NEXT:  entry:
 ; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
 ; ABORT-NEXT:    store i40 [[B]], ptr [[A]], align 4
 ; ABORT-NEXT:    ret void

 ; RECOVER-INLINE-NEXT:    ret void
 ;
 entry:
   store i40 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store_unaligned
+; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store_unaligned
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store_unaligned
+; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-LABEL: define void @test_store_unaligned
 ; ABORT-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-NEXT:  entry:
 ; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
 ; ABORT-NEXT:    store i64 [[B]], ptr [[A]], align 4
 ; ABORT-NEXT:    ret void

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=0 -S | FileCheck %s --check-prefixes=NOFASTPATH
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=1 -S | FileCheck %s --check-prefixes=FASTPATH
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
 ; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android10000"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i8 @test_load8
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; NOFASTPATH-LABEL: define i8 @test_load8
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load8
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
 ;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i16 @test_load16
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; CHECK-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; CHECK-NEXT:    ret i16 [[B]]
+;
+; NOFASTPATH-LABEL: define i16 @test_load16
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i16 [[B]]
+;
+; FASTPATH-LABEL: define i16 @test_load16
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; FASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i16 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
 ;

 ;
 
 entry:
   %b = load i16, ptr %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i32 @test_load32
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[B]]
+;
+; NOFASTPATH-LABEL: define i32 @test_load32
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i32 [[B]]
+;
+; FASTPATH-LABEL: define i32 @test_load32
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; FASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i32 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
 ;

 ;
 
 entry:
   %b = load i32, ptr %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i64 @test_load64
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; CHECK-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[B]]
+;
+; NOFASTPATH-LABEL: define i64 @test_load64
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; NOFASTPATH-NEXT:    ret i64 [[B]]
+;
+; FASTPATH-LABEL: define i64 @test_load64
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; FASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; FASTPATH-NEXT:    ret i64 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
 ;

 ;
 
 entry:
   %b = load i64, ptr %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i128 @test_load128
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; CHECK-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; CHECK-NEXT:    ret i128 [[B]]
+;
+; NOFASTPATH-LABEL: define i128 @test_load128
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; NOFASTPATH-NEXT:    ret i128 [[B]]
+;
+; FASTPATH-LABEL: define i128 @test_load128
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; FASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; FASTPATH-NEXT:    ret i128 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
 ;

 ;
 
 entry:
   %b = load i128, ptr %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i40 @test_load40
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; CHECK-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; CHECK-NEXT:    ret i40 [[B]]
+;
+; NOFASTPATH-LABEL: define i40 @test_load40
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i40 [[B]]
+;
+; FASTPATH-LABEL: define i40 @test_load40
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; FASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i40 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
 ;
 entry:
   %b = load i40, ptr %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store8
+; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; CHECK-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store8
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store8
+; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ;
 
 entry:
   store i8 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store16
+; CHECK-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; CHECK-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store16
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; NOFASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store16
+; FASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; FASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ;
 
 entry:
   store i16 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store32
+; CHECK-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; CHECK-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store32
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; NOFASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store32
+; FASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; FASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ;
 
 entry:
   store i32 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store64
+; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store64
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store64
+; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ;
 
 entry:
   store i64 %b, ptr %a, align 8
   ret void
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store128
+; CHECK-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; CHECK-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store128
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; NOFASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store128
+; FASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; FASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;

 ;
 
 entry:
   store i128 %b, ptr %a, align 16
   ret void
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store40
+; CHECK-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; CHECK-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store40
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; NOFASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store40
+; FASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; FASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void

 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 entry:
   store i40 %b, ptr %a, align 4
   ret void
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: define void @test_store_unaligned
+; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+; NOFASTPATH-LABEL: define void @test_store_unaligned
+; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret void
+;
+; FASTPATH-LABEL: define void @test_store_unaligned
+; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; FASTPATH-NEXT:    ret void
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void

 
 define i8 @test_load_noattr(ptr %a) {
 ; CHECK-LABEL: define i8 @test_load_noattr
 ; CHECK-SAME: (ptr [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_noattr
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_noattr
+; FASTPATH-SAME: (ptr [[A:%.*]]) {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {

 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
 ; CHECK-LABEL: define i8 @test_load_notmyattr
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_notmyattr
+; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_notmyattr
+; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {

 ;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
 ; CHECK-LABEL: define i8 @test_load_addrspace
-; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
+; NOFASTPATH-LABEL: define i8 @test_load_addrspace
+; NOFASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; NOFASTPATH-NEXT:  entry:
+; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; NOFASTPATH-NEXT:    ret i8 [[B]]
+;
+; FASTPATH-LABEL: define i8 @test_load_addrspace
+; FASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; FASTPATH-NEXT:  entry:
+; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; FASTPATH-NEXT:    ret i8 [[B]]
+;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {