[HWASAN] Inline fast pass of instrumentMemAccessOutline

[vitalybuka]

Summary

Usually pointer tag will match tag in the shadow, so we can keep
inlining this check keeping the rest in outlined part.

It imroves performance by about 25%, but increases code size by 30%.
Existing outlining reduces performance by 30%, but saves code size by 80%.
So we still significantly reduce code size with minimal performance loss.

[vitalybuka]

comment

[vitalybuka]

update

[fmayer]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

896–898

This comment is a bit confusing. Maybe "TODO: use the tag we loaded for the check below"?

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

896–898

That not what I meant to say, but I guess I will just remove TODO
I am not very positive that we will proceed on that one anyway

[vitalybuka]

rebase

[vitalybuka]

remove comment

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

HWAddressSanitizer::insertShadowTagCheck(Value *Ptr,
   return R;
 }
 
 void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
                                                     unsigned AccessSizeIndex,
                                                     Instruction *InsertBefore) {
   assert(!UsePageAliases);
   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
 
-  if (InlineFastPath) {
-    // TODO.
-  }
+  if (InlineFastPath)
+    InsertBefore = insertShadowTagCheck(Ptr, InsertBefore).TagMismatchTerm;
 
   IRBuilder<> IRB(InsertBefore);
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
   IRB.CreateCall(Intrinsic::getDeclaration(
                      M, UseShortGranules
                             ? Intrinsic::hwasan_check_memaccess_shortgranules
                             : Intrinsic::hwasan_check_memaccess),

llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -S | FileCheck %s
+; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=0 -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: define void @atomicrmw
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:

llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll

 ; CHECK-LABEL: define i8 @test_load8
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i8 [[B]]
 ;
 ; NOFASTPATH-LABEL: define i8 @test_load8
 ; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i8 [[B]]
 ;
 ; FASTPATH-LABEL: define i8 @test_load8
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1:![0-9]+]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 0)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i8 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1:![0-9]+]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1:![0-9]+]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define i16 @test_load16
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i16 [[B]]
 ;
 ; NOFASTPATH-LABEL: define i16 @test_load16
 ; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i16 [[B]]
 ;
 ; FASTPATH-LABEL: define i16 @test_load16
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 1)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i16 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define i32 @test_load32
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
 ; NOFASTPATH-LABEL: define i32 @test_load32
 ; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i32 [[B]]
 ;
 ; FASTPATH-LABEL: define i32 @test_load32
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 2)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i32 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define i64 @test_load64
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; CHECK-NEXT:    ret i64 [[B]]
 ;
 ; NOFASTPATH-LABEL: define i64 @test_load64
 ; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; NOFASTPATH-NEXT:    ret i64 [[B]]
 ;
 ; FASTPATH-LABEL: define i64 @test_load64
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 3)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; FASTPATH-NEXT:    ret i64 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define i128 @test_load128
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; CHECK-NEXT:    ret i128 [[B]]
 ;
 ; NOFASTPATH-LABEL: define i128 @test_load128
 ; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; NOFASTPATH-NEXT:    ret i128 [[B]]
 ;
 ; FASTPATH-LABEL: define i128 @test_load128
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 4)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; FASTPATH-NEXT:    ret i128 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define void @test_store8
 ; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; NOFASTPATH-LABEL: define void @test_store8
 ; NOFASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
 ; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store8
 ; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 16)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define void @test_store16
 ; CHECK-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; NOFASTPATH-LABEL: define void @test_store16
 ; NOFASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
 ; NOFASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store16
 ; FASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 17)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define void @test_store32
 ; CHECK-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; NOFASTPATH-LABEL: define void @test_store32
 ; NOFASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
 ; NOFASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store32
 ; FASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 18)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define void @test_store64
 ; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; CHECK-NEXT:    ret void
 ;
 ; NOFASTPATH-LABEL: define void @test_store64
 ; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
 ; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store64
 ; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 19)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; CHECK-LABEL: define void @test_store128
 ; CHECK-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
 ; CHECK-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; CHECK-NEXT:    ret void
 ;
 ; NOFASTPATH-LABEL: define void @test_store128
 ; NOFASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; NOFASTPATH-NEXT:  entry:
 ; NOFASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; NOFASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; NOFASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; NOFASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; NOFASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
 ; NOFASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store128
 ; FASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
 ; FASTPATH-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
 ; FASTPATH-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
 ; FASTPATH-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
 ; FASTPATH-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; FASTPATH-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 56
+; FASTPATH-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; FASTPATH-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; FASTPATH-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; FASTPATH-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; FASTPATH-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; FASTPATH-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; FASTPATH:       12:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[A]], i32 20)
+; FASTPATH-NEXT:    br label [[TMP13]]
+; FASTPATH:       13:
 ; FASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-DYNAMIC-SHADOW:       8:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-DYNAMIC-SHADOW:       9:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
 ; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

 ; RECOVER-DYNAMIC-SHADOW:       22:
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
 ;
 ; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
 ; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; ABORT-ZERO-BASED-SHADOW:       8:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    br label [[TMP9]]
+; ABORT-ZERO-BASED-SHADOW:       9:
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
 ;
 ; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
 ; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
 ; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64

llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test allocas with multiple lifetime ends, as frequently seen for exception
 ; handling.
 ;
-; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s
+; RUN: opt -passes=hwasan -hwasan-use-after-scope -hwasan-inline-fast-path-checks=0 -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 declare void @mayFail(ptr %x) sanitize_hwaddress
 declare void @onExcept(ptr %x) sanitize_hwaddress
 
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind

llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -S < %s | FileCheck %s
+; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -hwasan-inline-fast-path-checks=0 -S < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 @stackbuf = dso_local local_unnamed_addr global ptr null, align 8
 @jbuf = dso_local global [32 x i64] zeroinitializer, align 8
 
 declare void @may_jump()
 

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i8 [[B]]
 ;
 ; FASTPATH-LABEL: define i8 @test_load8
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1:![0-9]+]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i8 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i16 [[B]]
 ;
 ; FASTPATH-LABEL: define i16 @test_load16
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i16 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret i32 [[B]]
 ;
 ; FASTPATH-LABEL: define i32 @test_load32
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret i32 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; NOFASTPATH-NEXT:    ret i64 [[B]]
 ;
 ; FASTPATH-LABEL: define i64 @test_load64
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
 ; FASTPATH-NEXT:    ret i64 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
 ; NOFASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; NOFASTPATH-NEXT:    ret i128 [[B]]
 ;
 ; FASTPATH-LABEL: define i128 @test_load128
 ; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
 ; FASTPATH-NEXT:    ret i128 [[B]]
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
 ; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store8
 ; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
 ; NOFASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store16
 ; FASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
 ; NOFASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store32
 ; FASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
 ; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store64
 ; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)

 ; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
 ; NOFASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; NOFASTPATH-NEXT:    ret void
 ;
 ; FASTPATH-LABEL: define void @test_store128
 ; FASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; FASTPATH-NEXT:  entry:
 ; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; FASTPATH:       8:
 ; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; FASTPATH-NEXT:    br label [[TMP9]]
+; FASTPATH:       9:
 ; FASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
 ; FASTPATH-NEXT:    ret void
 ;
 ; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
 ; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
 ; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
 ; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)