hwasan: Read shadow address from ifunc if we don't need a frame record.

[pcc]

Summary

This saves a cbz+cold call in the interceptor ABI, as well as a realign
in both ABIs, trading off a dcache entry against some branch predictor
entries and some code size.

[pcc]

Do this in the platform ABI as well

[pcc]

• Make it possible to disable ifunc with a flag

[eugenis]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

165

Ifunc is known to be broken in static binaries on android.
This needs to be false by default.

[pcc]

• default to false

[pcc]

• Update tests

[pcc]

Use correct base

[pcc]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

165

Done. I guess we can use API level to default this at some point.

[eugenis]

LGTM

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
                               cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
                                        cl::desc("inline all checks"),
                                        cl::Hidden, cl::init(false));
 
 namespace {
 
 /// An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
 
   explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false)
       : FunctionPass(ID) {
     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
         ClEnableKhwasan : CompileKernel;
   }
 
   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 
   bool runOnFunction(Function &F) override;
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
 
+  Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
   Value *shadowBase();
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
     }
   }
 
   HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy));
   HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
 
-  if (Mapping.InGlobal)
-    ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
-                                       ArrayType::get(IRB.getInt8Ty(), 0));
 
   const std::string MemIntrinCallbackPrefix =
       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
   HWAsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
   HWAsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
   HWAsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy));
 
   HwasanThreadEnterFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy()));
 }
 
-Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
-  // Generate code only when dynamic addressing is needed.
-  if (Mapping.Offset != kDynamicShadowSentinel)
-    return nullptr;
-
-  if (Mapping.InGlobal) {
-    // An empty inline asm with input reg == output reg.
-    // An opaque no-op cast, basically.
-    InlineAsm *Asm = InlineAsm::get(
-        FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
-        StringRef(""), StringRef("=r,0"),
-        /*hasSideEffects=*/false);
-    return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+}
+
+Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
+  // Generate code only when dynamic addressing is needed.
+  if (Mapping.Offset != kDynamicShadowSentinel)
+    return nullptr;
+
+  if (Mapping.InGlobal) {
+    return getDynamicShadowIfunc(IRB);
   } else {
     Value *GlobalDynamicAddress =
         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
     return IRB.CreateLoad(GlobalDynamicAddress);
   }
 }
 
if (auto Comdat =
     GV->setComdat(Comdat);
 }
 
 Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
                                         bool WithFrameRecord) {
   if (!Mapping.InTls)
     return getDynamicShadowNonTls(IRB);
 
+  Function *F = IRB.GetInsertBlock()->getParent();
+  bool UseInterceptorABI =
+      F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor";
+
+  if (UseInterceptorABI && !WithFrameRecord && TargetTriple.isAndroid())
+    return getDynamicShadowIfunc(IRB);
+
   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
   assert(SlotPtr);
 
   Instruction *ThreadLong = IRB.CreateLoad(SlotPtr);
 
-  Function *F = IRB.GetInsertBlock()->getParent();
-  if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
+  if (UseInterceptorABI) {
     Value *ThreadLongEqZero =
         IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
     auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
         ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
         false, MDBuilder(*C).createBranchWeights(1, 100000)));
 
     IRB.SetInsertPoint(Br);
     // FIXME: This should call a new runtime function with a custom calling

llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll

 ; RUN: opt -S -hwasan < %s | FileCheck %s
 
-target triple = "x86_64-unknown-linux-gnu"
+target triple = "aarch64--linux-android"
 
 declare void @bar([16 x i32]* %p)
 
-define void @foo() sanitize_hwaddress "hwasan-abi"="interceptor" {
-  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+define void @alloca() sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[A:%[^ ]*]] = call i8* @llvm.thread.pointer()
+  ; CHECK: [[B:%[^ ]*]] = getelementptr i8, i8* [[A]], i32 48
+  ; CHECK: [[C:%[^ ]*]] = bitcast i8* [[B]] to i64*
+  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: [[ICMP:%[^ ]*]] = icmp eq i64 [[LOAD]], 0
   ; CHECK: br i1 [[ICMP]], label %[[INIT:[^,]*]], label %[[CONT:[^,]*]], !prof [[PROF:![0-9]+]]
 
   ; CHECK: [[INIT]]:
   ; CHECK: call void @__hwasan_thread_enter()
-  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: br label %[[CONT]]
 
   ; CHECK: [[CONT]]:
   ; CHECK: phi i64 [ [[LOAD]], %0 ], [ [[RELOAD]], %[[INIT]] ]
 
   %p = alloca [16 x i32]
   call void @bar([16 x i32]* %p)
   ret void
 }
 
+define i32 @load(i32* %p) sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[SHADOW:%[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+  ; CHECK: call void @llvm.hwasan.check.memaccess(i8* [[SHADOW]],
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
 ; CHECK: [[PROF]] = !{!"branch_weights", i32 1, i32 100000}