hwasan: Read shadow address from ifunc if we don't need a frame record.

[pcc]

Summary

This saves a cbz+cold call in the interceptor ABI, as well as a realign
in both ABIs, trading off a dcache entry against some branch predictor
entries and some code size.

[pcc]

Do this in the platform ABI as well

[pcc]

• Make it possible to disable ifunc with a flag

[eugenis]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

165 ↗

(On Diff #183176)

Ifunc is known to be broken in static binaries on android.
This needs to be false by default.

[pcc]

• default to false

[pcc]

• Update tests

[pcc]

Use correct base

[pcc]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

165 ↗

(On Diff #183176)

Done. I guess we can use API level to default this at some point.

[eugenis]

LGTM

compiler-rt/trunk/test/hwasan/lit.cfg

 # -*- Python -*-
 
 import os
 
 # Setup config name.
 config.name = 'HWAddressSanitizer' + getattr(config, 'name_suffix', 'default')
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
 # Setup default compiler flags used with -fsanitize=memory option.
 clang_cflags = [config.target_cflags] + config.debug_info_flags
 clang_cxxflags = config.cxx_mode_flags + clang_cflags
-clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls"] + clang_cflags
+clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls",
+                       "-mllvm", "-hwasan-allow-ifunc"] + clang_cflags
 clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
 config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
 config.substitutions.append( ("%clang_hwasan ", build_invocation(clang_hwasan_cflags)) )
 config.substitutions.append( ("%clangxx_hwasan ", build_invocation(clang_hwasan_cxxflags)) )

llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

static cl::opt<bool>
     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
                               cl::desc("instrument memory intrinsics"),
                               cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
                                        cl::desc("inline all checks"),
                                        cl::Hidden, cl::init(false));
 
+static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
+                                  cl::desc("allow the use of ifunc"),
+                                  cl::Hidden, cl::init(false));
+
 namespace {
 
 /// An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
 
   explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false)
       : FunctionPass(ID) {
     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
         ClEnableKhwasan : CompileKernel;
   }
 
   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 
   bool runOnFunction(Function &F) override;
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
 
+  Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
   Value *shadowBase();
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
     }
   }
 
   HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy));
   HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
 
-  if (Mapping.InGlobal)
-    ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
-                                       ArrayType::get(IRB.getInt8Ty(), 0));
 
   const std::string MemIntrinCallbackPrefix =
       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
   HWAsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
   HWAsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
   HWAsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy));
 
   HwasanThreadEnterFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy()));
 }
 
-Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
-  // Generate code only when dynamic addressing is needed.
-  if (Mapping.Offset != kDynamicShadowSentinel)
-    return nullptr;
-
-  if (Mapping.InGlobal) {
-    // An empty inline asm with input reg == output reg.
-    // An opaque no-op cast, basically.
-    InlineAsm *Asm = InlineAsm::get(
-        FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
-        StringRef(""), StringRef("=r,0"),
-        /*hasSideEffects=*/false);
-    return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+}
+
+Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
+  // Generate code only when dynamic addressing is needed.
+  if (Mapping.Offset != kDynamicShadowSentinel)
+    return nullptr;
+
+  if (Mapping.InGlobal) {
+    return getDynamicShadowIfunc(IRB);
   } else {
     Value *GlobalDynamicAddress =
         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
     return IRB.CreateLoad(GlobalDynamicAddress);
   }
 }
 
if (auto Comdat =
     GV->setComdat(Comdat);
 }
 
 Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
                                         bool WithFrameRecord) {
   if (!Mapping.InTls)
     return getDynamicShadowNonTls(IRB);
 
+  if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
+    return getDynamicShadowIfunc(IRB);
+
   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
   assert(SlotPtr);
 
   Instruction *ThreadLong = IRB.CreateLoad(SlotPtr);
 
   Function *F = IRB.GetInsertBlock()->getParent();
   if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
     Value *ThreadLongEqZero =

llvm/trunk/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll

-; RUN: opt -S -hwasan < %s | FileCheck %s
+; RUN: opt -S -hwasan -hwasan-allow-ifunc < %s | FileCheck %s
 
-target triple = "x86_64-unknown-linux-gnu"
+target triple = "aarch64--linux-android"
 
 declare void @bar([16 x i32]* %p)
 
-define void @foo() sanitize_hwaddress "hwasan-abi"="interceptor" {
-  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+define void @alloca() sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[A:%[^ ]*]] = call i8* @llvm.thread.pointer()
+  ; CHECK: [[B:%[^ ]*]] = getelementptr i8, i8* [[A]], i32 48
+  ; CHECK: [[C:%[^ ]*]] = bitcast i8* [[B]] to i64*
+  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: [[ICMP:%[^ ]*]] = icmp eq i64 [[LOAD]], 0
   ; CHECK: br i1 [[ICMP]], label %[[INIT:[^,]*]], label %[[CONT:[^,]*]], !prof [[PROF:![0-9]+]]
 
   ; CHECK: [[INIT]]:
   ; CHECK: call void @__hwasan_thread_enter()
-  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: br label %[[CONT]]
 
   ; CHECK: [[CONT]]:
   ; CHECK: phi i64 [ [[LOAD]], %0 ], [ [[RELOAD]], %[[INIT]] ]
 
   %p = alloca [16 x i32]
   call void @bar([16 x i32]* %p)
   ret void
 }
 
+define i32 @load(i32* %p) sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[SHADOW:%[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+  ; CHECK-NOT: icmp
+  ; CHECK: call void @llvm.hwasan.check.memaccess(i8* [[SHADOW]],
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
 ; CHECK: [[PROF]] = !{!"branch_weights", i32 1, i32 100000}

llvm/trunk/test/Instrumentation/HWAddressSanitizer/prologue.ll

 ; Test -hwasan-with-ifunc flag.
 ;
-; RUN: opt -hwasan -S < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
+; RUN: opt -hwasan -hwasan-allow-ifunc -S < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
 ; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECk-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android22"
 
 ; CHECK-IFUNC: @__hwasan_shadow = external global [0 x i8]
 ; CHECK-NOIFUNC: @__hwasan_shadow_memory_dynamic_address = external global i64
 
 define i32 @test_load(i32* %a) sanitize_hwaddress {
 ; First instrumentation in the function must be to load the dynamic shadow
 ; address into a local variable.
 ; CHECK-LABEL: @test_load
 ; CHECK: entry:
 
-; CHECK-IFUNC:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
-; CHECK-IFUNC:   @llvm.hwasan.check.memaccess(i8* %[[A]]
+; CHECK-NOGLOBAL:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+; CHECK-NOGLOBAL:   @llvm.hwasan.check.memaccess(i8* %[[A]]
 
 ; CHECK-GLOBAL: load i8*, i8** @__hwasan_shadow_memory_dynamic_address
 
-; CHECK-TLS:   %[[A:[^ ]*]] = call i8* @llvm.thread.pointer()
-; CHECK-TLS:   %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 48
-; CHECK-TLS:   %[[C:[^ ]*]] = bitcast i8* %[[B]] to i64*
-; CHECK-TLS:   %[[D:[^ ]*]] = load i64, i64* %[[C]]
-; CHECK-TLS:   %[[E:[^ ]*]] = or i64 %[[D]], 4294967295
-; CHECK-TLS:   = add i64 %[[E]], 1
-
 ; "store i64" is only used to update stack history (this input IR intentionally does not use any i64)
 ; W/o any allocas, the history is not updated, even if it is enabled explicitly with -hwasan-record-stack-history=1
 ; CHECK-NOT: store i64
 
 ; CHECK: ret i32
 
 entry:
   %x = load i32, i32* %a, align 4