Index: clang/include/clang/Basic/CodeGenOptions.h =================================================================== --- clang/include/clang/Basic/CodeGenOptions.h +++ clang/include/clang/Basic/CodeGenOptions.h @@ -286,6 +286,8 @@ /// Set of XRay instrumentation kinds to emit. XRayInstrSet XRayInstrumentationBundle; + std::vector DefaultFunctionAttrs; + public: // Define accessors/mutators for code generation options of enumeration type. #define CODEGENOPT(Name, Bits, Default) Index: clang/include/clang/Driver/CC1Options.td =================================================================== --- clang/include/clang/Driver/CC1Options.td +++ clang/include/clang/Driver/CC1Options.td @@ -163,6 +163,8 @@ def debug_info_kind_EQ : Joined<["-"], "debug-info-kind=">; def debug_info_macro : Flag<["-"], "debug-info-macro">, HelpText<"Emit macro debug information">; +def default_function_attr : Separate<["-"], "default-function-attr">, + HelpText<"Apply given attribute to all functions">; def dwarf_version_EQ : Joined<["-"], "dwarf-version=">; def debugger_tuning_EQ : Joined<["-"], "debugger-tuning=">; def fdebug_compilation_dir : Separate<["-"], "fdebug-compilation-dir">, Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -998,6 +998,10 @@ : Flag<["-"], "fno-sanitize-address-use-odr-indicator">, Group, HelpText<"Disable ODR indicator globals">; +def fsanitize_hwaddress_abi_EQ + : Joined<["-"], "fsanitize-hwaddress-abi=">, + Group, + HelpText<"Select the HWAddressSanitizer ABI to target (interceptor or platform, default interceptor)">; def fsanitize_recover : Flag<["-"], "fsanitize-recover">, Group; def fno_sanitize_recover : Flag<["-"], "fno-sanitize-recover">, Flags<[CoreOption, DriverOption]>, Index: clang/include/clang/Driver/SanitizerArgs.h =================================================================== --- clang/include/clang/Driver/SanitizerArgs.h +++ clang/include/clang/Driver/SanitizerArgs.h @@ -39,6 +39,7 @@ bool AsanPoisonCustomArrayCookie = false; bool AsanGlobalsDeadStripping = false; bool AsanUseOdrIndicator = false; + std::string HwasanAbi; bool LinkCXXRuntimes = false; bool NeedPIE = false; bool SafeStackRuntime = false; Index: clang/lib/CodeGen/CGCall.cpp =================================================================== --- clang/lib/CodeGen/CGCall.cpp +++ clang/lib/CodeGen/CGCall.cpp @@ -1816,6 +1816,12 @@ if (CodeGenOpts.FlushDenorm) FuncAttrs.addAttribute("nvptx-f32ftz", "true"); } + + for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) { + StringRef Var, Value; + std::tie(Var, Value) = Attr.split('='); + FuncAttrs.addAttribute(Var, Value); + } } void CodeGenModule::AddDefaultFnAttrs(llvm::Function &F) { Index: clang/lib/Driver/SanitizerArgs.cpp =================================================================== --- clang/lib/Driver/SanitizerArgs.cpp +++ clang/lib/Driver/SanitizerArgs.cpp @@ -741,6 +741,18 @@ AsanUseAfterScope = false; } + if (AllAddedKinds & HWAddress) { + if (Arg *HwasanAbiArg = + Args.getLastArg(options::OPT_fsanitize_hwaddress_abi_EQ)) { + HwasanAbi = HwasanAbiArg->getValue(); + if (HwasanAbi != "platform" && HwasanAbi != "interceptor") + D.Diag(clang::diag::err_drv_invalid_value) + << HwasanAbiArg->getAsString(Args) << HwasanAbi; + } else { + HwasanAbi = "interceptor"; + } + } + if (AllAddedKinds & SafeStack) { // SafeStack runtime is built into the system on Fuchsia. SafeStackRuntime = !TC.getTriple().isOSFuchsia(); @@ -913,6 +925,11 @@ if (AsanUseOdrIndicator) CmdArgs.push_back("-fsanitize-address-use-odr-indicator"); + if (!HwasanAbi.empty()) { + CmdArgs.push_back("-default-function-attr"); + CmdArgs.push_back(Args.MakeArgString("hwasan-abi=" + HwasanAbi)); + } + // MSan: Workaround for PR16386. // ASan: This is mainly to help LSan with cases such as // https://github.com/google/sanitizers/issues/373 Index: clang/lib/Frontend/CompilerInvocation.cpp =================================================================== --- clang/lib/Frontend/CompilerInvocation.cpp +++ clang/lib/Frontend/CompilerInvocation.cpp @@ -1318,6 +1318,8 @@ Opts.SpeculativeLoadHardening = Args.hasArg(OPT_mspeculative_load_hardening); + Opts.DefaultFunctionAttrs = Args.getAllArgValues(OPT_default_function_attr); + return Success; } Index: clang/test/CodeGen/default-function-attr.c =================================================================== --- /dev/null +++ clang/test/CodeGen/default-function-attr.c @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -default-function-attr foo=bar -emit-llvm %s -o - | FileCheck %s + +// CHECK: define void @foo() #[[X:[0-9]+]] +void foo() {} + +// CHECK: attributes #[[X]] = {{.*}} "foo"="bar" Index: clang/test/Driver/fsanitize.c =================================================================== --- clang/test/Driver/fsanitize.c +++ clang/test/Driver/fsanitize.c @@ -837,3 +837,11 @@ // // RUN: %clang -target x86_64-linux-gnu -fsanitize=scudo,kernel-memory %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SCUDO-KMSAN // CHECK-SCUDO-KMSAN: error: invalid argument '-fsanitize=kernel-memory' not allowed with '-fsanitize=scudo' + +// RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-INTERCEPTOR-ABI +// RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=interceptor %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-INTERCEPTOR-ABI +// RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=platform %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-PLATFORM-ABI +// RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=foo %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-FOO-ABI +// CHECK-HWASAN-INTERCEPTOR-ABI: "-default-function-attr" "hwasan-abi=interceptor" +// CHECK-HWASAN-PLATFORM-ABI: "-default-function-attr" "hwasan-abi=platform" +// CHECK-HWASAN-FOO-ABI: error: invalid value 'foo' in '-fsanitize-hwaddress-abi=foo' Index: compiler-rt/lib/hwasan/hwasan_interceptors.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan_interceptors.cc +++ compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -217,35 +217,6 @@ #endif #endif // HWASAN_WITH_INTERCEPTORS - -#if HWASAN_WITH_INTERCEPTORS -extern "C" int pthread_attr_init(void *attr); -extern "C" int pthread_attr_destroy(void *attr); - -struct ThreadStartArg { - thread_callback_t callback; - void *param; -}; - -static void *HwasanThreadStartFunc(void *arg) { - __hwasan_thread_enter(); - ThreadStartArg A = *reinterpret_cast(arg); - UnmapOrDie(arg, GetPageSizeCached()); - return A.callback(A.param); -} - -INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), - void * param) { - ScopedTaggingDisabler disabler; - ThreadStartArg *A = reinterpret_cast (MmapOrDie( - GetPageSizeCached(), "pthread_create")); - *A = {callback, param}; - int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr), - &HwasanThreadStartFunc, A); - return res; -} -#endif // HWASAN_WITH_INTERCEPTORS - static void BeforeFork() { StackDepotLockAll(); } @@ -285,7 +256,6 @@ INTERCEPT_FUNCTION(fork); #if HWASAN_WITH_INTERCEPTORS - INTERCEPT_FUNCTION(pthread_create); INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); #endif Index: compiler-rt/lib/hwasan/hwasan_linux.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan_linux.cc +++ compiler-rt/lib/hwasan/hwasan_linux.cc @@ -302,7 +302,12 @@ #endif Thread *GetCurrentThread() { - auto *R = (StackAllocationsRingBuffer*)GetCurrentThreadLongPtr(); + uptr *ThreadLong = GetCurrentThreadLongPtr(); +#if HWASAN_WITH_INTERCEPTORS + if (!*ThreadLong) + __hwasan_thread_enter(); +#endif + auto *R = (StackAllocationsRingBuffer *)ThreadLong; return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next())); } Index: llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -264,6 +264,7 @@ Function *HwasanTagMemoryFunc; Function *HwasanGenerateTagFunc; + Function *HwasanThreadEnterFunc; Constant *ShadowGlobal; @@ -391,6 +392,9 @@ HWAsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction( MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy)); + + HwasanThreadEnterFunc = checkSanitizerInterfaceFunction( + M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy())); } Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) { @@ -806,14 +810,35 @@ Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); assert(SlotPtr); - Value *ThreadLong = IRB.CreateLoad(SlotPtr); + Instruction *ThreadLong = IRB.CreateLoad(SlotPtr); + + Function *F = IRB.GetInsertBlock()->getParent(); + if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") { + Value *ThreadLongEqZero = + IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0)); + auto *Br = cast(SplitBlockAndInsertIfThen( + ThreadLongEqZero, cast(ThreadLongEqZero)->getNextNode(), + false, MDBuilder(*C).createBranchWeights(1, 100000))); + + IRB.SetInsertPoint(Br); + // FIXME: This should call a new runtime function with a custom calling + // convention to avoid needing to spill all arguments here. + IRB.CreateCall(HwasanThreadEnterFunc); + LoadInst *ReloadThreadLong = IRB.CreateLoad(SlotPtr); + + IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin()); + PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2); + ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent()); + ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent()); + ThreadLong = ThreadLongPhi; + } + // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI. Value *ThreadLongMaybeUntagged = TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); if (WithFrameRecord) { // Prepare ring buffer data. - Function *F = IRB.GetInsertBlock()->getParent(); auto PC = IRB.CreatePtrToInt(F, IntptrTy); auto GetStackPointerFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::frameaddress); Index: llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll =================================================================== --- /dev/null +++ llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll @@ -0,0 +1,25 @@ +; RUN: opt -S -hwasan < %s | FileCheck %s + +target triple = "x86_64-unknown-linux-gnu" + +declare void @bar([16 x i32]* %p) + +define void @foo() sanitize_hwaddress "hwasan-abi"="interceptor" { + ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls + ; CHECK: [[ICMP:%[^ ]*]] = icmp eq i64 [[LOAD]], 0 + ; CHECK: br i1 [[ICMP]], label %[[INIT:[^,]*]], label %[[CONT:[^,]*]], !prof [[PROF:![0-9]+]] + + ; CHECK: [[INIT]]: + ; CHECK: call void @__hwasan_thread_enter() + ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls + ; CHECK: br label %[[CONT]] + + ; CHECK: [[CONT]]: + ; CHECK: phi i64 [ [[LOAD]], %0 ], [ [[RELOAD]], %[[INIT]] ] + + %p = alloca [16 x i32] + call void @bar([16 x i32]* %p) + ret void +} + +; CHECK: [[PROF]] = !{!"branch_weights", i32 1, i32 100000}