Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -165,6 +165,7 @@ void initializeGuardWideningLegacyPassPass(PassRegistry&); void initializeHotColdSplittingLegacyPassPass(PassRegistry&); void initializeHWAddressSanitizerPass(PassRegistry&); +void initializeHWAddressSanitizerModulePass(PassRegistry &); void initializeIPCPPass(PassRegistry&); void initializeIPSCCPLegacyPassPass(PassRegistry&); void initializeIRCELegacyPassPass(PassRegistry&); Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -154,6 +154,7 @@ FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, bool Recover = false); +ModulePass *createHWAddressSanitizerModulePass(bool CompileKernel = false); // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation ModulePass *createDataFlowSanitizerPass( Index: lib/Transforms/Instrumentation/AddressSanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -2251,7 +2251,6 @@ std::string ELFUniqueModuleId = (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) ? getUniqueModuleId(&M) : ""; - if (!ELFUniqueModuleId.empty()) { InstrumentGlobalsELF(IRB, M, NewGlobals, Initializers, ELFUniqueModuleId); *CtorComdat = true; Index: lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -51,7 +51,12 @@ #define DEBUG_TYPE "hwasan" static const char *const kHwasanModuleCtorName = "hwasan.module_ctor"; +static const char *const kHwasanModuleDtorName = "hwasan.module_dtor"; static const char *const kHwasanInitName = "__hwasan_init"; +static const char *const kHwasanRegGlobalsName = "__hwasan_register_globals"; +static const char *const kHwasanUnregGlobalsName = + "__hwasan_unregister_globals"; +static const char *const kHwasanGlobalsListName = "__hwasan_globals_list"; static const char *const kHwasanShadowMemoryDynamicAddress = "__hwasan_shadow_memory_dynamic_address"; @@ -156,7 +161,17 @@ ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics", cl::desc("instrument memory intrinsics"), cl::Hidden, cl::init(true)); + +static cl::opt + ClInstrumentGlobals("hwasan-instrument-globals", + cl::desc("instrument global variables"), cl::Hidden, + cl::init(false)); + namespace { +// Get the section name for frame descriptions. Currently ELF-only. +inline const char *getFrameSection() { return "__hwasan_frames"; } +inline const char *getFrameSectionBeg() { return "__start___hwasan_frames"; } +inline const char *getFrameSectionEnd() { return "__stop___hwasan_frames"; } /// An instrumentation pass implementing detection of addressability bugs /// using tagged pointers. @@ -219,17 +234,6 @@ // the section beg/end to __hwasan_init_frames() at module init time. std::string createFrameString(ArrayRef Allocas); void createFrameGlobal(Function &F, const std::string &FrameString); - // Get the section name for frame descriptions. Currently ELF-only. - const char *getFrameSection() { return "__hwasan_frames"; } - const char *getFrameSectionBeg() { return "__start___hwasan_frames"; } - const char *getFrameSectionEnd() { return "__stop___hwasan_frames"; } - GlobalVariable *createFrameSectionBound(Module &M, Type *Ty, - const char *Name) { - auto GV = new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, - nullptr, Name); - GV->setVisibility(GlobalValue::HiddenVisibility); - return GV; - } /// This struct defines the shadow mapping using the rule: /// shadow = (mem >> Scale) + Offset. @@ -257,8 +261,6 @@ bool CompileKernel; bool Recover; - Function *HwasanCtorFunction; - Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes]; Function *HwasanMemoryAccessCallbackSized[2]; @@ -272,9 +274,39 @@ GlobalValue *ThreadPtrGlobal = nullptr; }; +class HWAddressSanitizerModule : public ModulePass { + bool CompileKernel; + Function *HwasanCtorFunction; + Type *Int8Ty, *Int8PtrTy, *Int64Ty, *VoidTy; + + void addGlobalsRegistrationCode(Module &M, GlobalVariable *AllGlobals, + size_t N); + IRBuilder<> createModuleDtor(Module &M); + bool globalWasGeneratedByCompiler(GlobalVariable *G); + bool shouldInstrumentGlobal(GlobalVariable *G); + GlobalVariable *makeGlobalVariableCopy(GlobalVariable *G); + GlobalVariable *createFrameSectionBound(Module &M, Type *Ty, + const char *Name); + +public: + // Pass identification, replacement for typeid + static char ID; + + explicit HWAddressSanitizerModule(bool CompileKernel = false) + : ModulePass(ID) { + this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 + ? ClEnableKhwasan + : CompileKernel; + } + + bool runOnModule(Module &M) override; + bool doInitialization(Module &M) override; + StringRef getPassName() const override { return "HWAddressSanitizerModule"; } +}; } // end anonymous namespace char HWAddressSanitizer::ID = 0; +char HWAddressSanitizerModule::ID = 0; INITIALIZE_PASS_BEGIN( HWAddressSanitizer, "hwasan", @@ -284,6 +316,11 @@ HWAddressSanitizer, "hwasan", "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false) +INITIALIZE_PASS( + HWAddressSanitizerModule, "hwasan-module", + "HWAddressSanitizer: detect memory bugs using tagged addressing." + "ModulePass", + false, false) FunctionPass *llvm::createHWAddressSanitizerPass(bool CompileKernel, bool Recover) { @@ -291,6 +328,10 @@ return new HWAddressSanitizer(CompileKernel, Recover); } +ModulePass *llvm::createHWAddressSanitizerModulePass(bool CompileKernel) { + return new HWAddressSanitizerModule(CompileKernel); +} + /// Module-level initialization. /// /// inserts a call to __hwasan_init to the module's constructor list. @@ -309,45 +350,11 @@ Int8PtrTy = IRB.getInt8PtrTy(); Int8Ty = IRB.getInt8Ty(); - HwasanCtorFunction = nullptr; - if (!CompileKernel) { - std::tie(HwasanCtorFunction, std::ignore) = - createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName, - kHwasanInitName, - /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName); - HwasanCtorFunction->setComdat(CtorComdat); - appendToGlobalCtors(M, HwasanCtorFunction, 0, HwasanCtorFunction); - - // Create a zero-length global in __hwasan_frame so that the linker will - // always create start and stop symbols. - // - // N.B. If we ever start creating associated metadata in this pass this - // global will need to be associated with the ctor. - Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0); - auto GV = - new GlobalVariable(M, Int8Arr0Ty, /*isConstantGlobal*/ true, - GlobalVariable::PrivateLinkage, - Constant::getNullValue(Int8Arr0Ty), "__hwasan"); - GV->setSection(getFrameSection()); - GV->setComdat(CtorComdat); - appendToCompilerUsed(M, GV); - - IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator()); - IRBCtor.CreateCall( - declareSanitizerInitFunction(M, "__hwasan_init_frames", - {Int8PtrTy, Int8PtrTy}), - {createFrameSectionBound(M, Int8Ty, getFrameSectionBeg()), - createFrameSectionBound(M, Int8Ty, getFrameSectionEnd())}); - } - if (!TargetTriple.isAndroid()) appendToCompilerUsed( M, ThreadPtrGlobal = new GlobalVariable( M, IntptrTy, false, GlobalVariable::ExternalLinkage, nullptr, "__hwasan_tls", nullptr, GlobalVariable::InitialExecTLSModel)); - return true; } @@ -944,7 +951,7 @@ } bool HWAddressSanitizer::runOnFunction(Function &F) { - if (&F == HwasanCtorFunction) + if (F.getName().startswith("__hwasan")) return false; if (!F.hasFnAttribute(Attribute::SanitizeHWAddress)) @@ -1039,3 +1046,196 @@ Offset = kDynamicShadowSentinel; } } + +bool HWAddressSanitizerModule::globalWasGeneratedByCompiler(GlobalVariable *G) { + StringRef Name = G->getName(); + return Name.startswith("__hwasan") || Name.startswith("llvm.") || + Name == getFrameSectionBeg() || Name == getFrameSectionEnd(); +} + +bool HWAddressSanitizerModule::shouldInstrumentGlobal(GlobalVariable *G) { + Type *Ty = G->getValueType(); + LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n"); + + if (!Ty->isSized() || G->isDeclaration()) + return false; + if (globalWasGeneratedByCompiler(G)) + return false; // Our own globals. + // Two problems with thread-locals: + // - The address of the main thread's copy can't be computed at link-time. + // - Need to poison all copies, not just the main thread's one. + if (G->isThreadLocal()) + return false; + // Values with interposable linkage can be replaced by the linker. + // With appending linkage we don't know size until the final link. + if (G->isInterposable() || G->hasAppendingLinkage()) + return false; + return G->hasName(); +} + +GlobalVariable * +HWAddressSanitizerModule::createFrameSectionBound(Module &M, Type *Ty, + const char *Name) { + auto GV = new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, + nullptr, Name); + GV->setVisibility(GlobalValue::HiddenVisibility); + return GV; +} + +bool HWAddressSanitizerModule::doInitialization(Module &M) { + if (CompileKernel) + return false; + + std::tie(HwasanCtorFunction, std::ignore) = + createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName, + kHwasanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}); + Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName); + HwasanCtorFunction->setComdat(CtorComdat); + appendToGlobalCtors(M, HwasanCtorFunction, 0, HwasanCtorFunction); + + IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator()); + Int8Ty = IRBCtor.getInt8Ty(); + Int8PtrTy = IRBCtor.getInt8PtrTy(); + Int64Ty = IRBCtor.getInt64Ty(); + VoidTy = IRBCtor.getVoidTy(); + // Create a zero-length global in __hwasan_frame so that the linker will + // always create start and stop symbols. + // + // N.B. If we ever start creating associated metadata in this pass this + // global will need to be associated with the ctor. + Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0); + auto GV = new GlobalVariable(M, Int8Arr0Ty, /*isConstantGlobal*/ true, + GlobalVariable::PrivateLinkage, + Constant::getNullValue(Int8Arr0Ty), "__hwasan"); + GV->setSection(getFrameSection()); + GV->setComdat(CtorComdat); + appendToCompilerUsed(M, GV); + + IRBCtor.CreateCall( + declareSanitizerInitFunction(M, "__hwasan_init_frames", + {Int8PtrTy, Int8PtrTy}), + {createFrameSectionBound(M, Int8Ty, getFrameSectionBeg()), + createFrameSectionBound(M, Int8Ty, getFrameSectionEnd())}); + return true; +} + +static size_t getTagForName(StringRef Name) { + // Use Knuth multiplicative hash to get tag value + return (hash_value(Name) * 2654435761) % 255 + 1; +} + +GlobalVariable * +HWAddressSanitizerModule::makeGlobalVariableCopy(GlobalVariable *G) { + GlobalValue::LinkageTypes Linkage = G->getLinkage(); + if (G->isConstant() && Linkage == GlobalValue::PrivateLinkage) + Linkage = GlobalValue::InternalLinkage; + GlobalVariable *NewGlobal = new GlobalVariable( + *G->getParent(), G->getValueType(), G->isConstant(), Linkage, + G->getInitializer(), "", G, G->getThreadLocalMode()); + NewGlobal->copyAttributesFrom(G); + NewGlobal->setComdat(G->getComdat()); + NewGlobal->setAlignment( + std::max(G->getAlignment(), 1u << kDefaultShadowScale)); + // Don't fold globals with memory tagging + NewGlobal->setUnnamedAddr(GlobalValue::UnnamedAddr::None); + return NewGlobal; +} + +IRBuilder<> HWAddressSanitizerModule::createModuleDtor(Module &M) { + LLVMContext &C = M.getContext(); + Function *Dtor = + Function::Create(FunctionType::get(VoidTy, false), + GlobalValue::InternalLinkage, kHwasanModuleDtorName, &M); + Dtor->setComdat(M.getOrInsertComdat(kHwasanModuleDtorName)); + appendToGlobalDtors(M, Dtor, 0, Dtor); + + BasicBlock *DtorBB = BasicBlock::Create(C, "", Dtor); + return IRBuilder<>(ReturnInst::Create(C, DtorBB)); +} + +void HWAddressSanitizerModule::addGlobalsRegistrationCode( + Module &M, GlobalVariable *AllGlobals, size_t N) { + Function *RegGlobals = checkSanitizerInterfaceFunction( + M.getOrInsertFunction(kHwasanRegGlobalsName, VoidTy, Int8PtrTy, Int64Ty)); + RegGlobals->setLinkage(Function::ExternalLinkage); + Function *UnregGlobals = + checkSanitizerInterfaceFunction(M.getOrInsertFunction( + kHwasanUnregGlobalsName, VoidTy, Int8PtrTy, Int64Ty)); + UnregGlobals->setLinkage(Function::ExternalLinkage); + + IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator()); + IRBCtor.CreateCall(RegGlobals, + {IRBCtor.CreatePointerCast(AllGlobals, Int8PtrTy), + ConstantInt::get(Int64Ty, N)}); + + IRBuilder<> IRBDtor = createModuleDtor(M); + IRBDtor.CreateCall(UnregGlobals, + {IRBDtor.CreatePointerCast(AllGlobals, Int8PtrTy), + ConstantInt::get(Int64Ty, N)}); +} + +static size_t AlignUp(size_t N, size_t Boundary) { + return (N + Boundary - 1) & ~(Boundary - 1); +} + +bool HWAddressSanitizerModule::runOnModule(Module &M) { + if (CompileKernel || !ClInstrumentGlobals) + return false; + + SmallVector GlobalsToChange; + + for (auto &G : M.globals()) + if (shouldInstrumentGlobal(&G)) + GlobalsToChange.push_back(&G); + + size_t N = GlobalsToChange.size(); + if (N == 0) + return false; + + IRBuilder<> IRB(M.getContext()); + // Global description structure type: + // - global pointer: i8* + // - size: i64 + // - tag: i64 + // size is aligned to (1 << kDefaultShadowScale) bytes + StructType *GlobalStructTy = StructType::get(Int8PtrTy, Int64Ty, Int64Ty); + SmallVector Initializers(N); + for (size_t i = 0; i < N; i++) { + GlobalVariable *G = GlobalsToChange[i]; + GlobalVariable *NewGlobal = makeGlobalVariableCopy(G); + + size_t Tag = getTagForName(G->getName()); + size_t SizeInBytes = M.getDataLayout().getTypeAllocSize(G->getValueType()); + Value *Indices[] = {IRB.getInt64(Tag << 56)}; + // Replace 'i64 *@foo' with + // 'i64* bitcast ( + // i8* getelementptr (i8, i8* bitcast (i64* @foo to i8*), i64 N) + // to i64*)' + // Where N is tag shifted left by 56 bits + G->replaceAllUsesWith(ConstantExpr::getBitCast( + ConstantExpr::getGetElementPtr( + nullptr, ConstantExpr::getBitCast(NewGlobal, Int8PtrTy), Indices), + NewGlobal->getType())); + NewGlobal->takeName(G); + G->eraseFromParent(); + + Initializers[i] = ConstantStruct::get( + GlobalStructTy, ConstantExpr::getPointerCast(NewGlobal, Int8PtrTy), + ConstantInt::get(Int64Ty, + AlignUp(SizeInBytes, 1 << kDefaultShadowScale)), + ConstantInt::get(Int64Ty, Tag)); + } + + ArrayType *ArrayOfGlobalStructTy = + ArrayType::get(Initializers[0]->getType(), N); + auto *AllGlobals = new GlobalVariable( + M, ArrayOfGlobalStructTy, false, GlobalVariable::InternalLinkage, + ConstantArray::get(ArrayOfGlobalStructTy, Initializers), + kHwasanGlobalsListName); + + appendToCompilerUsed(M, AllGlobals); + addGlobalsRegistrationCode(M, AllGlobals, N); + return true; +} \ No newline at end of file Index: lib/Transforms/Instrumentation/Instrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/Instrumentation.cpp +++ lib/Transforms/Instrumentation/Instrumentation.cpp @@ -113,6 +113,7 @@ initializeInstrProfilingLegacyPassPass(Registry); initializeMemorySanitizerLegacyPassPass(Registry); initializeHWAddressSanitizerPass(Registry); + initializeHWAddressSanitizerModulePass(Registry); initializeThreadSanitizerLegacyPassPass(Registry); initializeSanitizerCoverageModulePass(Registry); initializeDataFlowSanitizerPass(Registry); Index: test/Instrumentation/HWAddressSanitizer/globals.ll =================================================================== --- test/Instrumentation/HWAddressSanitizer/globals.ll +++ test/Instrumentation/HWAddressSanitizer/globals.ll @@ -0,0 +1,26 @@ +; RUN: opt -hwasan-module -hwasan-instrument-globals %s -S | FileCheck %s +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@q = dso_local local_unnamed_addr global i64 2, align 8 +; CHECK: @q = dso_local global i64 2, align 16 + +; 121 is a tag for @q, 16 is aligned size +; CHECK: @__hwasan_globals_list = internal global [1 x { i8*, i64, i64 }] [{ i8*, i64, i64 } { i8* bitcast (i64* @q to i8*), i64 16, i64 121 }] +; CHECK-NEXT: @llvm.compiler.used = {{.*}} @__hwasan_globals_list +; CHECK-NEXT: @llvm.global_dtors = {{.*}} @hwasan.module_dtor + +; Function Attrs: nounwind uwtable +define dso_local i32 @main() local_unnamed_addr { + tail call void @llvm.memset.p0i8.i64(i8* align 8 bitcast (i64* @q to i8*), i8 0, i64 32, i1 false) +; 8718968878589280256 == 121 << 56 +; CHECK: tail call void @llvm.memset.p0i8.i64(i8* align 8 getelementptr (i8, i8* bitcast (i64* @q to i8*), i64 8718968878589280256), i8 0, i64 32, i1 false) + ret i32 0 +} + +declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) + +; CHECK: define internal void @hwasan.module_ctor() +; CHECK-NEXT: call void @__hwasan_init() +; CHECK-NEXT: call void @__hwasan_init_frames(i8* @__start___hwasan_frames, i8* @__stop___hwasan_frames) +; CHECK-NEXT: call void @__hwasan_register_globals(i8* bitcast ([1 x { i8*, i64, i64 }]* @__hwasan_globals_list to i8*), i64 1)