Index: llvm/trunk/lib/Transforms/Utils/ModuleUtils.cpp =================================================================== --- llvm/trunk/lib/Transforms/Utils/ModuleUtils.cpp (revision 351313) +++ llvm/trunk/lib/Transforms/Utils/ModuleUtils.cpp (revision 351314) @@ -1,292 +1,314 @@ //===-- ModuleUtils.cpp - Functions to manipulate Modules -----------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This family of functions perform manipulations on Modules. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/ModuleUtils.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; static void appendToGlobalArray(const char *Array, Module &M, Function *F, int Priority, Constant *Data) { IRBuilder<> IRB(M.getContext()); FunctionType *FnTy = FunctionType::get(IRB.getVoidTy(), false); // Get the current set of static global constructors and add the new ctor // to the list. SmallVector CurrentCtors; StructType *EltTy; if (GlobalVariable *GVCtor = M.getNamedGlobal(Array)) { ArrayType *ATy = cast(GVCtor->getValueType()); StructType *OldEltTy = cast(ATy->getElementType()); // Upgrade a 2-field global array type to the new 3-field format if needed. if (Data && OldEltTy->getNumElements() < 3) EltTy = StructType::get(IRB.getInt32Ty(), PointerType::getUnqual(FnTy), IRB.getInt8PtrTy()); else EltTy = OldEltTy; if (Constant *Init = GVCtor->getInitializer()) { unsigned n = Init->getNumOperands(); CurrentCtors.reserve(n + 1); for (unsigned i = 0; i != n; ++i) { auto Ctor = cast(Init->getOperand(i)); if (EltTy != OldEltTy) Ctor = ConstantStruct::get(EltTy, Ctor->getAggregateElement((unsigned)0), Ctor->getAggregateElement(1), Constant::getNullValue(IRB.getInt8PtrTy())); CurrentCtors.push_back(Ctor); } } GVCtor->eraseFromParent(); } else { // Use the new three-field struct if there isn't one already. EltTy = StructType::get(IRB.getInt32Ty(), PointerType::getUnqual(FnTy), IRB.getInt8PtrTy()); } // Build a 2 or 3 field global_ctor entry. We don't take a comdat key. Constant *CSVals[3]; CSVals[0] = IRB.getInt32(Priority); CSVals[1] = F; // FIXME: Drop support for the two element form in LLVM 4.0. if (EltTy->getNumElements() >= 3) CSVals[2] = Data ? ConstantExpr::getPointerCast(Data, IRB.getInt8PtrTy()) : Constant::getNullValue(IRB.getInt8PtrTy()); Constant *RuntimeCtorInit = ConstantStruct::get(EltTy, makeArrayRef(CSVals, EltTy->getNumElements())); CurrentCtors.push_back(RuntimeCtorInit); // Create a new initializer. ArrayType *AT = ArrayType::get(EltTy, CurrentCtors.size()); Constant *NewInit = ConstantArray::get(AT, CurrentCtors); // Create the new global variable and replace all uses of // the old global variable with the new one. (void)new GlobalVariable(M, NewInit->getType(), false, GlobalValue::AppendingLinkage, NewInit, Array); } void llvm::appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data) { appendToGlobalArray("llvm.global_ctors", M, F, Priority, Data); } void llvm::appendToGlobalDtors(Module &M, Function *F, int Priority, Constant *Data) { appendToGlobalArray("llvm.global_dtors", M, F, Priority, Data); } static void appendToUsedList(Module &M, StringRef Name, ArrayRef Values) { GlobalVariable *GV = M.getGlobalVariable(Name); SmallPtrSet InitAsSet; SmallVector Init; if (GV) { ConstantArray *CA = dyn_cast(GV->getInitializer()); for (auto &Op : CA->operands()) { Constant *C = cast_or_null(Op); if (InitAsSet.insert(C).second) Init.push_back(C); } GV->eraseFromParent(); } Type *Int8PtrTy = llvm::Type::getInt8PtrTy(M.getContext()); for (auto *V : Values) { Constant *C = ConstantExpr::getBitCast(V, Int8PtrTy); if (InitAsSet.insert(C).second) Init.push_back(C); } if (Init.empty()) return; ArrayType *ATy = ArrayType::get(Int8PtrTy, Init.size()); GV = new llvm::GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage, ConstantArray::get(ATy, Init), Name); GV->setSection("llvm.metadata"); } void llvm::appendToUsed(Module &M, ArrayRef Values) { appendToUsedList(M, "llvm.used", Values); } void llvm::appendToCompilerUsed(Module &M, ArrayRef Values) { appendToUsedList(M, "llvm.compiler.used", Values); } Function *llvm::checkSanitizerInterfaceFunction(Constant *FuncOrBitcast) { if (isa(FuncOrBitcast)) return cast(FuncOrBitcast); FuncOrBitcast->print(errs()); errs() << '\n'; std::string Err; raw_string_ostream Stream(Err); Stream << "Sanitizer interface function redefined: " << *FuncOrBitcast; report_fatal_error(Err); } Function *llvm::declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef InitArgTypes) { assert(!InitName.empty() && "Expected init function name"); Function *F = checkSanitizerInterfaceFunction(M.getOrInsertFunction( InitName, FunctionType::get(Type::getVoidTy(M.getContext()), InitArgTypes, false), AttributeList())); F->setLinkage(Function::ExternalLinkage); return F; } std::pair llvm::createSanitizerCtorAndInitFunctions( Module &M, StringRef CtorName, StringRef InitName, ArrayRef InitArgTypes, ArrayRef InitArgs, StringRef VersionCheckName) { assert(!InitName.empty() && "Expected init function name"); assert(InitArgs.size() == InitArgTypes.size() && "Sanitizer's init function expects different number of arguments"); Function *InitFunction = declareSanitizerInitFunction(M, InitName, InitArgTypes); Function *Ctor = Function::Create( FunctionType::get(Type::getVoidTy(M.getContext()), false), GlobalValue::InternalLinkage, CtorName, &M); BasicBlock *CtorBB = BasicBlock::Create(M.getContext(), "", Ctor); IRBuilder<> IRB(ReturnInst::Create(M.getContext(), CtorBB)); IRB.CreateCall(InitFunction, InitArgs); if (!VersionCheckName.empty()) { Function *VersionCheckFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction( VersionCheckName, FunctionType::get(IRB.getVoidTy(), {}, false), AttributeList())); IRB.CreateCall(VersionCheckFunction, {}); } return std::make_pair(Ctor, InitFunction); } +std::pair +llvm::getOrCreateSanitizerCtorAndInitFunctions( + Module &M, StringRef CtorName, StringRef InitName, + ArrayRef InitArgTypes, ArrayRef InitArgs, + function_ref FunctionsCreatedCallback, + StringRef VersionCheckName) { + assert(!CtorName.empty() && "Expected ctor function name"); + + if (Function *Ctor = M.getFunction(CtorName)) + // FIXME: Sink this logic into the module, similar to the handling of + // globals. This will make moving to a concurrent model much easier. + if (Ctor->arg_size() == 0 || + Ctor->getReturnType() == Type::getVoidTy(M.getContext())) + return {Ctor, declareSanitizerInitFunction(M, InitName, InitArgTypes)}; + + Function *Ctor, *InitFunction; + std::tie(Ctor, InitFunction) = llvm::createSanitizerCtorAndInitFunctions( + M, CtorName, InitName, InitArgTypes, InitArgs, VersionCheckName); + FunctionsCreatedCallback(Ctor, InitFunction); + return std::make_pair(Ctor, InitFunction); +} + Function *llvm::getOrCreateInitFunction(Module &M, StringRef Name) { assert(!Name.empty() && "Expected init function name"); if (Function *F = M.getFunction(Name)) { if (F->arg_size() != 0 || F->getReturnType() != Type::getVoidTy(M.getContext())) { std::string Err; raw_string_ostream Stream(Err); Stream << "Sanitizer interface function defined with wrong type: " << *F; report_fatal_error(Err); } return F; } Function *F = checkSanitizerInterfaceFunction(M.getOrInsertFunction( Name, AttributeList(), Type::getVoidTy(M.getContext()))); F->setLinkage(Function::ExternalLinkage); appendToGlobalCtors(M, F, 0); return F; } void llvm::filterDeadComdatFunctions( Module &M, SmallVectorImpl &DeadComdatFunctions) { // Build a map from the comdat to the number of entries in that comdat we // think are dead. If this fully covers the comdat group, then the entire // group is dead. If we find another entry in the comdat group though, we'll // have to preserve the whole group. SmallDenseMap ComdatEntriesCovered; for (Function *F : DeadComdatFunctions) { Comdat *C = F->getComdat(); assert(C && "Expected all input GVs to be in a comdat!"); ComdatEntriesCovered[C] += 1; } auto CheckComdat = [&](Comdat &C) { auto CI = ComdatEntriesCovered.find(&C); if (CI == ComdatEntriesCovered.end()) return; // If this could have been covered by a dead entry, just subtract one to // account for it. if (CI->second > 0) { CI->second -= 1; return; } // If we've already accounted for all the entries that were dead, the // entire comdat is alive so remove it from the map. ComdatEntriesCovered.erase(CI); }; auto CheckAllComdats = [&] { for (Function &F : M.functions()) if (Comdat *C = F.getComdat()) { CheckComdat(*C); if (ComdatEntriesCovered.empty()) return; } for (GlobalVariable &GV : M.globals()) if (Comdat *C = GV.getComdat()) { CheckComdat(*C); if (ComdatEntriesCovered.empty()) return; } for (GlobalAlias &GA : M.aliases()) if (Comdat *C = GA.getComdat()) { CheckComdat(*C); if (ComdatEntriesCovered.empty()) return; } }; CheckAllComdats(); if (ComdatEntriesCovered.empty()) { DeadComdatFunctions.clear(); return; } // Remove the entries that were not covering. erase_if(DeadComdatFunctions, [&](GlobalValue *GV) { return ComdatEntriesCovered.find(GV->getComdat()) == ComdatEntriesCovered.end(); }); } std::string llvm::getUniqueModuleId(Module *M) { MD5 Md5; bool ExportsSymbols = false; auto AddGlobal = [&](GlobalValue &GV) { if (GV.isDeclaration() || GV.getName().startswith("llvm.") || !GV.hasExternalLinkage() || GV.hasComdat()) return; ExportsSymbols = true; Md5.update(GV.getName()); Md5.update(ArrayRef{0}); }; for (auto &F : *M) AddGlobal(F); for (auto &GV : M->globals()) AddGlobal(GV); for (auto &GA : M->aliases()) AddGlobal(GA); for (auto &IF : M->ifuncs()) AddGlobal(IF); if (!ExportsSymbols) return ""; MD5::MD5Result R; Md5.final(R); SmallString<32> Str; MD5::stringifyResult(R, Str); return ("$" + Str).str(); } Index: llvm/trunk/lib/Transforms/Instrumentation/ThreadSanitizer.cpp =================================================================== --- llvm/trunk/lib/Transforms/Instrumentation/ThreadSanitizer.cpp (revision 351313) +++ llvm/trunk/lib/Transforms/Instrumentation/ThreadSanitizer.cpp (revision 351314) @@ -1,703 +1,732 @@ //===-- ThreadSanitizer.cpp - race detector -------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer, a race detector. // // The tool is under development, for the details about previous versions see // http://code.google.com/p/data-race-test // // The instrumentation phase is quite simple: // - Insert calls to run-time library before every memory access. // - Optimizations may apply to avoid instrumenting some of the accesses. // - Insert calls at function entry/exit. // The rest is handled by the run-time library. //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/EscapeEnumerator.h" #include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; #define DEBUG_TYPE "tsan" static cl::opt ClInstrumentMemoryAccesses( "tsan-instrument-memory-accesses", cl::init(true), cl::desc("Instrument memory accesses"), cl::Hidden); static cl::opt ClInstrumentFuncEntryExit( "tsan-instrument-func-entry-exit", cl::init(true), cl::desc("Instrument function entry and exit"), cl::Hidden); static cl::opt ClHandleCxxExceptions( "tsan-handle-cxx-exceptions", cl::init(true), cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), cl::Hidden); static cl::opt ClInstrumentAtomics( "tsan-instrument-atomics", cl::init(true), cl::desc("Instrument atomics"), cl::Hidden); static cl::opt ClInstrumentMemIntrinsics( "tsan-instrument-memintrinsics", cl::init(true), cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); STATISTIC(NumOmittedReadsBeforeWrite, "Number of reads ignored due to following writes"); STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); STATISTIC(NumOmittedReadsFromConstantGlobals, "Number of reads from constant globals"); STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); static const char *const kTsanModuleCtorName = "tsan.module_ctor"; static const char *const kTsanInitName = "__tsan_init"; namespace { /// ThreadSanitizer: instrument the code in module to find races. -struct ThreadSanitizer : public FunctionPass { - ThreadSanitizer() : FunctionPass(ID) {} - StringRef getPassName() const override; - void getAnalysisUsage(AnalysisUsage &AU) const override; - bool runOnFunction(Function &F) override; - bool doInitialization(Module &M) override; - static char ID; // Pass identification, replacement for typeid. +/// +/// Instantiating ThreadSanitizer inserts the tsan runtime library API function +/// declarations into the module if they don't exist already. Instantiating +/// ensures the __tsan_init function is in the list of global constructors for +/// the module. +struct ThreadSanitizer { + ThreadSanitizer(Module &M); + bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); - private: +private: void initializeCallbacks(Module &M); bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); bool instrumentAtomic(Instruction *I, const DataLayout &DL); bool instrumentMemIntrinsic(Instruction *I); void chooseInstructionsToInstrument(SmallVectorImpl &Local, SmallVectorImpl &All, const DataLayout &DL); bool addrPointsToConstantData(Value *Addr); int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); void InsertRuntimeIgnores(Function &F); Type *IntptrTy; IntegerType *OrdTy; // Callbacks to run-time library are computed in doInitialization. Function *TsanFuncEntry; Function *TsanFuncExit; Function *TsanIgnoreBegin; Function *TsanIgnoreEnd; // Accesses sizes are powers of two: 1, 2, 4, 8, 16. static const size_t kNumberOfAccessSizes = 5; Function *TsanRead[kNumberOfAccessSizes]; Function *TsanWrite[kNumberOfAccessSizes]; Function *TsanUnalignedRead[kNumberOfAccessSizes]; Function *TsanUnalignedWrite[kNumberOfAccessSizes]; Function *TsanAtomicLoad[kNumberOfAccessSizes]; Function *TsanAtomicStore[kNumberOfAccessSizes]; Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; Function *TsanAtomicCAS[kNumberOfAccessSizes]; Function *TsanAtomicThreadFence; Function *TsanAtomicSignalFence; Function *TsanVptrUpdate; Function *TsanVptrLoad; Function *MemmoveFn, *MemcpyFn, *MemsetFn; Function *TsanCtorFunction; }; + +struct ThreadSanitizerLegacyPass : FunctionPass { + ThreadSanitizerLegacyPass() : FunctionPass(ID) {} + StringRef getPassName() const override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &F) override; + bool doInitialization(Module &M) override; + static char ID; // Pass identification, replacement for typeid. +private: + Optional TSan; +}; } // namespace -char ThreadSanitizer::ID = 0; -INITIALIZE_PASS_BEGIN( - ThreadSanitizer, "tsan", - "ThreadSanitizer: detects data races.", - false, false) +PreservedAnalyses ThreadSanitizerPass::run(Function &F, + FunctionAnalysisManager &FAM) { + ThreadSanitizer TSan(*F.getParent()); + if (TSan.sanitizeFunction(F, FAM.getResult(F))) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +char ThreadSanitizerLegacyPass::ID = 0; +INITIALIZE_PASS_BEGIN(ThreadSanitizerLegacyPass, "tsan", + "ThreadSanitizer: detects data races.", false, false) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) -INITIALIZE_PASS_END( - ThreadSanitizer, "tsan", - "ThreadSanitizer: detects data races.", - false, false) +INITIALIZE_PASS_END(ThreadSanitizerLegacyPass, "tsan", + "ThreadSanitizer: detects data races.", false, false) -StringRef ThreadSanitizer::getPassName() const { return "ThreadSanitizer"; } +StringRef ThreadSanitizerLegacyPass::getPassName() const { + return "ThreadSanitizerLegacyPass"; +} -void ThreadSanitizer::getAnalysisUsage(AnalysisUsage &AU) const { +void ThreadSanitizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); } -FunctionPass *llvm::createThreadSanitizerPass() { - return new ThreadSanitizer(); +bool ThreadSanitizerLegacyPass::doInitialization(Module &M) { + TSan.emplace(M); + return true; +} + +bool ThreadSanitizerLegacyPass::runOnFunction(Function &F) { + auto &TLI = getAnalysis().getTLI(); + TSan->sanitizeFunction(F, TLI); + return true; +} + +FunctionPass *llvm::createThreadSanitizerLegacyPassPass() { + return new ThreadSanitizerLegacyPass(); } void ThreadSanitizer::initializeCallbacks(Module &M) { IRBuilder<> IRB(M.getContext()); AttributeList Attr; Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex, Attribute::NoUnwind); // Initialize the callbacks. TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_func_entry", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); TsanFuncExit = checkSanitizerInterfaceFunction( M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy())); TsanIgnoreBegin = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_ignore_thread_begin", Attr, IRB.getVoidTy())); TsanIgnoreEnd = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_ignore_thread_end", Attr, IRB.getVoidTy())); OrdTy = IRB.getInt32Ty(); for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { const unsigned ByteSize = 1U << i; const unsigned BitSize = ByteSize * 8; std::string ByteSizeStr = utostr(ByteSize); std::string BitSizeStr = utostr(BitSize); SmallString<32> ReadName("__tsan_read" + ByteSizeStr); TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( ReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); SmallString<32> WriteName("__tsan_write" + ByteSizeStr); TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( WriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); TsanUnalignedRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); TsanUnalignedWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); Type *Ty = Type::getIntNTy(M.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy)); SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy)); for (int op = AtomicRMWInst::FIRST_BINOP; op <= AtomicRMWInst::LAST_BINOP; ++op) { TsanAtomicRMW[op][i] = nullptr; const char *NamePart = nullptr; if (op == AtomicRMWInst::Xchg) NamePart = "_exchange"; else if (op == AtomicRMWInst::Add) NamePart = "_fetch_add"; else if (op == AtomicRMWInst::Sub) NamePart = "_fetch_sub"; else if (op == AtomicRMWInst::And) NamePart = "_fetch_and"; else if (op == AtomicRMWInst::Or) NamePart = "_fetch_or"; else if (op == AtomicRMWInst::Xor) NamePart = "_fetch_xor"; else if (op == AtomicRMWInst::Nand) NamePart = "_fetch_nand"; else continue; SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy)); } SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + "_compare_exchange_val"); TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( AtomicCASName, Attr, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy)); } TsanVptrUpdate = checkSanitizerInterfaceFunction( M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy())); TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_vptr_read", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_atomic_thread_fence", Attr, IRB.getVoidTy(), OrdTy)); TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( "__tsan_atomic_signal_fence", Attr, IRB.getVoidTy(), OrdTy)); MemmoveFn = checkSanitizerInterfaceFunction( M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); MemcpyFn = checkSanitizerInterfaceFunction( M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); MemsetFn = checkSanitizerInterfaceFunction( M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy)); } -bool ThreadSanitizer::doInitialization(Module &M) { +ThreadSanitizer::ThreadSanitizer(Module &M) { const DataLayout &DL = M.getDataLayout(); IntptrTy = DL.getIntPtrType(M.getContext()); - std::tie(TsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( - M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - - appendToGlobalCtors(M, TsanCtorFunction, 0); - - return true; + std::tie(TsanCtorFunction, std::ignore) = + getOrCreateSanitizerCtorAndInitFunctions( + M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + // This callback is invoked when the functions are created the first + // time. Hook them into the global ctors list in that case: + [&](Function *Ctor, Function *) { appendToGlobalCtors(M, Ctor, 0); }); } static bool isVtableAccess(Instruction *I) { if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) return Tag->isTBAAVtableAccess(); return false; } // Do not instrument known races/"benign races" that come from compiler // instrumentatin. The user has no way of suppressing them. static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { // Peel off GEPs and BitCasts. Addr = Addr->stripInBoundsOffsets(); if (GlobalVariable *GV = dyn_cast(Addr)) { if (GV->hasSection()) { StringRef SectionName = GV->getSection(); // Check if the global is in the PGO counters section. auto OF = Triple(M->getTargetTriple()).getObjectFormat(); if (SectionName.endswith( getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) return false; } // Check if the global is private gcov data. if (GV->getName().startswith("__llvm_gcov") || GV->getName().startswith("__llvm_gcda")) return false; } // Do not instrument acesses from different address spaces; we cannot deal // with them. if (Addr) { Type *PtrTy = cast(Addr->getType()->getScalarType()); if (PtrTy->getPointerAddressSpace() != 0) return false; } return true; } bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { // If this is a GEP, just analyze its pointer operand. if (GetElementPtrInst *GEP = dyn_cast(Addr)) Addr = GEP->getPointerOperand(); if (GlobalVariable *GV = dyn_cast(Addr)) { if (GV->isConstant()) { // Reads from constant globals can not race with any writes. NumOmittedReadsFromConstantGlobals++; return true; } } else if (LoadInst *L = dyn_cast(Addr)) { if (isVtableAccess(L)) { // Reads from a vtable pointer can not race with any writes. NumOmittedReadsFromVtable++; return true; } } return false; } // Instrumenting some of the accesses may be proven redundant. // Currently handled: // - read-before-write (within same BB, no calls between) // - not captured variables // // We do not handle some of the patterns that should not survive // after the classic compiler optimizations. // E.g. two reads from the same temp should be eliminated by CSE, // two writes should be eliminated by DSE, etc. // // 'Local' is a vector of insns within the same BB (no calls between). // 'All' is a vector of insns that will be instrumented. void ThreadSanitizer::chooseInstructionsToInstrument( SmallVectorImpl &Local, SmallVectorImpl &All, const DataLayout &DL) { SmallPtrSet WriteTargets; // Iterate from the end. for (Instruction *I : reverse(Local)) { if (StoreInst *Store = dyn_cast(I)) { Value *Addr = Store->getPointerOperand(); if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) continue; WriteTargets.insert(Addr); } else { LoadInst *Load = cast(I); Value *Addr = Load->getPointerOperand(); if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) continue; if (WriteTargets.count(Addr)) { // We will write to this temp, so no reason to analyze the read. NumOmittedReadsBeforeWrite++; continue; } if (addrPointsToConstantData(Addr)) { // Addr points to some constant data -- it can not race with any writes. continue; } } Value *Addr = isa(*I) ? cast(I)->getPointerOperand() : cast(I)->getPointerOperand(); if (isa(GetUnderlyingObject(Addr, DL)) && !PointerMayBeCaptured(Addr, true, true)) { // The variable is addressable but not captured, so it cannot be // referenced from a different thread and participate in a data race // (see llvm/Analysis/CaptureTracking.h for details). NumOmittedNonCaptured++; continue; } All.push_back(I); } Local.clear(); } static bool isAtomic(Instruction *I) { // TODO: Ask TTI whether synchronization scope is between threads. if (LoadInst *LI = dyn_cast(I)) return LI->isAtomic() && LI->getSyncScopeID() != SyncScope::SingleThread; if (StoreInst *SI = dyn_cast(I)) return SI->isAtomic() && SI->getSyncScopeID() != SyncScope::SingleThread; if (isa(I)) return true; if (isa(I)) return true; if (isa(I)) return true; return false; } void ThreadSanitizer::InsertRuntimeIgnores(Function &F) { IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); IRB.CreateCall(TsanIgnoreBegin); EscapeEnumerator EE(F, "tsan_ignore_cleanup", ClHandleCxxExceptions); while (IRBuilder<> *AtExit = EE.Next()) { AtExit->CreateCall(TsanIgnoreEnd); } } -bool ThreadSanitizer::runOnFunction(Function &F) { +bool ThreadSanitizer::sanitizeFunction(Function &F, + const TargetLibraryInfo &TLI) { // This is required to prevent instrumenting call to __tsan_init from within // the module constructor. if (&F == TsanCtorFunction) return false; initializeCallbacks(*F.getParent()); SmallVector AllLoadsAndStores; SmallVector LocalLoadsAndStores; SmallVector AtomicAccesses; SmallVector MemIntrinCalls; bool Res = false; bool HasCalls = false; bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); const DataLayout &DL = F.getParent()->getDataLayout(); - const TargetLibraryInfo *TLI = - &getAnalysis().getTLI(); // Traverse all instructions, collect loads/stores/returns, check for calls. for (auto &BB : F) { for (auto &Inst : BB) { if (isAtomic(&Inst)) AtomicAccesses.push_back(&Inst); else if (isa(Inst) || isa(Inst)) LocalLoadsAndStores.push_back(&Inst); else if (isa(Inst) || isa(Inst)) { if (CallInst *CI = dyn_cast(&Inst)) - maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI); + maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); if (isa(Inst)) MemIntrinCalls.push_back(&Inst); HasCalls = true; chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); } } chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); } // We have collected all loads and stores. // FIXME: many of these accesses do not need to be checked for races // (e.g. variables that do not escape, etc). // Instrument memory accesses only if we want to report bugs in the function. if (ClInstrumentMemoryAccesses && SanitizeFunction) for (auto Inst : AllLoadsAndStores) { Res |= instrumentLoadOrStore(Inst, DL); } // Instrument atomic memory accesses in any case (they can be used to // implement synchronization). if (ClInstrumentAtomics) for (auto Inst : AtomicAccesses) { Res |= instrumentAtomic(Inst, DL); } if (ClInstrumentMemIntrinsics && SanitizeFunction) for (auto Inst : MemIntrinCalls) { Res |= instrumentMemIntrinsic(Inst); } if (F.hasFnAttribute("sanitize_thread_no_checking_at_run_time")) { assert(!F.hasFnAttribute(Attribute::SanitizeThread)); if (HasCalls) InsertRuntimeIgnores(F); } // Instrument function entry/exit points if there were instrumented accesses. if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); Value *ReturnAddress = IRB.CreateCall( Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), IRB.getInt32(0)); IRB.CreateCall(TsanFuncEntry, ReturnAddress); EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions); while (IRBuilder<> *AtExit = EE.Next()) { AtExit->CreateCall(TsanFuncExit, {}); } Res = true; } return Res; } bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, const DataLayout &DL) { IRBuilder<> IRB(I); bool IsWrite = isa(*I); Value *Addr = IsWrite ? cast(I)->getPointerOperand() : cast(I)->getPointerOperand(); // swifterror memory addresses are mem2reg promoted by instruction selection. // As such they cannot have regular uses like an instrumentation function and // it makes no sense to track them as memory. if (Addr->isSwiftError()) return false; int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; if (IsWrite && isVtableAccess(I)) { LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n"); Value *StoredValue = cast(I)->getValueOperand(); // StoredValue may be a vector type if we are storing several vptrs at once. // In this case, just take the first element of the vector since this is // enough to find vptr races. if (isa(StoredValue->getType())) StoredValue = IRB.CreateExtractElement( StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); if (StoredValue->getType()->isIntegerTy()) StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); // Call TsanVptrUpdate. IRB.CreateCall(TsanVptrUpdate, {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); NumInstrumentedVtableWrites++; return true; } if (!IsWrite && isVtableAccess(I)) { IRB.CreateCall(TsanVptrLoad, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); NumInstrumentedVtableReads++; return true; } const unsigned Alignment = IsWrite ? cast(I)->getAlignment() : cast(I)->getAlignment(); Type *OrigTy = cast(Addr->getType())->getElementType(); const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); Value *OnAccessFunc = nullptr; if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; else OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); if (IsWrite) NumInstrumentedWrites++; else NumInstrumentedReads++; return true; } static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { uint32_t v = 0; switch (ord) { case AtomicOrdering::NotAtomic: llvm_unreachable("unexpected atomic ordering!"); case AtomicOrdering::Unordered: LLVM_FALLTHROUGH; case AtomicOrdering::Monotonic: v = 0; break; // Not specified yet: // case AtomicOrdering::Consume: v = 1; break; case AtomicOrdering::Acquire: v = 2; break; case AtomicOrdering::Release: v = 3; break; case AtomicOrdering::AcquireRelease: v = 4; break; case AtomicOrdering::SequentiallyConsistent: v = 5; break; } return IRB->getInt32(v); } // If a memset intrinsic gets inlined by the code gen, we will miss races on it. // So, we either need to ensure the intrinsic is not inlined, or instrument it. // We do not instrument memset/memmove/memcpy intrinsics (too complicated), // instead we simply replace them with regular function calls, which are then // intercepted by the run-time. // Since tsan is running after everyone else, the calls should not be // replaced back with intrinsics. If that becomes wrong at some point, // we will need to call e.g. __tsan_memset to avoid the intrinsics. bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { IRBuilder<> IRB(I); if (MemSetInst *M = dyn_cast(I)) { IRB.CreateCall( MemsetFn, {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); I->eraseFromParent(); } else if (MemTransferInst *M = dyn_cast(I)) { IRB.CreateCall( isa(M) ? MemcpyFn : MemmoveFn, {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); I->eraseFromParent(); } return false; } // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x // standards. For background see C++11 standard. A slightly older, publicly // available draft of the standard (not entirely up-to-date, but close enough // for casual browsing) is available here: // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf // The following page contains more background information: // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { IRBuilder<> IRB(I); if (LoadInst *LI = dyn_cast(I)) { Value *Addr = LI->getPointerOperand(); int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; const unsigned ByteSize = 1U << Idx; const unsigned BitSize = ByteSize * 8; Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), createOrdering(&IRB, LI->getOrdering())}; Type *OrigTy = cast(Addr->getType())->getElementType(); Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy); I->replaceAllUsesWith(Cast); } else if (StoreInst *SI = dyn_cast(I)) { Value *Addr = SI->getPointerOperand(); int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; const unsigned ByteSize = 1U << Idx; const unsigned BitSize = ByteSize * 8; Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty), createOrdering(&IRB, SI->getOrdering())}; CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); ReplaceInstWithInst(I, C); } else if (AtomicRMWInst *RMWI = dyn_cast(I)) { Value *Addr = RMWI->getPointerOperand(); int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; if (!F) return false; const unsigned ByteSize = 1U << Idx; const unsigned BitSize = ByteSize * 8; Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), createOrdering(&IRB, RMWI->getOrdering())}; CallInst *C = CallInst::Create(F, Args); ReplaceInstWithInst(I, C); } else if (AtomicCmpXchgInst *CASI = dyn_cast(I)) { Value *Addr = CASI->getPointerOperand(); int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; const unsigned ByteSize = 1U << Idx; const unsigned BitSize = ByteSize * 8; Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); Value *CmpOperand = IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty); Value *NewOperand = IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty); Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), CmpOperand, NewOperand, createOrdering(&IRB, CASI->getSuccessOrdering()), createOrdering(&IRB, CASI->getFailureOrdering())}; CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); Value *Success = IRB.CreateICmpEQ(C, CmpOperand); Value *OldVal = C; Type *OrigOldValTy = CASI->getNewValOperand()->getType(); if (Ty != OrigOldValTy) { // The value is a pointer, so we need to cast the return value. OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); } Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0); Res = IRB.CreateInsertValue(Res, Success, 1); I->replaceAllUsesWith(Res); I->eraseFromParent(); } else if (FenceInst *FI = dyn_cast(I)) { Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; Function *F = FI->getSyncScopeID() == SyncScope::SingleThread ? TsanAtomicSignalFence : TsanAtomicThreadFence; CallInst *C = CallInst::Create(F, Args); ReplaceInstWithInst(I, C); } return true; } int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL) { Type *OrigPtrTy = Addr->getType(); Type *OrigTy = cast(OrigPtrTy)->getElementType(); assert(OrigTy->isSized()); uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { NumAccessesWithBadSize++; // Ignore all unusual sizes. return -1; } size_t Idx = countTrailingZeros(TypeSize / 8); assert(Idx < kNumberOfAccessSizes); return Idx; } Index: llvm/trunk/lib/Transforms/Instrumentation/Instrumentation.cpp =================================================================== --- llvm/trunk/lib/Transforms/Instrumentation/Instrumentation.cpp (revision 351313) +++ llvm/trunk/lib/Transforms/Instrumentation/Instrumentation.cpp (revision 351314) @@ -1,126 +1,126 @@ //===-- Instrumentation.cpp - TransformUtils Infrastructure ---------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file defines the common initialization infrastructure for the // Instrumentation library. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Instrumentation.h" #include "llvm-c/Initialization.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/InitializePasses.h" #include "llvm/PassRegistry.h" using namespace llvm; /// Moves I before IP. Returns new insert point. static BasicBlock::iterator moveBeforeInsertPoint(BasicBlock::iterator I, BasicBlock::iterator IP) { // If I is IP, move the insert point down. if (I == IP) return ++IP; // Otherwise, move I before IP and return IP. I->moveBefore(&*IP); return IP; } /// Instrumentation passes often insert conditional checks into entry blocks. /// Call this function before splitting the entry block to move instructions /// that must remain in the entry block up before the split point. Static /// allocas and llvm.localescape calls, for example, must remain in the entry /// block. BasicBlock::iterator llvm::PrepareToSplitEntryBlock(BasicBlock &BB, BasicBlock::iterator IP) { assert(&BB.getParent()->getEntryBlock() == &BB); for (auto I = IP, E = BB.end(); I != E; ++I) { bool KeepInEntry = false; if (auto *AI = dyn_cast(I)) { if (AI->isStaticAlloca()) KeepInEntry = true; } else if (auto *II = dyn_cast(I)) { if (II->getIntrinsicID() == llvm::Intrinsic::localescape) KeepInEntry = true; } if (KeepInEntry) IP = moveBeforeInsertPoint(I, IP); } return IP; } // Create a constant for Str so that we can pass it to the run-time lib. GlobalVariable *llvm::createPrivateGlobalForString(Module &M, StringRef Str, bool AllowMerging, const char *NamePrefix) { Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str); // We use private linkage for module-local strings. If they can be merged // with another one, we set the unnamed_addr attribute. GlobalVariable *GV = new GlobalVariable(M, StrConst->getType(), true, GlobalValue::PrivateLinkage, StrConst, NamePrefix); if (AllowMerging) GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); GV->setAlignment(1); // Strings may not be merged w/o setting align 1. return GV; } Comdat *llvm::GetOrCreateFunctionComdat(Function &F, Triple &T, const std::string &ModuleId) { if (auto Comdat = F.getComdat()) return Comdat; assert(F.hasName()); Module *M = F.getParent(); std::string Name = F.getName(); // Make a unique comdat name for internal linkage things on ELF. On COFF, the // name of the comdat group identifies the leader symbol of the comdat group. // The linkage of the leader symbol is considered during comdat resolution, // and internal symbols with the same name from different objects will not be // merged. if (T.isOSBinFormatELF() && F.hasLocalLinkage()) { if (ModuleId.empty()) return nullptr; Name += ModuleId; } // Make a new comdat for the function. Use the "no duplicates" selection kind // for non-weak symbols if the object file format supports it. Comdat *C = M->getOrInsertComdat(Name); if (T.isOSBinFormatCOFF() && !F.isWeakForLinker()) C->setSelectionKind(Comdat::NoDuplicates); F.setComdat(C); return C; } /// initializeInstrumentation - Initialize all passes in the TransformUtils /// library. void llvm::initializeInstrumentation(PassRegistry &Registry) { initializeAddressSanitizerPass(Registry); initializeAddressSanitizerModulePass(Registry); initializeBoundsCheckingLegacyPassPass(Registry); initializeControlHeightReductionLegacyPassPass(Registry); initializeGCOVProfilerLegacyPassPass(Registry); initializePGOInstrumentationGenLegacyPassPass(Registry); initializePGOInstrumentationUseLegacyPassPass(Registry); initializePGOIndirectCallPromotionLegacyPassPass(Registry); initializePGOMemOPSizeOptLegacyPassPass(Registry); initializeInstrProfilingLegacyPassPass(Registry); initializeMemorySanitizerLegacyPassPass(Registry); initializeHWAddressSanitizerPass(Registry); - initializeThreadSanitizerPass(Registry); + initializeThreadSanitizerLegacyPassPass(Registry); initializeSanitizerCoverageModulePass(Registry); initializeDataFlowSanitizerPass(Registry); initializeEfficiencySanitizerPass(Registry); } /// LLVMInitializeInstrumentation - C binding for /// initializeInstrumentation. void LLVMInitializeInstrumentation(LLVMPassRegistryRef R) { initializeInstrumentation(*unwrap(R)); } Index: llvm/trunk/lib/Passes/PassRegistry.def =================================================================== --- llvm/trunk/lib/Passes/PassRegistry.def (revision 351313) +++ llvm/trunk/lib/Passes/PassRegistry.def (revision 351314) @@ -1,275 +1,276 @@ //===- PassRegistry.def - Registry of passes --------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is used as the registry of passes that are part of the core LLVM // libraries. This file describes both transformation passes and analyses // Analyses are registered while transformation passes have names registered // that can be used when providing a textual pass pipeline. // //===----------------------------------------------------------------------===// // NOTE: NO INCLUDE GUARD DESIRED! #ifndef MODULE_ANALYSIS #define MODULE_ANALYSIS(NAME, CREATE_PASS) #endif MODULE_ANALYSIS("callgraph", CallGraphAnalysis()) MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis()) MODULE_ANALYSIS("module-summary", ModuleSummaryIndexAnalysis()) MODULE_ANALYSIS("no-op-module", NoOpModuleAnalysis()) MODULE_ANALYSIS("profile-summary", ProfileSummaryAnalysis()) MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis()) MODULE_ANALYSIS("targetlibinfo", TargetLibraryAnalysis()) MODULE_ANALYSIS("verify", VerifierAnalysis()) MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #ifndef MODULE_ALIAS_ANALYSIS #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ MODULE_ANALYSIS(NAME, CREATE_PASS) #endif MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA()) #undef MODULE_ALIAS_ANALYSIS #undef MODULE_ANALYSIS #ifndef MODULE_PASS #define MODULE_PASS(NAME, CREATE_PASS) #endif MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("called-value-propagation", CalledValuePropagationPass()) MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass()) MODULE_PASS("cg-profile", CGProfilePass()) MODULE_PASS("constmerge", ConstantMergePass()) MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) MODULE_PASS("deadargelim", DeadArgumentEliminationPass()) MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass()) MODULE_PASS("forceattrs", ForceFunctionAttrsPass()) MODULE_PASS("function-import", FunctionImportPass()) MODULE_PASS("globaldce", GlobalDCEPass()) MODULE_PASS("globalopt", GlobalOptPass()) MODULE_PASS("globalsplit", GlobalSplitPass()) MODULE_PASS("hotcoldsplit", HotColdSplittingPass()) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrprof", InstrProfiling()) MODULE_PASS("internalize", InternalizePass()) MODULE_PASS("invalidate", InvalidateAllAnalysesPass()) MODULE_PASS("ipsccp", IPSCCPPass()) MODULE_PASS("lowertypetests", LowerTypeTestsPass(nullptr, nullptr)) MODULE_PASS("name-anon-globals", NameAnonGlobalPass()) MODULE_PASS("no-op-module", NoOpModulePass()) MODULE_PASS("partial-inliner", PartialInlinerPass()) MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion()) MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen()) MODULE_PASS("pgo-instr-use", PGOInstrumentationUse()) MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass()) MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs())) MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs())) MODULE_PASS("print", PrintModulePass(dbgs())) MODULE_PASS("print-lcg", LazyCallGraphPrinterPass(dbgs())) MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(dbgs())) MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs())) MODULE_PASS("rewrite-statepoints-for-gc", RewriteStatepointsForGC()) MODULE_PASS("rewrite-symbols", RewriteSymbolPass()) MODULE_PASS("rpo-functionattrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass()) MODULE_PASS("synthetic-counts-propagation", SyntheticCountsPropagation()) MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass(nullptr, nullptr)) MODULE_PASS("verify", VerifierPass()) #undef MODULE_PASS #ifndef CGSCC_ANALYSIS #define CGSCC_ANALYSIS(NAME, CREATE_PASS) #endif CGSCC_ANALYSIS("no-op-cgscc", NoOpCGSCCAnalysis()) CGSCC_ANALYSIS("fam-proxy", FunctionAnalysisManagerCGSCCProxy()) CGSCC_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #undef CGSCC_ANALYSIS #ifndef CGSCC_PASS #define CGSCC_PASS(NAME, CREATE_PASS) #endif CGSCC_PASS("argpromotion", ArgumentPromotionPass()) CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass()) CGSCC_PASS("inline", InlinerPass()) CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass()) #undef CGSCC_PASS #ifndef FUNCTION_ANALYSIS #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) #endif FUNCTION_ANALYSIS("aa", AAManager()) FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis()) FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis()) FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis()) FUNCTION_ANALYSIS("postdomtree", PostDominatorTreeAnalysis()) FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis()) FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis()) FUNCTION_ANALYSIS("loops", LoopAnalysis()) FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis()) FUNCTION_ANALYSIS("da", DependenceAnalysis()) FUNCTION_ANALYSIS("memdep", MemoryDependenceAnalysis()) FUNCTION_ANALYSIS("memoryssa", MemorySSAAnalysis()) FUNCTION_ANALYSIS("phi-values", PhiValuesAnalysis()) FUNCTION_ANALYSIS("regions", RegionInfoAnalysis()) FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis()) FUNCTION_ANALYSIS("opt-remark-emit", OptimizationRemarkEmitterAnalysis()) FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis()) FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis()) FUNCTION_ANALYSIS("targetlibinfo", TargetLibraryAnalysis()) FUNCTION_ANALYSIS("targetir", TM ? TM->getTargetIRAnalysis() : TargetIRAnalysis()) FUNCTION_ANALYSIS("verify", VerifierAnalysis()) FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #ifndef FUNCTION_ALIAS_ANALYSIS #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ FUNCTION_ANALYSIS(NAME, CREATE_PASS) #endif FUNCTION_ALIAS_ANALYSIS("basic-aa", BasicAA()) FUNCTION_ALIAS_ANALYSIS("cfl-anders-aa", CFLAndersAA()) FUNCTION_ALIAS_ANALYSIS("cfl-steens-aa", CFLSteensAA()) FUNCTION_ALIAS_ANALYSIS("scev-aa", SCEVAA()) FUNCTION_ALIAS_ANALYSIS("scoped-noalias-aa", ScopedNoAliasAA()) FUNCTION_ALIAS_ANALYSIS("type-based-aa", TypeBasedAA()) #undef FUNCTION_ALIAS_ANALYSIS #undef FUNCTION_ANALYSIS #ifndef FUNCTION_PASS #define FUNCTION_PASS(NAME, CREATE_PASS) #endif FUNCTION_PASS("aa-eval", AAEvaluator()) FUNCTION_PASS("adce", ADCEPass()) FUNCTION_PASS("add-discriminators", AddDiscriminatorsPass()) FUNCTION_PASS("aggressive-instcombine", AggressiveInstCombinePass()) FUNCTION_PASS("alignment-from-assumptions", AlignmentFromAssumptionsPass()) FUNCTION_PASS("bdce", BDCEPass()) FUNCTION_PASS("bounds-checking", BoundsCheckingPass()) FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass()) FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass()) FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass()) FUNCTION_PASS("dce", DCEPass()) FUNCTION_PASS("div-rem-pairs", DivRemPairsPass()) FUNCTION_PASS("dse", DSEPass()) FUNCTION_PASS("dot-cfg", CFGPrinterPass()) FUNCTION_PASS("dot-cfg-only", CFGOnlyPrinterPass()) FUNCTION_PASS("early-cse", EarlyCSEPass(/*UseMemorySSA=*/false)) FUNCTION_PASS("early-cse-memssa", EarlyCSEPass(/*UseMemorySSA=*/true)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass(/*PostInlining=*/false)) FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass()) FUNCTION_PASS("post-inline-ee-instrument", EntryExitInstrumenterPass(/*PostInlining=*/true)) FUNCTION_PASS("gvn-hoist", GVNHoistPass()) FUNCTION_PASS("instcombine", InstCombinePass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("float2int", Float2IntPass()) FUNCTION_PASS("no-op-function", NoOpFunctionPass()) FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass()) FUNCTION_PASS("loweratomic", LowerAtomicPass()) FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass()) FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass()) FUNCTION_PASS("guard-widening", GuardWideningPass()) FUNCTION_PASS("gvn", GVN()) FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass()) FUNCTION_PASS("loop-simplify", LoopSimplifyPass()) FUNCTION_PASS("loop-sink", LoopSinkPass()) FUNCTION_PASS("lowerinvoke", LowerInvokePass()) FUNCTION_PASS("mem2reg", PromotePass()) FUNCTION_PASS("memcpyopt", MemCpyOptPass()) FUNCTION_PASS("mldst-motion", MergedLoadStoreMotionPass()) FUNCTION_PASS("nary-reassociate", NaryReassociatePass()) FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass()) FUNCTION_PASS("lcssa", LCSSAPass()) FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass()) FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass()) FUNCTION_PASS("loop-distribute", LoopDistributePass()) FUNCTION_PASS("loop-vectorize", LoopVectorizePass()) FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt()) FUNCTION_PASS("print", PrintFunctionPass(dbgs())) FUNCTION_PASS("print", AssumptionPrinterPass(dbgs())) FUNCTION_PASS("print", BlockFrequencyPrinterPass(dbgs())) FUNCTION_PASS("print", BranchProbabilityPrinterPass(dbgs())) FUNCTION_PASS("print", DependenceAnalysisPrinterPass(dbgs())) FUNCTION_PASS("print", DominatorTreePrinterPass(dbgs())) FUNCTION_PASS("print", PostDominatorTreePrinterPass(dbgs())) FUNCTION_PASS("print", DemandedBitsPrinterPass(dbgs())) FUNCTION_PASS("print", DominanceFrontierPrinterPass(dbgs())) FUNCTION_PASS("print", LoopPrinterPass(dbgs())) FUNCTION_PASS("print", MemorySSAPrinterPass(dbgs())) FUNCTION_PASS("print", PhiValuesPrinterPass(dbgs())) FUNCTION_PASS("print", RegionInfoPrinterPass(dbgs())) FUNCTION_PASS("print", ScalarEvolutionPrinterPass(dbgs())) FUNCTION_PASS("print", StackSafetyPrinterPass(dbgs())) FUNCTION_PASS("reassociate", ReassociatePass()) FUNCTION_PASS("scalarizer", ScalarizerPass()) FUNCTION_PASS("sccp", SCCPPass()) FUNCTION_PASS("simplify-cfg", SimplifyCFGPass()) FUNCTION_PASS("sink", SinkingPass()) FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass()) FUNCTION_PASS("speculative-execution", SpeculativeExecutionPass()) FUNCTION_PASS("spec-phis", SpeculateAroundPHIsPass()) FUNCTION_PASS("sroa", SROA()) FUNCTION_PASS("tailcallelim", TailCallElimPass()) FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass()) FUNCTION_PASS("verify", VerifierPass()) FUNCTION_PASS("verify", DominatorTreeVerifierPass()) FUNCTION_PASS("verify", LoopVerifierPass()) FUNCTION_PASS("verify", MemorySSAVerifierPass()) FUNCTION_PASS("verify", RegionInfoVerifierPass()) FUNCTION_PASS("view-cfg", CFGViewerPass()) FUNCTION_PASS("view-cfg-only", CFGOnlyViewerPass()) FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass()) FUNCTION_PASS("msan", MemorySanitizerPass()) +FUNCTION_PASS("tsan", ThreadSanitizerPass()) #undef FUNCTION_PASS #ifndef FUNCTION_PASS_WITH_PARAMS #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) #endif FUNCTION_PASS_WITH_PARAMS("unroll", [](LoopUnrollOptions Opts) { return LoopUnrollPass(Opts); }, parseLoopUnrollOptions) #undef FUNCTION_PASS_WITH_PARAMS #ifndef LOOP_ANALYSIS #define LOOP_ANALYSIS(NAME, CREATE_PASS) #endif LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis()) LOOP_ANALYSIS("access-info", LoopAccessAnalysis()) LOOP_ANALYSIS("ivusers", IVUsersAnalysis()) LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) #undef LOOP_ANALYSIS #ifndef LOOP_PASS #define LOOP_PASS(NAME, CREATE_PASS) #endif LOOP_PASS("invalidate", InvalidateAllAnalysesPass()) LOOP_PASS("licm", LICMPass()) LOOP_PASS("loop-idiom", LoopIdiomRecognizePass()) LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass()) LOOP_PASS("rotate", LoopRotatePass()) LOOP_PASS("no-op-loop", NoOpLoopPass()) LOOP_PASS("print", PrintLoopPass(dbgs())) LOOP_PASS("loop-deletion", LoopDeletionPass()) LOOP_PASS("simplify-cfg", LoopSimplifyCFGPass()) LOOP_PASS("strength-reduce", LoopStrengthReducePass()) LOOP_PASS("indvars", IndVarSimplifyPass()) LOOP_PASS("irce", IRCEPass()) LOOP_PASS("unroll-and-jam", LoopUnrollAndJamPass()) LOOP_PASS("unroll-full", LoopFullUnrollPass()) LOOP_PASS("unswitch", SimpleLoopUnswitchPass()) LOOP_PASS("print-access-info", LoopAccessInfoPrinterPass(dbgs())) LOOP_PASS("print", IVUsersPrinterPass(dbgs())) LOOP_PASS("loop-predication", LoopPredicationPass()) #undef LOOP_PASS Index: llvm/trunk/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/trunk/lib/Passes/PassBuilder.cpp (revision 351313) +++ llvm/trunk/lib/Passes/PassBuilder.cpp (revision 351314) @@ -1,2072 +1,2073 @@ //===- Parsing, selection, and construction of pass pipelines -------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// \file /// /// This file provides the implementation of the PassBuilder based on our /// static pass registry as well as related functionality. It also provides /// helpers to aid in analyzing, debugging, and testing passes and pass /// pipelines. /// //===----------------------------------------------------------------------===// #include "llvm/Passes/PassBuilder.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasAnalysisEvaluator.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFGPrinter.h" #include "llvm/Analysis/CFLAndersAliasAnalysis.h" #include "llvm/Analysis/CFLSteensAliasAnalysis.h" #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IVUsers.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/LazyValueInfo.h" #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/PhiValues.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/ScopedNoAliasAA.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/UnreachableBlockElim.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Regex.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/Transforms/IPO/CalledValuePropagation.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" #include "llvm/Transforms/IPO/ElimAvailExtern.h" #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/IPO/GlobalDCE.h" #include "llvm/Transforms/IPO/GlobalOpt.h" #include "llvm/Transforms/IPO/GlobalSplit.h" #include "llvm/Transforms/IPO/HotColdSplitting.h" #include "llvm/Transforms/IPO/InferFunctionAttrs.h" #include "llvm/Transforms/IPO/Inliner.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/PartialInlining.h" #include "llvm/Transforms/IPO/SCCP.h" #include "llvm/Transforms/IPO/SampleProfile.h" #include "llvm/Transforms/IPO/StripDeadPrototypes.h" #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/BoundsChecking.h" #include "llvm/Transforms/Instrumentation/CGProfile.h" #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h" #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Scalar/ADCE.h" #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h" #include "llvm/Transforms/Scalar/BDCE.h" #include "llvm/Transforms/Scalar/CallSiteSplitting.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h" #include "llvm/Transforms/Scalar/DCE.h" #include "llvm/Transforms/Scalar/DeadStoreElimination.h" #include "llvm/Transforms/Scalar/DivRemPairs.h" #include "llvm/Transforms/Scalar/EarlyCSE.h" #include "llvm/Transforms/Scalar/Float2Int.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Scalar/GuardWidening.h" #include "llvm/Transforms/Scalar/IVUsersPrinter.h" #include "llvm/Transforms/Scalar/IndVarSimplify.h" #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h" #include "llvm/Transforms/Scalar/InstSimplifyPass.h" #include "llvm/Transforms/Scalar/JumpThreading.h" #include "llvm/Transforms/Scalar/LICM.h" #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h" #include "llvm/Transforms/Scalar/LoopDataPrefetch.h" #include "llvm/Transforms/Scalar/LoopDeletion.h" #include "llvm/Transforms/Scalar/LoopDistribute.h" #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h" #include "llvm/Transforms/Scalar/LoopInstSimplify.h" #include "llvm/Transforms/Scalar/LoopLoadElimination.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopPredication.h" #include "llvm/Transforms/Scalar/LoopRotation.h" #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h" #include "llvm/Transforms/Scalar/LoopSink.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" #include "llvm/Transforms/Scalar/LoopUnrollPass.h" #include "llvm/Transforms/Scalar/LowerAtomic.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h" #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h" #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/Transforms/Scalar/NaryReassociate.h" #include "llvm/Transforms/Scalar/NewGVN.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/Reassociate.h" #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h" #include "llvm/Transforms/Scalar/SCCP.h" #include "llvm/Transforms/Scalar/SROA.h" #include "llvm/Transforms/Scalar/Scalarizer.h" #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" #include "llvm/Transforms/Scalar/SimplifyCFG.h" #include "llvm/Transforms/Scalar/Sink.h" #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h" #include "llvm/Transforms/Scalar/SpeculativeExecution.h" #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include "llvm/Transforms/Utils/Mem2Reg.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/SymbolRewriter.h" #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h" #include "llvm/Transforms/Vectorize/LoopVectorize.h" #include "llvm/Transforms/Vectorize/SLPVectorizer.h" using namespace llvm; static cl::opt MaxDevirtIterations("pm-max-devirt-iterations", cl::ReallyHidden, cl::init(4)); static cl::opt RunPartialInlining("enable-npm-partial-inlining", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run Partial inlinining pass")); static cl::opt RunNewGVN("enable-npm-newgvn", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run NewGVN instead of GVN")); static cl::opt EnableEarlyCSEMemSSA( "enable-npm-earlycse-memssa", cl::init(true), cl::Hidden, cl::desc("Enable the EarlyCSE w/ MemorySSA pass for the new PM (default = on)")); static cl::opt EnableGVNHoist( "enable-npm-gvn-hoist", cl::init(false), cl::Hidden, cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); static cl::opt EnableGVNSink( "enable-npm-gvn-sink", cl::init(false), cl::Hidden, cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); static cl::opt EnableUnrollAndJam( "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden, cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)")); static cl::opt EnableSyntheticCounts( "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run synthetic function entry count generation " "pass")); static Regex DefaultAliasRegex( "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$"); static cl::opt EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden, cl::desc("Enable control height reduction optimization (CHR)")); extern cl::opt EnableHotColdSplit; static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) { switch (Level) { case PassBuilder::O0: case PassBuilder::O1: case PassBuilder::O2: case PassBuilder::O3: return false; case PassBuilder::Os: case PassBuilder::Oz: return true; } llvm_unreachable("Invalid optimization level!"); } namespace { /// No-op module pass which does nothing. struct NoOpModulePass { PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpModulePass"; } }; /// No-op module analysis. class NoOpModuleAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Module &, ModuleAnalysisManager &) { return Result(); } static StringRef name() { return "NoOpModuleAnalysis"; } }; /// No-op CGSCC pass which does nothing. struct NoOpCGSCCPass { PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &, LazyCallGraph &, CGSCCUpdateResult &UR) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpCGSCCPass"; } }; /// No-op CGSCC analysis. class NoOpCGSCCAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) { return Result(); } static StringRef name() { return "NoOpCGSCCAnalysis"; } }; /// No-op function pass which does nothing. struct NoOpFunctionPass { PreservedAnalyses run(Function &F, FunctionAnalysisManager &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpFunctionPass"; } }; /// No-op function analysis. class NoOpFunctionAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Function &, FunctionAnalysisManager &) { return Result(); } static StringRef name() { return "NoOpFunctionAnalysis"; } }; /// No-op loop pass which does nothing. struct NoOpLoopPass { PreservedAnalyses run(Loop &L, LoopAnalysisManager &, LoopStandardAnalysisResults &, LPMUpdater &) { return PreservedAnalyses::all(); } static StringRef name() { return "NoOpLoopPass"; } }; /// No-op loop analysis. class NoOpLoopAnalysis : public AnalysisInfoMixin { friend AnalysisInfoMixin; static AnalysisKey Key; public: struct Result {}; Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) { return Result(); } static StringRef name() { return "NoOpLoopAnalysis"; } }; AnalysisKey NoOpModuleAnalysis::Key; AnalysisKey NoOpCGSCCAnalysis::Key; AnalysisKey NoOpFunctionAnalysis::Key; AnalysisKey NoOpLoopAnalysis::Key; } // End anonymous namespace. void PassBuilder::invokePeepholeEPCallbacks( FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { for (auto &C : PeepholeEPCallbacks) C(FPM, Level); } void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ MAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : ModuleAnalysisRegistrationCallbacks) C(MAM); } void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ CGAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : CGSCCAnalysisRegistrationCallbacks) C(CGAM); } void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ FAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : FunctionAnalysisRegistrationCallbacks) C(FAM); } void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) { #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ LAM.registerPass([&] { return CREATE_PASS; }); #include "PassRegistry.def" for (auto &C : LoopAnalysisRegistrationCallbacks) C(LAM); } FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { assert(Level != O0 && "Must request optimizations!"); FunctionPassManager FPM(DebugLogging); // Form SSA out of local memory accesses after breaking apart aggregates into // scalars. FPM.addPass(SROA()); // Catch trivial redundancies FPM.addPass(EarlyCSEPass(EnableEarlyCSEMemSSA)); // Hoisting of scalars and load expressions. if (EnableGVNHoist) FPM.addPass(GVNHoistPass()); // Global value numbering based sinking. if (EnableGVNSink) { FPM.addPass(GVNSinkPass()); FPM.addPass(SimplifyCFGPass()); } // Speculative execution if the target has divergent branches; otherwise nop. FPM.addPass(SpeculativeExecutionPass()); // Optimize based on known information about branches, and cleanup afterward. FPM.addPass(JumpThreadingPass()); FPM.addPass(CorrelatedValuePropagationPass()); FPM.addPass(SimplifyCFGPass()); if (Level == O3) FPM.addPass(AggressiveInstCombinePass()); FPM.addPass(InstCombinePass()); if (!isOptimizingForSize(Level)) FPM.addPass(LibCallsShrinkWrapPass()); invokePeepholeEPCallbacks(FPM, Level); // For PGO use pipeline, try to optimize memory intrinsics such as memcpy // using the size value profile. Don't perform this when optimizing for size. if (PGOOpt && !PGOOpt->ProfileUseFile.empty() && !isOptimizingForSize(Level)) FPM.addPass(PGOMemOPSizeOpt()); FPM.addPass(TailCallElimPass()); FPM.addPass(SimplifyCFGPass()); // Form canonically associated expression trees, and simplify the trees using // basic mathematical properties. For example, this will form (nearly) // minimal multiplication trees. FPM.addPass(ReassociatePass()); // Add the primary loop simplification pipeline. // FIXME: Currently this is split into two loop pass pipelines because we run // some function passes in between them. These can and should be removed // and/or replaced by scheduling the loop pass equivalents in the correct // positions. But those equivalent passes aren't powerful enough yet. // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to // fully replace `SimplifyCFGPass`, and the closest to the other we have is // `LoopInstSimplify`. LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); // Simplify the loop body. We do this initially to clean up after other loop // passes run, either when iterating on a loop or on inner loops with // implications on the outer loop. LPM1.addPass(LoopInstSimplifyPass()); LPM1.addPass(LoopSimplifyCFGPass()); // Rotate Loop - disable header duplication at -Oz LPM1.addPass(LoopRotatePass(Level != Oz)); LPM1.addPass(LICMPass()); LPM1.addPass(SimpleLoopUnswitchPass()); LPM2.addPass(IndVarSimplifyPass()); LPM2.addPass(LoopIdiomRecognizePass()); for (auto &C : LateLoopOptimizationsEPCallbacks) C(LPM2, Level); LPM2.addPass(LoopDeletionPass()); // Do not enable unrolling in PreLinkThinLTO phase during sample PGO // because it changes IR to makes profile annotation in back compile // inaccurate. if (Phase != ThinLTOPhase::PreLink || !PGOOpt || PGOOpt->SampleProfileFile.empty()) LPM2.addPass(LoopFullUnrollPass(Level)); for (auto &C : LoopOptimizerEndEPCallbacks) C(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. FPM.addPass(RequireAnalysisPass()); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging)); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging)); // Eliminate redundancies. if (Level != O1) { // These passes add substantial compile time so skip them at O1. FPM.addPass(MergedLoadStoreMotionPass()); if (RunNewGVN) FPM.addPass(NewGVNPass()); else FPM.addPass(GVN()); } // Specially optimize memory movement as it doesn't look like dataflow in SSA. FPM.addPass(MemCpyOptPass()); // Sparse conditional constant propagation. // FIXME: It isn't clear why we do this *after* loop passes rather than // before... FPM.addPass(SCCPPass()); // Delete dead bit computations (instcombine runs after to fold away the dead // computations, and then ADCE will run later to exploit any new DCE // opportunities that creates). FPM.addPass(BDCEPass()); // Run instcombine after redundancy and dead bit elimination to exploit // opportunities opened up by them. FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); // Re-consider control flow based optimizations after redundancy elimination, // redo DCE, etc. FPM.addPass(JumpThreadingPass()); FPM.addPass(CorrelatedValuePropagationPass()); FPM.addPass(DSEPass()); FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); for (auto &C : ScalarOptimizerLateEPCallbacks) C(FPM, Level); // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. FPM.addPass(ADCEPass()); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); if (EnableCHR && Level == O3 && PGOOpt && (!PGOOpt->ProfileUseFile.empty() || !PGOOpt->SampleProfileFile.empty())) FPM.addPass(ControlHeightReductionPass()); return FPM; } void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, PassBuilder::OptimizationLevel Level, bool RunProfileGen, std::string ProfileGenFile, std::string ProfileUseFile, std::string ProfileRemappingFile) { // Generally running simplification passes and the inliner with an high // threshold results in smaller executables, but there may be cases where // the size grows, so let's be conservative here and skip this simplification // at -Os/Oz. if (!isOptimizingForSize(Level)) { InlineParams IP; // In the old pass manager, this is a cl::opt. Should still this be one? IP.DefaultThreshold = 75; // FIXME: The hint threshold has the same value used by the regular inliner. // This should probably be lowered after performance testing. // FIXME: this comment is cargo culted from the old pass manager, revisit). IP.HintThreshold = 325; CGSCCPassManager CGPipeline(DebugLogging); CGPipeline.addPass(InlinerPass(IP)); FunctionPassManager FPM; FPM.addPass(SROA()); FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. FPM.addPass(InstCombinePass()); // Combine silly sequences. invokePeepholeEPCallbacks(FPM, Level); CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline))); } // Delete anything that is now dead to make sure that we don't instrument // dead code. Instrumentation can end up keeping dead code around and // dramatically increase code size. MPM.addPass(GlobalDCEPass()); if (RunProfileGen) { MPM.addPass(PGOInstrumentationGen()); FunctionPassManager FPM; FPM.addPass( createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging)); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); // Add the profile lowering pass. InstrProfOptions Options; if (!ProfileGenFile.empty()) Options.InstrProfileOutput = ProfileGenFile; Options.DoCounterPromotion = true; MPM.addPass(InstrProfiling(Options)); } if (!ProfileUseFile.empty()) MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile)); } static InlineParams getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { auto O3 = PassBuilder::O3; unsigned OptLevel = Level > O3 ? 2 : Level; unsigned SizeLevel = Level > O3 ? Level - O3 : 0; return getInlineParams(OptLevel, SizeLevel); } ModulePassManager PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) { ModulePassManager MPM(DebugLogging); // Do basic inference of function attributes from known properties of system // libraries and other oracles. MPM.addPass(InferFunctionAttrsPass()); // Create an early function pass manager to cleanup the output of the // frontend. FunctionPassManager EarlyFPM(DebugLogging); EarlyFPM.addPass(SimplifyCFGPass()); EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); EarlyFPM.addPass(LowerExpectIntrinsicPass()); if (Level == O3) EarlyFPM.addPass(CallSiteSplittingPass()); // In SamplePGO ThinLTO backend, we need instcombine before profile annotation // to convert bitcast to direct calls so that they can be inlined during the // profile annotation prepration step. // More details about SamplePGO design can be found in: // https://research.google.com/pubs/pub45290.html // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. if (PGOOpt && !PGOOpt->SampleProfileFile.empty() && Phase == ThinLTOPhase::PostLink) EarlyFPM.addPass(InstCombinePass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) { // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile, PGOOpt->ProfileRemappingFile, Phase == ThinLTOPhase::PreLink)); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. if (Phase != ThinLTOPhase::PreLink) // We perform early indirect call promotion here, before globalopt. // This is important for the ThinLTO backend phase because otherwise // imported available_externally functions look unreferenced and are // removed. MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, true)); } // Interprocedural constant propagation now that basic cleanup has occurred // and prior to optimizing globals. // FIXME: This position in the pipeline hasn't been carefully considered in // years, it should be re-analyzed. MPM.addPass(IPSCCPPass()); // Attach metadata to indirect call sites indicating the set of functions // they may target at run-time. This should follow IPSCCP. MPM.addPass(CalledValuePropagationPass()); // Optimize globals to try and fold them into constants. MPM.addPass(GlobalOptPass()); // Promote any localized globals to SSA registers. // FIXME: Should this instead by a run of SROA? // FIXME: We should probably run instcombine and simplify-cfg afterward to // delete control flows that are dead once globals have been folded to // constants. MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); // Remove any dead arguments exposed by cleanups and constand folding // globals. MPM.addPass(DeadArgumentEliminationPass()); // Create a small function pass pipeline to cleanup after all the global // optimizations. FunctionPassManager GlobalCleanupPM(DebugLogging); GlobalCleanupPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(GlobalCleanupPM, Level); GlobalCleanupPM.addPass(SimplifyCFGPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); // Add all the requested passes for instrumentation PGO, if requested. if (PGOOpt && Phase != ThinLTOPhase::PostLink && (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) { addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen, PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile, PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } // Synthesize function entry counts for non-PGO compilation. if (EnableSyntheticCounts && !PGOOpt) MPM.addPass(SyntheticCountsPropagation()); // Require the GlobalsAA analysis for the module so we can query it within // the CGSCC pipeline. MPM.addPass(RequireAnalysisPass()); // Require the ProfileSummaryAnalysis for the module so we can query it within // the inliner pass. MPM.addPass(RequireAnalysisPass()); // Now begin the main postorder CGSCC pipeline. // FIXME: The current CGSCC pipeline has its origins in the legacy pass // manager and trying to emulate its precise behavior. Much of this doesn't // make a lot of sense and we should revisit the core CGSCC structure. CGSCCPassManager MainCGPipeline(DebugLogging); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an // invoke or a call. // Run the inliner first. The theory is that we are walking bottom-up and so // the callees have already been fully optimized, and we want to inline them // into the callers so that our optimizations can reflect that. // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO // because it makes profile annotation in the backend inaccurate. InlineParams IP = getInlineParamsFromOptLevel(Level); if (Phase == ThinLTOPhase::PreLink && PGOOpt && !PGOOpt->SampleProfileFile.empty()) IP.HotCallSiteThreshold = 0; MainCGPipeline.addPass(InlinerPass(IP)); // Now deduce any function attributes based in the current code. MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); // When at O3 add argument promotion to the pass pipeline. // FIXME: It isn't at all clear why this should be limited to O3. if (Level == O3) MainCGPipeline.addPass(ArgumentPromotionPass()); // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( buildFunctionSimplificationPipeline(Level, Phase, DebugLogging))); // We only want to do hot cold splitting once for ThinLTO, during the // post-link ThinLTO. if (EnableHotColdSplit && Phase != ThinLTOPhase::PreLink) MPM.addPass(HotColdSplittingPass()); for (auto &C : CGSCCOptimizerLateEPCallbacks) C(MainCGPipeline, Level); // We wrap the CGSCC pipeline in a devirtualization repeater. This will try // to detect when we devirtualize indirect calls and iterate the SCC passes // in that case to try and catch knock-on inlining or function attrs // opportunities. Then we add it to the module pipeline by walking the SCCs // in postorder (or bottom-up). MPM.addPass( createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass( std::move(MainCGPipeline), MaxDevirtIterations))); return MPM; } ModulePassManager PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, bool DebugLogging) { ModulePassManager MPM(DebugLogging); // Optimize globals now that the module is fully simplified. MPM.addPass(GlobalOptPass()); MPM.addPass(GlobalDCEPass()); // Run partial inlining pass to partially inline functions that have // large bodies. if (RunPartialInlining) MPM.addPass(PartialInlinerPass()); // Remove avail extern fns and globals definitions since we aren't compiling // an object file for later LTO. For LTO we want to preserve these so they // are eligible for inlining at link-time. Note if they are unreferenced they // will be removed by GlobalDCE later, so this only impacts referenced // available externally globals. Eventually they will be suppressed during // codegen, but eliminating here enables more opportunity for GlobalDCE as it // may make globals referenced by available external functions dead and saves // running remaining passes on the eliminated functions. MPM.addPass(EliminateAvailableExternallyPass()); // Do RPO function attribute inference across the module to forward-propagate // attributes where applicable. // FIXME: Is this really an optimization rather than a canonicalization? MPM.addPass(ReversePostOrderFunctionAttrsPass()); // Re-require GloblasAA here prior to function passes. This is particularly // useful as the above will have inlined, DCE'ed, and function-attr // propagated everything. We should at this point have a reasonably minimal // and richly annotated call graph. By computing aliasing and mod/ref // information for all local globals here, the late loop passes and notably // the vectorizer will be able to use them to help recognize vectorizable // memory operations. MPM.addPass(RequireAnalysisPass()); FunctionPassManager OptimizePM(DebugLogging); OptimizePM.addPass(Float2IntPass()); // FIXME: We need to run some loop optimizations to re-rotate loops after // simplify-cfg and others undo their rotation. // Optimize the loop execution. These passes operate on entire loop nests // rather than on each loop in an inside-out manner, and so they are actually // function passes. for (auto &C : VectorizerStartEPCallbacks) C(OptimizePM, Level); // First rotate loops that may have been un-rotated by prior passes. OptimizePM.addPass( createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging)); // Distribute loops to allow partial vectorization. I.e. isolate dependences // into separate loop that would otherwise inhibit vectorization. This is // currently only performed for loops marked with the metadata // llvm.loop.distribute=true or when -enable-loop-distribute is specified. OptimizePM.addPass(LoopDistributePass()); // Now run the core loop vectorizer. OptimizePM.addPass(LoopVectorizePass()); // Eliminate loads by forwarding stores from the previous iteration to loads // of the current iteration. OptimizePM.addPass(LoopLoadEliminationPass()); // Cleanup after the loop optimization passes. OptimizePM.addPass(InstCombinePass()); // Now that we've formed fast to execute loop structures, we do further // optimizations. These are run afterward as they might block doing complex // analyses and transforms such as what are needed for loop vectorization. // Cleanup after loop vectorization, etc. Simplification passes like CVP and // GVN, loop transforms, and others have already run, so it's now better to // convert to more optimized IR using more aggressive simplify CFG options. // The extra sinking transform can create larger basic blocks, so do this // before SLP vectorization. OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions(). forwardSwitchCondToPhi(true). convertSwitchToLookupTable(true). needCanonicalLoops(false). sinkCommonInsts(true))); // Optimize parallel scalar instruction chains into SIMD instructions. OptimizePM.addPass(SLPVectorizerPass()); OptimizePM.addPass(InstCombinePass()); // Unroll small loops to hide loop backedge latency and saturate any parallel // execution resources of an out-of-order processor. We also then need to // clean up redundancies and loop invariant code. // FIXME: It would be really good to use a loop-integrated instruction // combiner for cleanup here so that the unrolling and LICM can be pipelined // across the loop nests. // We do UnrollAndJam in a separate LPM to ensure it happens before unroll if (EnableUnrollAndJam) { OptimizePM.addPass( createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level))); } OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(Level))); OptimizePM.addPass(WarnMissedTransformationsPass()); OptimizePM.addPass(InstCombinePass()); OptimizePM.addPass(RequireAnalysisPass()); OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); // Now that we've vectorized and unrolled loops, we may have more refined // alignment information, try to re-derive it here. OptimizePM.addPass(AlignmentFromAssumptionsPass()); // LoopSink pass sinks instructions hoisted by LICM, which serves as a // canonicalization pass that enables other optimizations. As a result, // LoopSink pass needs to be a very late IR pass to avoid undoing LICM // result too early. OptimizePM.addPass(LoopSinkPass()); // And finally clean up LCSSA form before generating code. OptimizePM.addPass(InstSimplifyPass()); // This hoists/decomposes div/rem ops. It should run after other sink/hoist // passes to avoid re-sinking, but before SimplifyCFG because it can allow // flattening of blocks. OptimizePM.addPass(DivRemPairsPass()); // LoopSink (and other loop passes since the last simplifyCFG) might have // resulted in single-entry-single-exit or empty blocks. Clean up the CFG. OptimizePM.addPass(SimplifyCFGPass()); // Optimize PHIs by speculating around them when profitable. Note that this // pass needs to be run after any PRE or similar pass as it is essentially // inserting redudnancies into the progrem. This even includes SimplifyCFG. OptimizePM.addPass(SpeculateAroundPHIsPass()); for (auto &C : OptimizerLastEPCallbacks) C(OptimizePM, Level); // Add the core optimizing pipeline. MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); MPM.addPass(CGProfilePass()); // Now we need to do some global optimization transforms. // FIXME: It would seem like these should come first in the optimization // pipeline and maybe be the bottom of the canonicalization pipeline? Weird // ordering here. MPM.addPass(GlobalDCEPass()); MPM.addPass(ConstantMergePass()); return MPM; } ModulePassManager PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, bool DebugLogging) { assert(Level != O0 && "Must request optimizations for the default pipeline!"); ModulePassManager MPM(DebugLogging); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // Apply module pipeline start EP callback. for (auto &C : PipelineStartEPCallbacks) C(MPM); if (PGOOpt && PGOOpt->SamplePGOSupport) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Add the core simplification pipeline. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None, DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); return MPM; } ModulePassManager PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level, bool DebugLogging) { assert(Level != O0 && "Must request optimizations for the default pipeline!"); ModulePassManager MPM(DebugLogging); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); if (PGOOpt && PGOOpt->SamplePGOSupport) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. for (auto &C : PipelineStartEPCallbacks) C(MPM); // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink, DebugLogging)); // Run partial inlining pass to partially inline functions that have // large bodies. // FIXME: It isn't clear whether this is really the right place to run this // in ThinLTO. Because there is another canonicalization and simplification // phase that will run after the thin link, running this here ends up with // less information than will be available later and it may grow functions in // ways that aren't beneficial. if (RunPartialInlining) MPM.addPass(PartialInlinerPass()); // Reduce the size of the IR as much as possible. MPM.addPass(GlobalOptPass()); return MPM; } ModulePassManager PassBuilder::buildThinLTODefaultPipeline( OptimizationLevel Level, bool DebugLogging, const ModuleSummaryIndex *ImportSummary) { ModulePassManager MPM(DebugLogging); if (ImportSummary) { // These passes import type identifier resolutions for whole-program // devirtualization and CFI. They must run early because other passes may // disturb the specific instruction patterns that these passes look for, // creating dependencies on resolutions that may not appear in the summary. // // For example, GVN may transform the pattern assume(type.test) appearing in // two basic blocks into assume(phi(type.test, type.test)), which would // transform a dependency on a WPD resolution into a dependency on a type // identifier resolution for CFI. // // Also, WPD has access to more precise information than ICP and can // devirtualize more effectively, so it should operate on the IR first. MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary)); MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary)); } // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // During the ThinLTO backend phase we perform early indirect call promotion // here, before globalopt. Otherwise imported available_externally functions // look unreferenced and are removed. // FIXME: move this into buildModuleSimplificationPipeline to merge the logic // with SamplePGO. if (!PGOOpt || PGOOpt->SampleProfileFile.empty()) MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, false /* SamplePGO */)); // Add the core simplification pipeline. MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink, DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); return MPM; } ModulePassManager PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level, bool DebugLogging) { assert(Level != O0 && "Must request optimizations for the default pipeline!"); // FIXME: We should use a customized pre-link pipeline! return buildPerModuleDefaultPipeline(Level, DebugLogging); } ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging, ModuleSummaryIndex *ExportSummary) { assert(Level != O0 && "Must request optimizations for the default pipeline!"); ModulePassManager MPM(DebugLogging); if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) { // Load sample profile before running the LTO optimization pipeline. MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile, PGOOpt->ProfileRemappingFile, false /* ThinLTOPhase::PreLink */)); } // Remove unused virtual tables to improve the quality of code generated by // whole-program devirtualization and bitset lowering. MPM.addPass(GlobalDCEPass()); // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); // Do basic inference of function attributes from known properties of system // libraries and other oracles. MPM.addPass(InferFunctionAttrsPass()); if (Level > 1) { FunctionPassManager EarlyFPM(DebugLogging); EarlyFPM.addPass(CallSiteSplittingPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); // Indirect call promotion. This should promote all the targets that are // left by the earlier promotion pass that promotes intra-module targets. // This two-step promotion is to save the compile time. For LTO, it should // produce the same result as if we only do promotion here. MPM.addPass(PGOIndirectCallPromotion( true /* InLTO */, PGOOpt && !PGOOpt->SampleProfileFile.empty())); // Propagate constants at call sites into the functions they call. This // opens opportunities for globalopt (and inlining) by substituting function // pointers passed as arguments to direct uses of functions. MPM.addPass(IPSCCPPass()); // Attach metadata to indirect call sites indicating the set of functions // they may target at run-time. This should follow IPSCCP. MPM.addPass(CalledValuePropagationPass()); } // Now deduce any function attributes based in the current code. MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( PostOrderFunctionAttrsPass())); // Do RPO function attribute inference across the module to forward-propagate // attributes where applicable. // FIXME: Is this really an optimization rather than a canonicalization? MPM.addPass(ReversePostOrderFunctionAttrsPass()); // Use inragne annotations on GEP indices to split globals where beneficial. MPM.addPass(GlobalSplitPass()); // Run whole program optimization of virtual call when the list of callees // is fixed. MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); // Stop here at -O1. if (Level == 1) { // The LowerTypeTestsPass needs to run to lower type metadata and the // type.test intrinsics. The pass does nothing if CFI is disabled. MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); return MPM; } // Optimize globals to try and fold them into constants. MPM.addPass(GlobalOptPass()); // Promote any localized globals to SSA registers. MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); // Linking modules together can lead to duplicate global constant, only // keep one copy of each constant. MPM.addPass(ConstantMergePass()); // Remove unused arguments from functions. MPM.addPass(DeadArgumentEliminationPass()); // Reduce the code after globalopt and ipsccp. Both can open up significant // simplification opportunities, and both can propagate functions through // function pointers. When this happens, we often have to resolve varargs // calls, etc, so let instcombine do this. FunctionPassManager PeepholeFPM(DebugLogging); if (Level == O3) PeepholeFPM.addPass(AggressiveInstCombinePass()); PeepholeFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(PeepholeFPM, Level); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an // invoke or a call. // Run the inliner now. MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( InlinerPass(getInlineParamsFromOptLevel(Level)))); // Optimize globals again after we ran the inliner. MPM.addPass(GlobalOptPass()); // Garbage collect dead functions. // FIXME: Add ArgumentPromotion pass after once it's ported. MPM.addPass(GlobalDCEPass()); FunctionPassManager FPM(DebugLogging); // The IPO Passes may leave cruft around. Clean up after them. FPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(FPM, Level); FPM.addPass(JumpThreadingPass()); // Break up allocas FPM.addPass(SROA()); // Run a few AA driver optimizations here and now to cleanup the code. MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( PostOrderFunctionAttrsPass())); // FIXME: here we run IP alias analysis in the legacy PM. FunctionPassManager MainFPM; // FIXME: once we fix LoopPass Manager, add LICM here. // FIXME: once we provide support for enabling MLSM, add it here. // FIXME: once we provide support for enabling NewGVN, add it here. if (RunNewGVN) MainFPM.addPass(NewGVNPass()); else MainFPM.addPass(GVN()); // Remove dead memcpy()'s. MainFPM.addPass(MemCpyOptPass()); // Nuke dead stores. MainFPM.addPass(DSEPass()); // FIXME: at this point, we run a bunch of loop passes: // indVarSimplify, loopDeletion, loopInterchange, loopUnrool, // loopVectorize. Enable them once the remaining issue with LPM // are sorted out. MainFPM.addPass(InstCombinePass()); MainFPM.addPass(SimplifyCFGPass()); MainFPM.addPass(SCCPPass()); MainFPM.addPass(InstCombinePass()); MainFPM.addPass(BDCEPass()); // FIXME: We may want to run SLPVectorizer here. // After vectorization, assume intrinsics may tell us more // about pointer alignments. #if 0 MainFPM.add(AlignmentFromAssumptionsPass()); #endif // FIXME: Conditionally run LoadCombine here, after it's ported // (in case we still have this pass, given its questionable usefulness). MainFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); // Create a function that performs CFI checks for cross-DSO calls with // targets in the current module. MPM.addPass(CrossDSOCFIPass()); // Lower type metadata and the type.test intrinsic. This pass supports // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs // to be run at link time if CFI is enabled. This pass does nothing if // CFI is disabled. MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); // Add late LTO optimization passes. // Delete basic blocks, which optimization passes may have killed. MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass())); // Drop bodies of available eternally objects to improve GlobalDCE. MPM.addPass(EliminateAvailableExternallyPass()); // Now that we have optimized the program, discard unreachable functions. MPM.addPass(GlobalDCEPass()); // FIXME: Enable MergeFuncs, conditionally, after ported, maybe. return MPM; } AAManager PassBuilder::buildDefaultAAPipeline() { AAManager AA; // The order in which these are registered determines their priority when // being queried. // First we register the basic alias analysis that provides the majority of // per-function local AA logic. This is a stateless, on-demand local set of // AA techniques. AA.registerFunctionAnalysis(); // Next we query fast, specialized alias analyses that wrap IR-embedded // information about aliasing. AA.registerFunctionAnalysis(); AA.registerFunctionAnalysis(); // Add support for querying global aliasing information when available. // Because the `AAManager` is a function analysis and `GlobalsAA` is a module // analysis, all that the `AAManager` can do is query for any *cached* // results from `GlobalsAA` through a readonly proxy. AA.registerModuleAnalysis(); return AA; } static Optional parseRepeatPassName(StringRef Name) { if (!Name.consume_front("repeat<") || !Name.consume_back(">")) return None; int Count; if (Name.getAsInteger(0, Count) || Count <= 0) return None; return Count; } static Optional parseDevirtPassName(StringRef Name) { if (!Name.consume_front("devirt<") || !Name.consume_back(">")) return None; int Count; if (Name.getAsInteger(0, Count) || Count <= 0) return None; return Count; } static bool checkParametrizedPassName(StringRef Name, StringRef PassName) { if (!Name.consume_front(PassName)) return false; // normal pass name w/o parameters == default parameters if (Name.empty()) return true; return Name.startswith("<") && Name.endswith(">"); } namespace { /// This performs customized parsing of pass name with parameters. /// /// We do not need parametrization of passes in textual pipeline very often, /// yet on a rare occasion ability to specify parameters right there can be /// useful. /// /// \p Name - parameterized specification of a pass from a textual pipeline /// is a string in a form of : /// PassName '<' parameter-list '>' /// /// Parameter list is being parsed by the parser callable argument, \p Parser, /// It takes a string-ref of parameters and returns either StringError or a /// parameter list in a form of a custom parameters type, all wrapped into /// Expected<> template class. /// template auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name, StringRef PassName) -> decltype(Parser(StringRef{})) { using ParametersT = typename decltype(Parser(StringRef{}))::value_type; StringRef Params = Name; if (!Params.consume_front(PassName)) { assert(false && "unable to strip pass name from parametrized pass specification"); } if (Params.empty()) return ParametersT{}; if (!Params.consume_front("<") || !Params.consume_back(">")) { assert(false && "invalid format for parametrized pass name"); } Expected Result = Parser(Params); assert((Result || Result.template errorIsA()) && "Pass parameter parser can only return StringErrors."); return std::move(Result); } /// Parser of parameters for LoopUnroll pass. Expected parseLoopUnrollOptions(StringRef Params) { LoopUnrollOptions UnrollOpts; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); int OptLevel = StringSwitch(ParamName) .Case("O0", 0) .Case("O1", 1) .Case("O2", 2) .Case("O3", 3) .Default(-1); if (OptLevel >= 0) { UnrollOpts.setOptLevel(OptLevel); continue; } bool Enable = !ParamName.consume_front("no-"); if (ParamName == "partial") { UnrollOpts.setPartial(Enable); } else if (ParamName == "peeling") { UnrollOpts.setPeeling(Enable); } else if (ParamName == "runtime") { UnrollOpts.setRuntime(Enable); } else if (ParamName == "upperbound") { UnrollOpts.setUpperBound(Enable); } else { return make_error( formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(), inconvertibleErrorCode()); } } return UnrollOpts; } } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline /// alias. static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { return Name.startswith("default") || Name.startswith("thinlto") || Name.startswith("lto"); } /// Tests whether registered callbacks will accept a given pass name. /// /// When parsing a pipeline text, the type of the outermost pipeline may be /// omitted, in which case the type is automatically determined from the first /// pass name in the text. This may be a name that is handled through one of the /// callbacks. We check this through the oridinary parsing callbacks by setting /// up a dummy PassManager in order to not force the client to also handle this /// type of query. template static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) { if (!Callbacks.empty()) { PassManagerT DummyPM; for (auto &CB : Callbacks) if (CB(Name, DummyPM, {})) return true; } return false; } template static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { // Manually handle aliases for pre-configured pipeline fragments. if (startsWithDefaultPipelineAliasPrefix(Name)) return DefaultAliasRegex.match(Name); // Explicitly handle pass manager names. if (Name == "module") return true; if (Name == "cgscc") return true; if (Name == "function") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define MODULE_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "cgscc") return true; if (Name == "function") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; if (parseDevirtPassName(Name)) return true; #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "function") return true; if (Name == "loop") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) \ return true; #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } template static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "loop") return true; // Explicitly handle custom-parsed pass names. if (parseRepeatPassName(Name)) return true; #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; #include "PassRegistry.def" return callbacksAcceptPassName(Name, Callbacks); } Optional> PassBuilder::parsePipelineText(StringRef Text) { std::vector ResultPipeline; SmallVector *, 4> PipelineStack = { &ResultPipeline}; for (;;) { std::vector &Pipeline = *PipelineStack.back(); size_t Pos = Text.find_first_of(",()"); Pipeline.push_back({Text.substr(0, Pos), {}}); // If we have a single terminating name, we're done. if (Pos == Text.npos) break; char Sep = Text[Pos]; Text = Text.substr(Pos + 1); if (Sep == ',') // Just a name ending in a comma, continue. continue; if (Sep == '(') { // Push the inner pipeline onto the stack to continue processing. PipelineStack.push_back(&Pipeline.back().InnerPipeline); continue; } assert(Sep == ')' && "Bogus separator!"); // When handling the close parenthesis, we greedily consume them to avoid // empty strings in the pipeline. do { // If we try to pop the outer pipeline we have unbalanced parentheses. if (PipelineStack.size() == 1) return None; PipelineStack.pop_back(); } while (Text.consume_front(")")); // Check if we've finished parsing. if (Text.empty()) break; // Otherwise, the end of an inner pipeline always has to be followed by // a comma, and then we can continue. if (!Text.consume_front(",")) return None; } if (PipelineStack.size() > 1) // Unbalanced paretheses. return None; assert(PipelineStack.back() == &ResultPipeline && "Wrong pipeline at the bottom of the stack!"); return {std::move(ResultPipeline)}; } Error PassBuilder::parseModulePass(ModulePassManager &MPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "module") { ModulePassManager NestedMPM(DebugLogging); if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(std::move(NestedMPM)); return Error::success(); } if (Name == "cgscc") { CGSCCPassManager CGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); return Error::success(); } if (Name == "function") { FunctionPassManager FPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { ModulePassManager NestedMPM(DebugLogging); if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM))); return Error::success(); } for (auto &C : ModulePipelineParsingCallbacks) if (C(Name, MPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as module pipeline", Name).str(), inconvertibleErrorCode()); ; } // Manually handle aliases for pre-configured pipeline fragments. if (startsWithDefaultPipelineAliasPrefix(Name)) { SmallVector Matches; if (!DefaultAliasRegex.match(Name, &Matches)) return make_error( formatv("unknown default pipeline alias '{0}'", Name).str(), inconvertibleErrorCode()); assert(Matches.size() == 3 && "Must capture two matched strings!"); OptimizationLevel L = StringSwitch(Matches[2]) .Case("O0", O0) .Case("O1", O1) .Case("O2", O2) .Case("O3", O3) .Case("Os", Os) .Case("Oz", Oz); if (L == O0) // At O0 we do nothing at all! return Error::success(); if (Matches[1] == "default") { MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging)); } else if (Matches[1] == "thinlto-pre-link") { MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging)); } else if (Matches[1] == "thinlto") { MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr)); } else if (Matches[1] == "lto-pre-link") { MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging)); } else { assert(Matches[1] == "lto" && "Not one of the matched options!"); MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr)); } return Error::success(); } // Finally expand the basic registered passes from the .inc file. #define MODULE_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ MPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ MPM.addPass( \ RequireAnalysisPass< \ std::remove_reference::type, Module>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ MPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : ModulePipelineParsingCallbacks) if (C(Name, MPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown module pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "cgscc") { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. CGPM.addPass(std::move(NestedCGPM)); return Error::success(); } if (Name == "function") { FunctionPassManager FPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM))); return Error::success(); } if (auto MaxRepetitions = parseDevirtPassName(Name)) { CGSCCPassManager NestedCGPM(DebugLogging); if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; CGPM.addPass( createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions)); return Error::success(); } for (auto &C : CGSCCPipelineParsingCallbacks) if (C(Name, CGPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define CGSCC_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ CGPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ CGPM.addPass(RequireAnalysisPass< \ std::remove_reference::type, \ LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \ CGSCCUpdateResult &>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ CGPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : CGSCCPipelineParsingCallbacks) if (C(Name, CGPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown cgscc pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { auto &Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "function") { FunctionPassManager NestedFPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. FPM.addPass(std::move(NestedFPM)); return Error::success(); } if (Name == "loop") { LoopPassManager LPM(DebugLogging); if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. FPM.addPass( createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { FunctionPassManager NestedFPM(DebugLogging); if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM))); return Error::success(); } for (auto &C : FunctionPipelineParsingCallbacks) if (C(Name, FPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as function pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ FPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (checkParametrizedPassName(Name, NAME)) { \ auto Params = parsePassParameters(PARSER, Name, NAME); \ if (!Params) \ return Params.takeError(); \ FPM.addPass(CREATE_PASS(Params.get())); \ return Error::success(); \ } #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ FPM.addPass( \ RequireAnalysisPass< \ std::remove_reference::type, Function>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ FPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : FunctionPipelineParsingCallbacks) if (C(Name, FPM, InnerPipeline)) return Error::success(); return make_error( formatv("unknown function pass '{0}'", Name).str(), inconvertibleErrorCode()); } Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, bool VerifyEachPass, bool DebugLogging) { StringRef Name = E.Name; auto &InnerPipeline = E.InnerPipeline; // First handle complex passes like the pass managers which carry pipelines. if (!InnerPipeline.empty()) { if (Name == "loop") { LoopPassManager NestedLPM(DebugLogging); if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; // Add the nested pass manager with the appropriate adaptor. LPM.addPass(std::move(NestedLPM)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { LoopPassManager NestedLPM(DebugLogging); if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, DebugLogging)) return Err; LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM))); return Error::success(); } for (auto &C : LoopPipelineParsingCallbacks) if (C(Name, LPM, InnerPipeline)) return Error::success(); // Normal passes can't have pipelines. return make_error( formatv("invalid use of '{0}' pass as loop pipeline", Name).str(), inconvertibleErrorCode()); } // Now expand the basic registered passes from the .inc file. #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ LPM.addPass(CREATE_PASS); \ return Error::success(); \ } #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">") { \ LPM.addPass(RequireAnalysisPass< \ std::remove_reference::type, Loop, \ LoopAnalysisManager, LoopStandardAnalysisResults &, \ LPMUpdater &>()); \ return Error::success(); \ } \ if (Name == "invalidate<" NAME ">") { \ LPM.addPass(InvalidateAnalysisPass< \ std::remove_reference::type>()); \ return Error::success(); \ } #include "PassRegistry.def" for (auto &C : LoopPipelineParsingCallbacks) if (C(Name, LPM, InnerPipeline)) return Error::success(); return make_error(formatv("unknown loop pass '{0}'", Name).str(), inconvertibleErrorCode()); } bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) { #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (Name == NAME) { \ AA.registerModuleAnalysis< \ std::remove_reference::type>(); \ return true; \ } #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ if (Name == NAME) { \ AA.registerFunctionAnalysis< \ std::remove_reference::type>(); \ return true; \ } #include "PassRegistry.def" for (auto &C : AAParsingCallbacks) if (C(Name, AA)) return true; return false; } Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging)) return Err; // FIXME: No verifier support for Loop passes! } return Error::success(); } Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging)) return Err; if (VerifyEachPass) FPM.addPass(VerifierPass()); } return Error::success(); } Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging)) return Err; // FIXME: No verifier support for CGSCC passes! } return Error::success(); } void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, ModuleAnalysisManager &MAM) { MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); }); MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); }); CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); }); FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); }); FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); }); FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); }); LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); }); } Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, ArrayRef Pipeline, bool VerifyEachPass, bool DebugLogging) { for (const auto &Element : Pipeline) { if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging)) return Err; if (VerifyEachPass) MPM.addPass(VerifierPass()); } return Error::success(); } // Primary pass pipeline description parsing routine for a \c ModulePassManager // FIXME: Should this routine accept a TargetMachine or require the caller to // pre-populate the analysis managers with target-specific stuff? Error PassBuilder::parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); // If the first name isn't at the module layer, wrap the pipeline up // automatically. StringRef FirstName = Pipeline->front().Name; if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) { if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) { Pipeline = {{"cgscc", std::move(*Pipeline)}}; } else if (isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) { Pipeline = {{"function", std::move(*Pipeline)}}; } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) { Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}}; } else { for (auto &C : TopLevelPipelineParsingCallbacks) if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging)) return Error::success(); // Unknown pass or pipeline name! auto &InnerPipeline = Pipeline->front().InnerPipeline; return make_error( formatv("unknown {0} name '{1}'", (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName) .str(), inconvertibleErrorCode()); } } if (auto Err = parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c CGSCCPassManager Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); StringRef FirstName = Pipeline->front().Name; if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) return make_error( formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName, PipelineText) .str(), inconvertibleErrorCode()); if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c // FunctionPassManager Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); StringRef FirstName = Pipeline->front().Name; if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) return make_error( formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName, PipelineText) .str(), inconvertibleErrorCode()); if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } // Primary pass pipeline description parsing routine for a \c LoopPassManager Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM, StringRef PipelineText, bool VerifyEachPass, bool DebugLogging) { auto Pipeline = parsePipelineText(PipelineText); if (!Pipeline || Pipeline->empty()) return make_error( formatv("invalid pipeline '{0}'", PipelineText).str(), inconvertibleErrorCode()); if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging)) return Err; return Error::success(); } Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { // If the pipeline just consists of the word 'default' just replace the AA // manager with our default one. if (PipelineText == "default") { AA = buildDefaultAAPipeline(); return Error::success(); } while (!PipelineText.empty()) { StringRef Name; std::tie(Name, PipelineText) = PipelineText.split(','); if (!parseAAPassName(AA, Name)) return make_error( formatv("unknown alias analysis name '{0}'", Name).str(), inconvertibleErrorCode()); } return Error::success(); } Index: llvm/trunk/test/Instrumentation/ThreadSanitizer/tsan_basic.ll =================================================================== --- llvm/trunk/test/Instrumentation/ThreadSanitizer/tsan_basic.ll (revision 351313) +++ llvm/trunk/test/Instrumentation/ThreadSanitizer/tsan_basic.ll (revision 351314) @@ -1,82 +1,82 @@ ; RUN: opt < %s -tsan -S | FileCheck %s +; RUN: opt < %s -passes=tsan -S | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "x86_64-unknown-linux-gnu" define i32 @read_4_bytes(i32* %a) sanitize_thread { entry: %tmp1 = load i32, i32* %a, align 4 ret i32 %tmp1 } ; CHECK: @llvm.global_ctors = {{.*}}@tsan.module_ctor ; CHECK: define i32 @read_4_bytes(i32* %a) ; CHECK: call void @__tsan_func_entry(i8* %0) ; CHECK-NEXT: %1 = bitcast i32* %a to i8* ; CHECK-NEXT: call void @__tsan_read4(i8* %1) ; CHECK-NEXT: %tmp1 = load i32, i32* %a, align 4 ; CHECK-NEXT: call void @__tsan_func_exit() ; CHECK: ret i32 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) ; Check that tsan converts mem intrinsics back to function calls. define void @MemCpyTest(i8* nocapture %x, i8* nocapture %y) sanitize_thread { entry: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %x, i8* align 4 %y, i64 16, i1 false) ret void ; CHECK: define void @MemCpyTest ; CHECK: call i8* @memcpy ; CHECK: ret void } define void @MemMoveTest(i8* nocapture %x, i8* nocapture %y) sanitize_thread { entry: tail call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 %x, i8* align 4 %y, i64 16, i1 false) ret void ; CHECK: define void @MemMoveTest ; CHECK: call i8* @memmove ; CHECK: ret void } define void @MemSetTest(i8* nocapture %x) sanitize_thread { entry: tail call void @llvm.memset.p0i8.i64(i8* align 4 %x, i8 77, i64 16, i1 false) ret void ; CHECK: define void @MemSetTest ; CHECK: call i8* @memset ; CHECK: ret void } ; CHECK-LABEL: @SwiftError ; CHECK-NOT: __tsan_read ; CHECK-NOT: __tsan_write ; CHECK: ret define void @SwiftError(i8** swifterror) sanitize_thread { %swifterror_ptr_value = load i8*, i8** %0 store i8* null, i8** %0 %swifterror_addr = alloca swifterror i8* %swifterror_ptr_value_2 = load i8*, i8** %swifterror_addr store i8* null, i8** %swifterror_addr ret void } ; CHECK-LABEL: @SwiftErrorCall ; CHECK-NOT: __tsan_read ; CHECK-NOT: __tsan_write ; CHECK: ret define void @SwiftErrorCall(i8** swifterror) sanitize_thread { %swifterror_addr = alloca swifterror i8* store i8* null, i8** %0 call void @SwiftError(i8** %0) ret void } - ; CHECK: define internal void @tsan.module_ctor() ; CHECK: call void @__tsan_init() Index: llvm/trunk/include/llvm/InitializePasses.h =================================================================== --- llvm/trunk/include/llvm/InitializePasses.h (revision 351313) +++ llvm/trunk/include/llvm/InitializePasses.h (revision 351314) @@ -1,415 +1,415 @@ //===- llvm/InitializePasses.h - Initialize All Passes ----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file contains the declarations for the pass initialization routines // for the entire LLVM project. // //===----------------------------------------------------------------------===// #ifndef LLVM_INITIALIZEPASSES_H #define LLVM_INITIALIZEPASSES_H namespace llvm { class PassRegistry; /// Initialize all passes linked into the TransformUtils library. void initializeCore(PassRegistry&); /// Initialize all passes linked into the TransformUtils library. void initializeTransformUtils(PassRegistry&); /// Initialize all passes linked into the ScalarOpts library. void initializeScalarOpts(PassRegistry&); /// Initialize all passes linked into the ObjCARCOpts library. void initializeObjCARCOpts(PassRegistry&); /// Initialize all passes linked into the Vectorize library. void initializeVectorization(PassRegistry&); /// Initialize all passes linked into the InstCombine library. void initializeInstCombine(PassRegistry&); /// Initialize all passes linked into the AggressiveInstCombine library. void initializeAggressiveInstCombine(PassRegistry&); /// Initialize all passes linked into the IPO library. void initializeIPO(PassRegistry&); /// Initialize all passes linked into the Instrumentation library. void initializeInstrumentation(PassRegistry&); /// Initialize all passes linked into the Analysis library. void initializeAnalysis(PassRegistry&); /// Initialize all passes linked into the Coroutines library. void initializeCoroutines(PassRegistry&); /// Initialize all passes linked into the CodeGen library. void initializeCodeGen(PassRegistry&); /// Initialize all passes linked into the GlobalISel library. void initializeGlobalISel(PassRegistry&); /// Initialize all passes linked into the CodeGen library. void initializeTarget(PassRegistry&); void initializeAAEvalLegacyPassPass(PassRegistry&); void initializeAAResultsWrapperPassPass(PassRegistry&); void initializeADCELegacyPassPass(PassRegistry&); void initializeAddDiscriminatorsLegacyPassPass(PassRegistry&); void initializeAddressSanitizerModulePass(PassRegistry&); void initializeAddressSanitizerPass(PassRegistry&); void initializeAggressiveInstCombinerLegacyPassPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); void initializeAlwaysInlinerLegacyPassPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); void initializeBlockExtractorPass(PassRegistry &); void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&); void initializeBoundsCheckingLegacyPassPass(PassRegistry&); void initializeBranchFolderPassPass(PassRegistry&); void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&); void initializeBranchRelaxationPass(PassRegistry&); void initializeBreakCriticalEdgesPass(PassRegistry&); void initializeBreakFalseDepsPass(PassRegistry&); void initializeCanonicalizeAliasesLegacyPassPass(PassRegistry &); void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&); void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&); void initializeCFGPrinterLegacyPassPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry&); void initializeCallSiteSplittingLegacyPassPass(PassRegistry&); void initializeCalledValuePropagationLegacyPassPass(PassRegistry &); void initializeCodeGenPreparePass(PassRegistry&); void initializeConstantHoistingLegacyPassPass(PassRegistry&); void initializeConstantMergeLegacyPassPass(PassRegistry&); void initializeConstantPropagationPass(PassRegistry&); void initializeControlHeightReductionLegacyPassPass(PassRegistry&); void initializeCorrelatedValuePropagationPass(PassRegistry&); void initializeCostModelAnalysisPass(PassRegistry&); void initializeCrossDSOCFIPass(PassRegistry&); void initializeDAEPass(PassRegistry&); void initializeDAHPass(PassRegistry&); void initializeDCELegacyPassPass(PassRegistry&); void initializeDSELegacyPassPass(PassRegistry&); void initializeDataFlowSanitizerPass(PassRegistry&); void initializeDeadInstEliminationPass(PassRegistry&); void initializeDeadMachineInstructionElimPass(PassRegistry&); void initializeDelinearizationPass(PassRegistry&); void initializeDemandedBitsWrapperPassPass(PassRegistry&); void initializeDependenceAnalysisPass(PassRegistry&); void initializeDependenceAnalysisWrapperPassPass(PassRegistry&); void initializeDetectDeadLanesPass(PassRegistry&); void initializeDivRemPairsLegacyPassPass(PassRegistry&); void initializeDomOnlyPrinterPass(PassRegistry&); void initializeDomOnlyViewerPass(PassRegistry&); void initializeDomPrinterPass(PassRegistry&); void initializeDomViewerPass(PassRegistry&); void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); void initializeDwarfEHPreparePass(PassRegistry&); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); void initializeEarlyIfConverterPass(PassRegistry&); void initializeEarlyMachineLICMPass(PassRegistry&); void initializeEarlyTailDuplicatePass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); void initializeEfficiencySanitizerPass(PassRegistry&); void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry&); void initializeEntryExitInstrumenterPass(PassRegistry&); void initializeExpandISelPseudosPass(PassRegistry&); void initializeExpandMemCmpPassPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); void initializeExpandReductionsPass(PassRegistry&); void initializeMakeGuardsExplicitLegacyPassPass(PassRegistry&); void initializeExternalAAWrapperPassPass(PassRegistry&); void initializeFEntryInserterPass(PassRegistry&); void initializeFinalizeMachineBundlesPass(PassRegistry&); void initializeFlattenCFGPassPass(PassRegistry&); void initializeFloat2IntLegacyPassPass(PassRegistry&); void initializeForceFunctionAttrsLegacyPassPass(PassRegistry&); void initializeForwardControlFlowIntegrityPass(PassRegistry&); void initializeFuncletLayoutPass(PassRegistry&); void initializeFunctionImportLegacyPassPass(PassRegistry&); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGCOVProfilerLegacyPassPass(PassRegistry&); void initializeGVNHoistLegacyPassPass(PassRegistry&); void initializeGVNLegacyPassPass(PassRegistry&); void initializeGVNSinkLegacyPassPass(PassRegistry&); void initializeGlobalDCELegacyPassPass(PassRegistry&); void initializeGlobalMergePass(PassRegistry&); void initializeGlobalOptLegacyPassPass(PassRegistry&); void initializeGlobalSplitPass(PassRegistry&); void initializeGlobalsAAWrapperPassPass(PassRegistry&); void initializeGuardWideningLegacyPassPass(PassRegistry&); void initializeHotColdSplittingLegacyPassPass(PassRegistry&); void initializeHWAddressSanitizerPass(PassRegistry&); void initializeIPCPPass(PassRegistry&); void initializeIPSCCPLegacyPassPass(PassRegistry&); void initializeIRCELegacyPassPass(PassRegistry&); void initializeIRTranslatorPass(PassRegistry&); void initializeIVUsersWrapperPassPass(PassRegistry&); void initializeIfConverterPass(PassRegistry&); void initializeImplicitNullChecksPass(PassRegistry&); void initializeIndVarSimplifyLegacyPassPass(PassRegistry&); void initializeIndirectBrExpandPassPass(PassRegistry&); void initializeInferAddressSpacesPass(PassRegistry&); void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstCountPass(PassRegistry&); void initializeInstNamerPass(PassRegistry&); void initializeInstSimplifyLegacyPassPass(PassRegistry &); void initializeInstrProfilingLegacyPassPass(PassRegistry&); void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstructionSelectPass(PassRegistry&); void initializeInterleavedAccessPass(PassRegistry&); void initializeInterleavedLoadCombinePass(PassRegistry &); void initializeInternalizeLegacyPassPass(PassRegistry&); void initializeIntervalPartitionPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&); void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyValueInfoPrinterPass(PassRegistry&); void initializeLazyValueInfoWrapperPassPass(PassRegistry&); void initializeLegacyDivergenceAnalysisPass(PassRegistry&); void initializeLegacyLICMPassPass(PassRegistry&); void initializeLegacyLoopSinkPassPass(PassRegistry&); void initializeLegalizerPass(PassRegistry&); void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &); void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&); void initializeLintPass(PassRegistry&); void initializeLiveDebugValuesPass(PassRegistry&); void initializeLiveDebugVariablesPass(PassRegistry&); void initializeLiveIntervalsPass(PassRegistry&); void initializeLiveRangeShrinkPass(PassRegistry&); void initializeLiveRegMatrixPass(PassRegistry&); void initializeLiveStacksPass(PassRegistry&); void initializeLiveVariablesPass(PassRegistry&); void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry&); void initializeLoaderPassPass(PassRegistry&); void initializeLocalStackSlotPassPass(PassRegistry&); void initializeLocalizerPass(PassRegistry&); void initializeLoopAccessLegacyAnalysisPass(PassRegistry&); void initializeLoopDataPrefetchLegacyPassPass(PassRegistry&); void initializeLoopDeletionLegacyPassPass(PassRegistry&); void initializeLoopDistributeLegacyPass(PassRegistry&); void initializeLoopExtractorPass(PassRegistry&); void initializeLoopGuardWideningLegacyPassPass(PassRegistry&); void initializeLoopIdiomRecognizeLegacyPassPass(PassRegistry&); void initializeLoopInfoWrapperPassPass(PassRegistry&); void initializeLoopInstSimplifyLegacyPassPass(PassRegistry&); void initializeLoopInterchangePass(PassRegistry&); void initializeLoopLoadEliminationPass(PassRegistry&); void initializeLoopPassPass(PassRegistry&); void initializeLoopPredicationLegacyPassPass(PassRegistry&); void initializeLoopRerollPass(PassRegistry&); void initializeLoopRotateLegacyPassPass(PassRegistry&); void initializeLoopSimplifyCFGLegacyPassPass(PassRegistry&); void initializeLoopSimplifyPass(PassRegistry&); void initializeLoopStrengthReducePass(PassRegistry&); void initializeLoopUnrollAndJamPass(PassRegistry&); void initializeLoopUnrollPass(PassRegistry&); void initializeLoopUnswitchPass(PassRegistry&); void initializeLoopVectorizePass(PassRegistry&); void initializeLoopVersioningLICMPass(PassRegistry&); void initializeLoopVersioningPassPass(PassRegistry&); void initializeLowerAtomicLegacyPassPass(PassRegistry&); void initializeLowerEmuTLSPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokeLegacyPassPass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); void initializeLowerTypeTestsPass(PassRegistry&); void initializeMIRCanonicalizerPass(PassRegistry &); void initializeMIRPrintingPassPass(PassRegistry&); void initializeMachineBlockFrequencyInfoPass(PassRegistry&); void initializeMachineBlockPlacementPass(PassRegistry&); void initializeMachineBlockPlacementStatsPass(PassRegistry&); void initializeMachineBranchProbabilityInfoPass(PassRegistry&); void initializeMachineCSEPass(PassRegistry&); void initializeMachineCombinerPass(PassRegistry&); void initializeMachineCopyPropagationPass(PassRegistry&); void initializeMachineDominanceFrontierPass(PassRegistry&); void initializeMachineDominatorTreePass(PassRegistry&); void initializeMachineFunctionPrinterPassPass(PassRegistry&); void initializeMachineLICMPass(PassRegistry&); void initializeMachineLoopInfoPass(PassRegistry&); void initializeMachineModuleInfoPass(PassRegistry&); void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&); void initializeMachineOutlinerPass(PassRegistry&); void initializeMachinePipelinerPass(PassRegistry&); void initializeMachinePostDominatorTreePass(PassRegistry&); void initializeMachineRegionInfoPassPass(PassRegistry&); void initializeMachineSchedulerPass(PassRegistry&); void initializeMachineSinkingPass(PassRegistry&); void initializeMachineTraceMetricsPass(PassRegistry&); void initializeMachineVerifierPassPass(PassRegistry&); void initializeMemCpyOptLegacyPassPass(PassRegistry&); void initializeMemDepPrinterPass(PassRegistry&); void initializeMemDerefPrinterPass(PassRegistry&); void initializeMemoryDependenceWrapperPassPass(PassRegistry&); void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerLegacyPassPass(PassRegistry&); void initializeMergeFunctionsPass(PassRegistry&); void initializeMergeICmpsPass(PassRegistry&); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&); void initializeMustExecutePrinterPass(PassRegistry&); void initializeNameAnonGlobalLegacyPassPass(PassRegistry&); void initializeNaryReassociateLegacyPassPass(PassRegistry&); void initializeNewGVNLegacyPassPass(PassRegistry&); void initializeObjCARCAAWrapperPassPass(PassRegistry&); void initializeObjCARCAPElimPass(PassRegistry&); void initializeObjCARCContractPass(PassRegistry&); void initializeObjCARCExpandPass(PassRegistry&); void initializeObjCARCOptPass(PassRegistry&); void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry&); void initializeOptimizePHIsPass(PassRegistry&); void initializePAEvalPass(PassRegistry&); void initializePEIPass(PassRegistry&); void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&); void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&); void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&); void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerLegacyPassPass(PassRegistry&); void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry&); void initializePatchableFunctionPass(PassRegistry&); void initializePeepholeOptimizerPass(PassRegistry&); void initializePhiValuesWrapperPassPass(PassRegistry&); void initializePhysicalRegisterUsageInfoPass(PassRegistry&); void initializePlaceBackedgeSafepointsImplPass(PassRegistry&); void initializePlaceSafepointsPass(PassRegistry&); void initializePostDomOnlyPrinterPass(PassRegistry&); void initializePostDomOnlyViewerPass(PassRegistry&); void initializePostDomPrinterPass(PassRegistry&); void initializePostDomViewerPass(PassRegistry&); void initializePostDominatorTreeWrapperPassPass(PassRegistry&); void initializePostInlineEntryExitInstrumenterPass(PassRegistry&); void initializePostMachineSchedulerPass(PassRegistry&); void initializePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); void initializePostRAHazardRecognizerPass(PassRegistry&); void initializePostRAMachineSinkingPass(PassRegistry&); void initializePostRASchedulerPass(PassRegistry&); void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry&); void initializePredicateInfoPrinterLegacyPassPass(PassRegistry&); void initializePrintBasicBlockPassPass(PassRegistry&); void initializePrintFunctionPassWrapperPass(PassRegistry&); void initializePrintModulePassWrapperPass(PassRegistry&); void initializeProcessImplicitDefsPass(PassRegistry&); void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&); void initializePromoteLegacyPassPass(PassRegistry&); void initializePruneEHPass(PassRegistry&); void initializeRABasicPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReachingDefAnalysisPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRegAllocFastPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); void initializeRegToMemPass(PassRegistry&); void initializeRegUsageInfoCollectorPass(PassRegistry&); void initializeRegUsageInfoPropagationPass(PassRegistry&); void initializeRegionInfoPassPass(PassRegistry&); void initializeRegionOnlyPrinterPass(PassRegistry&); void initializeRegionOnlyViewerPass(PassRegistry&); void initializeRegionPrinterPass(PassRegistry&); void initializeRegionViewerPass(PassRegistry&); void initializeRegisterCoalescerPass(PassRegistry&); void initializeRenameIndependentSubregsPass(PassRegistry&); void initializeResetMachineFunctionPass(PassRegistry&); void initializeReversePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); void initializeRewriteStatepointsForGCLegacyPassPass(PassRegistry &); void initializeRewriteSymbolsLegacyPassPass(PassRegistry&); void initializeSCCPLegacyPassPass(PassRegistry&); void initializeSCEVAAWrapperPassPass(PassRegistry&); void initializeSLPVectorizerPass(PassRegistry&); void initializeSROALegacyPassPass(PassRegistry&); void initializeSafeStackLegacyPassPass(PassRegistry&); void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&); void initializeSanitizerCoverageModulePass(PassRegistry&); void initializeScalarEvolutionWrapperPassPass(PassRegistry&); void initializeScalarizeMaskedMemIntrinPass(PassRegistry&); void initializeScalarizerLegacyPassPass(PassRegistry&); void initializeScavengerTestPass(PassRegistry&); void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&); void initializeSeparateConstOffsetFromGEPPass(PassRegistry&); void initializeShadowStackGCLoweringPass(PassRegistry&); void initializeShrinkWrapPass(PassRegistry&); void initializeSimpleInlinerPass(PassRegistry&); void initializeSimpleLoopUnswitchLegacyPassPass(PassRegistry&); void initializeSingleLoopExtractorPass(PassRegistry&); void initializeSinkingLegacyPassPass(PassRegistry&); void initializeSjLjEHPreparePass(PassRegistry&); void initializeSlotIndexesPass(PassRegistry&); void initializeSpeculativeExecutionLegacyPassPass(PassRegistry&); void initializeSpillPlacementPass(PassRegistry&); void initializeStackColoringPass(PassRegistry&); void initializeStackMapLivenessPass(PassRegistry&); void initializeStackProtectorPass(PassRegistry&); void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &); void initializeStackSafetyInfoWrapperPassPass(PassRegistry &); void initializeStackSlotColoringPass(PassRegistry&); void initializeStraightLineStrengthReducePass(PassRegistry&); void initializeStripDeadDebugInfoPass(PassRegistry&); void initializeStripDeadPrototypesLegacyPassPass(PassRegistry&); void initializeStripDebugDeclarePass(PassRegistry&); void initializeStripGCRelocatesPass(PassRegistry&); void initializeStripNonDebugSymbolsPass(PassRegistry&); void initializeStripNonLineTableDebugInfoPass(PassRegistry&); void initializeStripSymbolsPass(PassRegistry&); void initializeStructurizeCFGPass(PassRegistry&); void initializeTailCallElimPass(PassRegistry&); void initializeTailDuplicatePass(PassRegistry&); void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&); void initializeTargetPassConfigPass(PassRegistry&); void initializeTargetTransformInfoWrapperPassPass(PassRegistry&); -void initializeThreadSanitizerPass(PassRegistry&); +void initializeThreadSanitizerLegacyPassPass(PassRegistry&); void initializeTwoAddressInstructionPassPass(PassRegistry&); void initializeTypeBasedAAWrapperPassPass(PassRegistry&); void initializeUnifyFunctionExitNodesPass(PassRegistry&); void initializeUnpackMachineBundlesPass(PassRegistry&); void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&); void initializeUnreachableMachineBlockElimPass(PassRegistry&); void initializeVerifierLegacyPassPass(PassRegistry&); void initializeVirtRegMapPass(PassRegistry&); void initializeVirtRegRewriterPass(PassRegistry&); void initializeWarnMissedTransformationsLegacyPass(PassRegistry &); void initializeWasmEHPreparePass(PassRegistry&); void initializeWholeProgramDevirtPass(PassRegistry&); void initializeWinEHPreparePass(PassRegistry&); void initializeWriteBitcodePassPass(PassRegistry&); void initializeWriteThinLTOBitcodePass(PassRegistry&); void initializeXRayInstrumentationPass(PassRegistry&); } // end namespace llvm #endif // LLVM_INITIALIZEPASSES_H Index: llvm/trunk/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h =================================================================== --- llvm/trunk/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h (revision 0) +++ llvm/trunk/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h (revision 351314) @@ -0,0 +1,33 @@ +//===- Transforms/Instrumentation/MemorySanitizer.h - TSan Pass -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the thread sanitizer pass. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" + +namespace llvm { +// Insert ThreadSanitizer (race detection) instrumentation +FunctionPass *createThreadSanitizerLegacyPassPass(); + +/// A function pass for tsan instrumentation. +/// +/// Instruments functions to detect race conditions reads. This function pass +/// inserts calls to runtime library functions. If the functions aren't declared +/// yet, the pass inserts the declarations. Otherwise the existing globals are +struct ThreadSanitizerPass : public PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; +} // namespace llvm +#endif /* LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H */ Index: llvm/trunk/include/llvm/Transforms/Instrumentation.h =================================================================== --- llvm/trunk/include/llvm/Transforms/Instrumentation.h (revision 351313) +++ llvm/trunk/include/llvm/Transforms/Instrumentation.h (revision 351314) @@ -1,230 +1,227 @@ //===- Transforms/Instrumentation.h - Instrumentation passes ----*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file defines constructor functions for instrumentation passes. // //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_H #define LLVM_TRANSFORMS_INSTRUMENTATION_H #include "llvm/ADT/StringRef.h" #include "llvm/IR/BasicBlock.h" #include #include #include #include #include namespace llvm { class Triple; class FunctionPass; class ModulePass; class OptimizationRemarkEmitter; class Comdat; /// Instrumentation passes often insert conditional checks into entry blocks. /// Call this function before splitting the entry block to move instructions /// that must remain in the entry block up before the split point. Static /// allocas and llvm.localescape calls, for example, must remain in the entry /// block. BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB, BasicBlock::iterator IP); // Create a constant for Str so that we can pass it to the run-time lib. GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, bool AllowMerging, const char *NamePrefix = ""); // Returns F.getComdat() if it exists. // Otherwise creates a new comdat, sets F's comdat, and returns it. // Returns nullptr on failure. Comdat *GetOrCreateFunctionComdat(Function &F, Triple &T, const std::string &ModuleId); // Insert GCOV profiling instrumentation struct GCOVOptions { static GCOVOptions getDefault(); // Specify whether to emit .gcno files. bool EmitNotes; // Specify whether to modify the program to emit .gcda files when run. bool EmitData; // A four-byte version string. The meaning of a version string is described in // gcc's gcov-io.h char Version[4]; // Emit a "cfg checksum" that follows the "line number checksum" of a // function. This affects both .gcno and .gcda files. bool UseCfgChecksum; // Add the 'noredzone' attribute to added runtime library calls. bool NoRedZone; // Emit the name of the function in the .gcda files. This is redundant, as // the function identifier can be used to find the name from the .gcno file. bool FunctionNamesInData; // Emit the exit block immediately after the start block, rather than after // all of the function body's blocks. bool ExitBlockBeforeBody; // Regexes separated by a semi-colon to filter the files to instrument. std::string Filter; // Regexes separated by a semi-colon to filter the files to not instrument. std::string Exclude; }; ModulePass *createGCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()); // PGO Instrumention ModulePass *createPGOInstrumentationGenLegacyPass(); ModulePass * createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef("")); ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false); FunctionPass *createPGOMemOPSizeOptLegacyPass(); // The pgo-specific indirect call promotion function declared below is used by // the pgo-driven indirect call promotion and sample profile passes. It's a // wrapper around llvm::promoteCall, et al. that additionally computes !prof // metadata. We place it in a pgo namespace so it's not confused with the // generic utilities. namespace pgo { // Helper function that transforms Inst (either an indirect-call instruction, or // an invoke instruction , to a conditional call to F. This is like: // if (Inst.CalledValue == F) // F(...); // else // Inst(...); // end // TotalCount is the profile count value that the instruction executes. // Count is the profile count value that F is the target function. // These two values are used to update the branch weight. // If \p AttachProfToDirectCall is true, a prof metadata is attached to the // new direct call to contain \p Count. // Returns the promoted direct call instruction. Instruction *promoteIndirectCall(Instruction *Inst, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE); } // namespace pgo /// Options for the frontend instrumentation based profiling pass. struct InstrProfOptions { // Add the 'noredzone' attribute to added runtime library calls. bool NoRedZone = false; // Do counter register promotion bool DoCounterPromotion = false; // Use atomic profile counter increments. bool Atomic = false; // Name of the profile file to use as output std::string InstrProfileOutput; InstrProfOptions() = default; }; /// Insert frontend instrumentation based profiling. ModulePass *createInstrProfilingLegacyPass( const InstrProfOptions &Options = InstrProfOptions()); // Insert AddressSanitizer (address sanity checking) instrumentation FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false, bool Recover = false, bool UseAfterScope = false); ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false, bool Recover = false, bool UseGlobalsGC = true, bool UseOdrIndicator = true); FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, bool Recover = false); -// Insert ThreadSanitizer (race detection) instrumentation -FunctionPass *createThreadSanitizerPass(); - // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation ModulePass *createDataFlowSanitizerPass( const std::vector &ABIListFiles = std::vector(), void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr); // Options for EfficiencySanitizer sub-tools. struct EfficiencySanitizerOptions { enum Type { ESAN_None = 0, ESAN_CacheFrag, ESAN_WorkingSet, } ToolType = ESAN_None; EfficiencySanitizerOptions() = default; }; // Insert EfficiencySanitizer instrumentation. ModulePass *createEfficiencySanitizerPass( const EfficiencySanitizerOptions &Options = EfficiencySanitizerOptions()); // Options for sanitizer coverage instrumentation. struct SanitizerCoverageOptions { enum Type { SCK_None = 0, SCK_Function, SCK_BB, SCK_Edge } CoverageType = SCK_None; bool IndirectCalls = false; bool TraceBB = false; bool TraceCmp = false; bool TraceDiv = false; bool TraceGep = false; bool Use8bitCounters = false; bool TracePC = false; bool TracePCGuard = false; bool Inline8bitCounters = false; bool PCTable = false; bool NoPrune = false; bool StackDepth = false; SanitizerCoverageOptions() = default; }; // Insert SanitizerCoverage instrumentation. ModulePass *createSanitizerCoverageModulePass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); /// Calculate what to divide by to scale counts. /// /// Given the maximum count, calculate a divisor that will scale all the /// weights to strictly less than std::numeric_limits::max(). static inline uint64_t calculateCountScale(uint64_t MaxCount) { return MaxCount < std::numeric_limits::max() ? 1 : MaxCount / std::numeric_limits::max() + 1; } /// Scale an individual branch count. /// /// Scale a 64-bit weight down to 32-bits using \c Scale. /// static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) { uint64_t Scaled = Count / Scale; assert(Scaled <= std::numeric_limits::max() && "overflow 32-bits"); return Scaled; } } // end namespace llvm #endif // LLVM_TRANSFORMS_INSTRUMENTATION_H Index: llvm/trunk/include/llvm/Transforms/Utils/ModuleUtils.h =================================================================== --- llvm/trunk/include/llvm/Transforms/Utils/ModuleUtils.h (revision 351313) +++ llvm/trunk/include/llvm/Transforms/Utils/ModuleUtils.h (revision 351314) @@ -1,106 +1,119 @@ //===-- ModuleUtils.h - Functions to manipulate Modules ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This family of functions perform manipulations on Modules. // //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #include "llvm/ADT/StringRef.h" #include // for std::pair namespace llvm { template class ArrayRef; class Module; class Function; class GlobalValue; class GlobalVariable; class Constant; class StringRef; class Value; class Type; /// Append F to the list of global ctors of module M with the given Priority. /// This wraps the function in the appropriate structure and stores it along /// side other global constructors. For details see /// http://llvm.org/docs/LangRef.html#intg_global_ctors void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data = nullptr); /// Same as appendToGlobalCtors(), but for global dtors. void appendToGlobalDtors(Module &M, Function *F, int Priority, Constant *Data = nullptr); // Validate the result of Module::getOrInsertFunction called for an interface // function of given sanitizer. If the instrumented module defines a function // with the same name, their prototypes must match, otherwise // getOrInsertFunction returns a bitcast. Function *checkSanitizerInterfaceFunction(Constant *FuncOrBitcast); Function *declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef InitArgTypes); /// Creates sanitizer constructor function, and calls sanitizer's init /// function from it. /// \return Returns pair of pointers to constructor, and init functions /// respectively. std::pair createSanitizerCtorAndInitFunctions( Module &M, StringRef CtorName, StringRef InitName, ArrayRef InitArgTypes, ArrayRef InitArgs, StringRef VersionCheckName = StringRef()); +/// Creates sanitizer constructor function lazily. If a constructor and init +/// function already exist, this function returns it. Otherwise it calls \c +/// createSanitizerCtorAndInitFunctions. The FunctionsCreatedCallback is invoked +/// in that case, passing the new Ctor and Init function. +/// +/// \return Returns pair of pointers to constructor, and init functions +/// respectively. +std::pair getOrCreateSanitizerCtorAndInitFunctions( + Module &M, StringRef CtorName, StringRef InitName, + ArrayRef InitArgTypes, ArrayRef InitArgs, + function_ref FunctionsCreatedCallback, + StringRef VersionCheckName = StringRef()); + // Creates and returns a sanitizer init function without argument if it doesn't // exist, and adds it to the global constructors list. Otherwise it returns the // existing function. Function *getOrCreateInitFunction(Module &M, StringRef Name); /// Rename all the anon globals in the module using a hash computed from /// the list of public globals in the module. bool nameUnamedGlobals(Module &M); /// Adds global values to the llvm.used list. void appendToUsed(Module &M, ArrayRef Values); /// Adds global values to the llvm.compiler.used list. void appendToCompilerUsed(Module &M, ArrayRef Values); /// Filter out potentially dead comdat functions where other entries keep the /// entire comdat group alive. /// /// This is designed for cases where functions appear to become dead but remain /// alive due to other live entries in their comdat group. /// /// The \p DeadComdatFunctions container should only have pointers to /// `Function`s which are members of a comdat group and are believed to be /// dead. /// /// After this routine finishes, the only remaining `Function`s in \p /// DeadComdatFunctions are those where every member of the comdat is listed /// and thus removing them is safe (provided *all* are removed). void filterDeadComdatFunctions( Module &M, SmallVectorImpl &DeadComdatFunctions); /// Produce a unique identifier for this module by taking the MD5 sum of /// the names of the module's strong external symbols that are not comdat /// members. /// /// This identifier is normally guaranteed to be unique, or the program would /// fail to link due to multiply defined symbols. /// /// If the module has no strong external symbols (such a module may still have a /// semantic effect if it performs global initialization), we cannot produce a /// unique identifier for this module, so we return the empty string. std::string getUniqueModuleId(Module *M); } // End llvm namespace #endif // LLVM_TRANSFORMS_UTILS_MODULEUTILS_H Index: llvm/trunk/bindings/go/llvm/InstrumentationBindings.cpp =================================================================== --- llvm/trunk/bindings/go/llvm/InstrumentationBindings.cpp (revision 351313) +++ llvm/trunk/bindings/go/llvm/InstrumentationBindings.cpp (revision 351314) @@ -1,47 +1,48 @@ //===- InstrumentationBindings.cpp - instrumentation bindings -------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file defines C bindings for the instrumentation component. // //===----------------------------------------------------------------------===// #include "InstrumentationBindings.h" #include "llvm-c/Core.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" using namespace llvm; void LLVMAddAddressSanitizerFunctionPass(LLVMPassManagerRef PM) { unwrap(PM)->add(createAddressSanitizerFunctionPass()); } void LLVMAddAddressSanitizerModulePass(LLVMPassManagerRef PM) { unwrap(PM)->add(createAddressSanitizerModulePass()); } void LLVMAddThreadSanitizerPass(LLVMPassManagerRef PM) { - unwrap(PM)->add(createThreadSanitizerPass()); + unwrap(PM)->add(createThreadSanitizerLegacyPassPass()); } void LLVMAddMemorySanitizerLegacyPassPass(LLVMPassManagerRef PM) { unwrap(PM)->add(createMemorySanitizerLegacyPassPass()); } void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM, int ABIListFilesNum, const char **ABIListFiles) { std::vector ABIListFilesVec; for (int i = 0; i != ABIListFilesNum; ++i) { ABIListFilesVec.push_back(ABIListFiles[i]); } unwrap(PM)->add(createDataFlowSanitizerPass(ABIListFilesVec)); } Index: cfe/trunk/lib/CodeGen/BackendUtil.cpp =================================================================== --- cfe/trunk/lib/CodeGen/BackendUtil.cpp (revision 351313) +++ cfe/trunk/lib/CodeGen/BackendUtil.cpp (revision 351314) @@ -1,1451 +1,1452 @@ //===--- BackendUtil.cpp - LLVM Backend Utilities -------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "clang/CodeGen/BackendUtil.h" #include "clang/Basic/CodeGenOptions.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/LangOptions.h" #include "clang/Basic/TargetOptions.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearchOptions.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/Verifier.h" #include "llvm/LTO/LTOBackend.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Coroutines.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/BoundsChecking.h" #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/SymbolRewriter.h" #include using namespace clang; using namespace llvm; namespace { // Default filename used for profile generation. static constexpr StringLiteral DefaultProfileGenName = "default_%m.profraw"; class EmitAssemblyHelper { DiagnosticsEngine &Diags; const HeaderSearchOptions &HSOpts; const CodeGenOptions &CodeGenOpts; const clang::TargetOptions &TargetOpts; const LangOptions &LangOpts; Module *TheModule; Timer CodeGenerationTime; std::unique_ptr OS; TargetIRAnalysis getTargetIRAnalysis() const { if (TM) return TM->getTargetIRAnalysis(); return TargetIRAnalysis(); } void CreatePasses(legacy::PassManager &MPM, legacy::FunctionPassManager &FPM); /// Generates the TargetMachine. /// Leaves TM unchanged if it is unable to create the target machine. /// Some of our clang tests specify triples which are not built /// into clang. This is okay because these tests check the generated /// IR, and they require DataLayout which depends on the triple. /// In this case, we allow this method to fail and not report an error. /// When MustCreateTM is used, we print an error if we are unable to load /// the requested target. void CreateTargetMachine(bool MustCreateTM); /// Add passes necessary to emit assembly or LLVM IR. /// /// \return True on success. bool AddEmitPasses(legacy::PassManager &CodeGenPasses, BackendAction Action, raw_pwrite_stream &OS, raw_pwrite_stream *DwoOS); std::unique_ptr openOutputFile(StringRef Path) { std::error_code EC; auto F = llvm::make_unique(Path, EC, llvm::sys::fs::F_None); if (EC) { Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message(); F.reset(); } return F; } public: EmitAssemblyHelper(DiagnosticsEngine &_Diags, const HeaderSearchOptions &HeaderSearchOpts, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, const LangOptions &LOpts, Module *M) : Diags(_Diags), HSOpts(HeaderSearchOpts), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), TheModule(M), CodeGenerationTime("codegen", "Code Generation Time") {} ~EmitAssemblyHelper() { if (CodeGenOpts.DisableFree) BuryPointer(std::move(TM)); } std::unique_ptr TM; void EmitAssembly(BackendAction Action, std::unique_ptr OS); void EmitAssemblyWithNewPassManager(BackendAction Action, std::unique_ptr OS); }; // We need this wrapper to access LangOpts and CGOpts from extension functions // that we add to the PassManagerBuilder. class PassManagerBuilderWrapper : public PassManagerBuilder { public: PassManagerBuilderWrapper(const Triple &TargetTriple, const CodeGenOptions &CGOpts, const LangOptions &LangOpts) : PassManagerBuilder(), TargetTriple(TargetTriple), CGOpts(CGOpts), LangOpts(LangOpts) {} const Triple &getTargetTriple() const { return TargetTriple; } const CodeGenOptions &getCGOpts() const { return CGOpts; } const LangOptions &getLangOpts() const { return LangOpts; } private: const Triple &TargetTriple; const CodeGenOptions &CGOpts; const LangOptions &LangOpts; }; } static void addObjCARCAPElimPass(const PassManagerBuilder &Builder, PassManagerBase &PM) { if (Builder.OptLevel > 0) PM.add(createObjCARCAPElimPass()); } static void addObjCARCExpandPass(const PassManagerBuilder &Builder, PassManagerBase &PM) { if (Builder.OptLevel > 0) PM.add(createObjCARCExpandPass()); } static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase &PM) { if (Builder.OptLevel > 0) PM.add(createObjCARCOptPass()); } static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createAddDiscriminatorsPass()); } static void addBoundsCheckingPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createBoundsCheckingLegacyPass()); } static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); SanitizerCoverageOptions Opts; Opts.CoverageType = static_cast(CGOpts.SanitizeCoverageType); Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls; Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB; Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp; Opts.TraceDiv = CGOpts.SanitizeCoverageTraceDiv; Opts.TraceGep = CGOpts.SanitizeCoverageTraceGep; Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters; Opts.TracePC = CGOpts.SanitizeCoverageTracePC; Opts.TracePCGuard = CGOpts.SanitizeCoverageTracePCGuard; Opts.NoPrune = CGOpts.SanitizeCoverageNoPrune; Opts.Inline8bitCounters = CGOpts.SanitizeCoverageInline8bitCounters; Opts.PCTable = CGOpts.SanitizeCoveragePCTable; Opts.StackDepth = CGOpts.SanitizeCoverageStackDepth; PM.add(createSanitizerCoverageModulePass(Opts)); } // Check if ASan should use GC-friendly instrumentation for globals. // First of all, there is no point if -fdata-sections is off (expect for MachO, // where this is not a factor). Also, on ELF this feature requires an assembler // extension that only works with -integrated-as at the moment. static bool asanUseGlobalsGC(const Triple &T, const CodeGenOptions &CGOpts) { if (!CGOpts.SanitizeAddressGlobalsDeadStripping) return false; switch (T.getObjectFormat()) { case Triple::MachO: case Triple::COFF: return true; case Triple::ELF: return CGOpts.DataSections && !CGOpts.DisableIntegratedAS; default: return false; } } static void addAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const Triple &T = BuilderWrapper.getTargetTriple(); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Address); bool UseAfterScope = CGOpts.SanitizeAddressUseAfterScope; bool UseOdrIndicator = CGOpts.SanitizeAddressUseOdrIndicator; bool UseGlobalsGC = asanUseGlobalsGC(T, CGOpts); PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/ false, Recover, UseAfterScope)); PM.add(createAddressSanitizerModulePass(/*CompileKernel*/ false, Recover, UseGlobalsGC, UseOdrIndicator)); } static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createAddressSanitizerFunctionPass( /*CompileKernel*/ true, /*Recover*/ true, /*UseAfterScope*/ false)); PM.add(createAddressSanitizerModulePass( /*CompileKernel*/ true, /*Recover*/ true, /*UseGlobalsGC*/ true, /*UseOdrIndicator*/ false)); } static void addHWAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::HWAddress); PM.add(createHWAddressSanitizerPass(/*CompileKernel*/ false, Recover)); } static void addKernelHWAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createHWAddressSanitizerPass( /*CompileKernel*/ true, /*Recover*/ true)); } static void addGeneralOptsForMemorySanitizer(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM, bool CompileKernel) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); int TrackOrigins = CGOpts.SanitizeMemoryTrackOrigins; bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Memory); PM.add(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover, CompileKernel)); // MemorySanitizer inserts complex instrumentation that mostly follows // the logic of the original code, but operates on "shadow" values. // It can benefit from re-running some general purpose optimization passes. if (Builder.OptLevel > 0) { PM.add(createEarlyCSEPass()); PM.add(createReassociatePass()); PM.add(createLICMPass()); PM.add(createGVNPass()); PM.add(createInstructionCombiningPass()); PM.add(createDeadStoreEliminationPass()); } } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { addGeneralOptsForMemorySanitizer(Builder, PM, /*CompileKernel*/ false); } static void addKernelMemorySanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { addGeneralOptsForMemorySanitizer(Builder, PM, /*CompileKernel*/ true); } static void addThreadSanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - PM.add(createThreadSanitizerPass()); + PM.add(createThreadSanitizerLegacyPassPass()); } static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const LangOptions &LangOpts = BuilderWrapper.getLangOpts(); PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFiles)); } static void addEfficiencySanitizerPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast(Builder); const LangOptions &LangOpts = BuilderWrapper.getLangOpts(); EfficiencySanitizerOptions Opts; if (LangOpts.Sanitize.has(SanitizerKind::EfficiencyCacheFrag)) Opts.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag; else if (LangOpts.Sanitize.has(SanitizerKind::EfficiencyWorkingSet)) Opts.ToolType = EfficiencySanitizerOptions::ESAN_WorkingSet; PM.add(createEfficiencySanitizerPass(Opts)); } static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, const CodeGenOptions &CodeGenOpts) { TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple); if (!CodeGenOpts.SimplifyLibCalls) TLII->disableAllFunctions(); else { // Disable individual libc/libm calls in TargetLibraryInfo. LibFunc F; for (auto &FuncName : CodeGenOpts.getNoBuiltinFuncs()) if (TLII->getLibFunc(FuncName, F)) TLII->setUnavailable(F); } switch (CodeGenOpts.getVecLib()) { case CodeGenOptions::Accelerate: TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate); break; case CodeGenOptions::SVML: TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML); break; default: break; } return TLII; } static void addSymbolRewriterPass(const CodeGenOptions &Opts, legacy::PassManager *MPM) { llvm::SymbolRewriter::RewriteDescriptorList DL; llvm::SymbolRewriter::RewriteMapParser MapParser; for (const auto &MapFile : Opts.RewriteMapFiles) MapParser.parse(MapFile, &DL); MPM->add(createRewriteSymbolsPass(DL)); } static CodeGenOpt::Level getCGOptLevel(const CodeGenOptions &CodeGenOpts) { switch (CodeGenOpts.OptimizationLevel) { default: llvm_unreachable("Invalid optimization level!"); case 0: return CodeGenOpt::None; case 1: return CodeGenOpt::Less; case 2: return CodeGenOpt::Default; // O2/Os/Oz case 3: return CodeGenOpt::Aggressive; } } static Optional getCodeModel(const CodeGenOptions &CodeGenOpts) { unsigned CodeModel = llvm::StringSwitch(CodeGenOpts.CodeModel) .Case("tiny", llvm::CodeModel::Tiny) .Case("small", llvm::CodeModel::Small) .Case("kernel", llvm::CodeModel::Kernel) .Case("medium", llvm::CodeModel::Medium) .Case("large", llvm::CodeModel::Large) .Case("default", ~1u) .Default(~0u); assert(CodeModel != ~0u && "invalid code model!"); if (CodeModel == ~1u) return None; return static_cast(CodeModel); } static TargetMachine::CodeGenFileType getCodeGenFileType(BackendAction Action) { if (Action == Backend_EmitObj) return TargetMachine::CGFT_ObjectFile; else if (Action == Backend_EmitMCNull) return TargetMachine::CGFT_Null; else { assert(Action == Backend_EmitAssembly && "Invalid action!"); return TargetMachine::CGFT_AssemblyFile; } } static void initTargetOptions(llvm::TargetOptions &Options, const CodeGenOptions &CodeGenOpts, const clang::TargetOptions &TargetOpts, const LangOptions &LangOpts, const HeaderSearchOptions &HSOpts) { Options.ThreadModel = llvm::StringSwitch(CodeGenOpts.ThreadModel) .Case("posix", llvm::ThreadModel::POSIX) .Case("single", llvm::ThreadModel::Single); // Set float ABI type. assert((CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp" || CodeGenOpts.FloatABI == "hard" || CodeGenOpts.FloatABI.empty()) && "Invalid Floating Point ABI!"); Options.FloatABIType = llvm::StringSwitch(CodeGenOpts.FloatABI) .Case("soft", llvm::FloatABI::Soft) .Case("softfp", llvm::FloatABI::Soft) .Case("hard", llvm::FloatABI::Hard) .Default(llvm::FloatABI::Default); // Set FP fusion mode. switch (LangOpts.getDefaultFPContractMode()) { case LangOptions::FPC_Off: // Preserve any contraction performed by the front-end. (Strict performs // splitting of the muladd intrinsic in the backend.) Options.AllowFPOpFusion = llvm::FPOpFusion::Standard; break; case LangOptions::FPC_On: Options.AllowFPOpFusion = llvm::FPOpFusion::Standard; break; case LangOptions::FPC_Fast: Options.AllowFPOpFusion = llvm::FPOpFusion::Fast; break; } Options.UseInitArray = CodeGenOpts.UseInitArray; Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS; Options.CompressDebugSections = CodeGenOpts.getCompressDebugSections(); Options.RelaxELFRelocations = CodeGenOpts.RelaxELFRelocations; // Set EABI version. Options.EABIVersion = TargetOpts.EABIVersion; if (LangOpts.SjLjExceptions) Options.ExceptionModel = llvm::ExceptionHandling::SjLj; if (LangOpts.SEHExceptions) Options.ExceptionModel = llvm::ExceptionHandling::WinEH; if (LangOpts.DWARFExceptions) Options.ExceptionModel = llvm::ExceptionHandling::DwarfCFI; Options.NoInfsFPMath = CodeGenOpts.NoInfsFPMath; Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath; Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS; Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath; Options.StackAlignmentOverride = CodeGenOpts.StackAlignment; Options.FunctionSections = CodeGenOpts.FunctionSections; Options.DataSections = CodeGenOpts.DataSections; Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames; Options.EmulatedTLS = CodeGenOpts.EmulatedTLS; Options.ExplicitEmulatedTLS = CodeGenOpts.ExplicitEmulatedTLS; Options.DebuggerTuning = CodeGenOpts.getDebuggerTuning(); Options.EmitStackSizeSection = CodeGenOpts.StackSizeSection; Options.EmitAddrsig = CodeGenOpts.Addrsig; if (CodeGenOpts.getSplitDwarfMode() != CodeGenOptions::NoFission) Options.MCOptions.SplitDwarfFile = CodeGenOpts.SplitDwarfFile; Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll; Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels; Options.MCOptions.MCUseDwarfDirectory = !CodeGenOpts.NoDwarfDirectoryAsm; Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack; Options.MCOptions.MCIncrementalLinkerCompatible = CodeGenOpts.IncrementalLinkerCompatible; Options.MCOptions.MCPIECopyRelocations = CodeGenOpts.PIECopyRelocations; Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings; Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose; Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments; Options.MCOptions.ABIName = TargetOpts.ABI; for (const auto &Entry : HSOpts.UserEntries) if (!Entry.IsFramework && (Entry.Group == frontend::IncludeDirGroup::Quoted || Entry.Group == frontend::IncludeDirGroup::Angled || Entry.Group == frontend::IncludeDirGroup::System)) Options.MCOptions.IASSearchPaths.push_back( Entry.IgnoreSysRoot ? Entry.Path : HSOpts.Sysroot + Entry.Path); } static Optional getGCOVOptions(const CodeGenOptions &CodeGenOpts) { if (CodeGenOpts.DisableGCov) return None; if (!CodeGenOpts.EmitGcovArcs && !CodeGenOpts.EmitGcovNotes) return None; // Not using 'GCOVOptions::getDefault' allows us to avoid exiting if // LLVM's -default-gcov-version flag is set to something invalid. GCOVOptions Options; Options.EmitNotes = CodeGenOpts.EmitGcovNotes; Options.EmitData = CodeGenOpts.EmitGcovArcs; llvm::copy(CodeGenOpts.CoverageVersion, std::begin(Options.Version)); Options.UseCfgChecksum = CodeGenOpts.CoverageExtraChecksum; Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.FunctionNamesInData = !CodeGenOpts.CoverageNoFunctionNamesInData; Options.Filter = CodeGenOpts.ProfileFilterFiles; Options.Exclude = CodeGenOpts.ProfileExcludeFiles; Options.ExitBlockBeforeBody = CodeGenOpts.CoverageExitBlockBeforeBody; return Options; } void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, legacy::FunctionPassManager &FPM) { // Handle disabling of all LLVM passes, where we want to preserve the // internal module before any optimization. if (CodeGenOpts.DisableLLVMPasses) return; // Figure out TargetLibraryInfo. This needs to be added to MPM and FPM // manually (and not via PMBuilder), since some passes (eg. InstrProfiling) // are inserted before PMBuilder ones - they'd get the default-constructed // TLI with an unknown target otherwise. Triple TargetTriple(TheModule->getTargetTriple()); std::unique_ptr TLII( createTLII(TargetTriple, CodeGenOpts)); PassManagerBuilderWrapper PMBuilder(TargetTriple, CodeGenOpts, LangOpts); // At O0 and O1 we only run the always inliner which is more efficient. At // higher optimization levels we run the normal inliner. if (CodeGenOpts.OptimizationLevel <= 1) { bool InsertLifetimeIntrinsics = (CodeGenOpts.OptimizationLevel != 0 && !CodeGenOpts.DisableLifetimeMarkers); PMBuilder.Inliner = createAlwaysInlinerLegacyPass(InsertLifetimeIntrinsics); } else { // We do not want to inline hot callsites for SamplePGO module-summary build // because profile annotation will happen again in ThinLTO backend, and we // want the IR of the hot path to match the profile. PMBuilder.Inliner = createFunctionInliningPass( CodeGenOpts.OptimizationLevel, CodeGenOpts.OptimizeSize, (!CodeGenOpts.SampleProfileFile.empty() && CodeGenOpts.PrepareForThinLTO)); } PMBuilder.OptLevel = CodeGenOpts.OptimizationLevel; PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize; PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP; PMBuilder.LoopVectorize = CodeGenOpts.VectorizeLoop; PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops; PMBuilder.MergeFunctions = CodeGenOpts.MergeFunctions; PMBuilder.PrepareForThinLTO = CodeGenOpts.PrepareForThinLTO; PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO; PMBuilder.RerollLoops = CodeGenOpts.RerollLoops; MPM.add(new TargetLibraryInfoWrapperPass(*TLII)); if (TM) TM->adjustPassManager(PMBuilder); if (CodeGenOpts.DebugInfoForProfiling || !CodeGenOpts.SampleProfileFile.empty()) PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, addAddDiscriminatorsPass); // In ObjC ARC mode, add the main ARC optimization passes. if (LangOpts.ObjCAutoRefCount) { PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, addObjCARCExpandPass); PMBuilder.addExtension(PassManagerBuilder::EP_ModuleOptimizerEarly, addObjCARCAPElimPass); PMBuilder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate, addObjCARCOptPass); } if (LangOpts.CoroutinesTS) addCoroutinePassesToExtensionPoints(PMBuilder); if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) { PMBuilder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate, addBoundsCheckingPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addBoundsCheckingPass); } if (CodeGenOpts.SanitizeCoverageType || CodeGenOpts.SanitizeCoverageIndirectCalls || CodeGenOpts.SanitizeCoverageTraceCmp) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addSanitizerCoveragePass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addSanitizerCoveragePass); } if (LangOpts.Sanitize.has(SanitizerKind::Address)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addAddressSanitizerPasses); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addAddressSanitizerPasses); } if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addKernelAddressSanitizerPasses); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addKernelAddressSanitizerPasses); } if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addHWAddressSanitizerPasses); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addHWAddressSanitizerPasses); } if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addKernelHWAddressSanitizerPasses); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addKernelHWAddressSanitizerPasses); } if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addMemorySanitizerPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addMemorySanitizerPass); } if (LangOpts.Sanitize.has(SanitizerKind::KernelMemory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addKernelMemorySanitizerPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addKernelMemorySanitizerPass); } if (LangOpts.Sanitize.has(SanitizerKind::Thread)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addThreadSanitizerPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addThreadSanitizerPass); } if (LangOpts.Sanitize.has(SanitizerKind::DataFlow)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addDataFlowSanitizerPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addDataFlowSanitizerPass); } if (LangOpts.Sanitize.hasOneOf(SanitizerKind::Efficiency)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addEfficiencySanitizerPass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, addEfficiencySanitizerPass); } // Set up the per-function pass manager. FPM.add(new TargetLibraryInfoWrapperPass(*TLII)); if (CodeGenOpts.VerifyModule) FPM.add(createVerifierPass()); // Set up the per-module pass manager. if (!CodeGenOpts.RewriteMapFiles.empty()) addSymbolRewriterPass(CodeGenOpts, &MPM); if (Optional Options = getGCOVOptions(CodeGenOpts)) { MPM.add(createGCOVProfilerPass(*Options)); if (CodeGenOpts.getDebugInfo() == codegenoptions::NoDebugInfo) MPM.add(createStripSymbolsPass(true)); } if (CodeGenOpts.hasProfileClangInstr()) { InstrProfOptions Options; Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput; // TODO: Surface the option to emit atomic profile counter increments at // the driver level. Options.Atomic = LangOpts.Sanitize.has(SanitizerKind::Thread); MPM.add(createInstrProfilingLegacyPass(Options)); } if (CodeGenOpts.hasProfileIRInstr()) { PMBuilder.EnablePGOInstrGen = true; if (!CodeGenOpts.InstrProfileOutput.empty()) PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput; else PMBuilder.PGOInstrGen = DefaultProfileGenName; } if (CodeGenOpts.hasProfileIRUse()) PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath; if (!CodeGenOpts.SampleProfileFile.empty()) PMBuilder.PGOSampleUse = CodeGenOpts.SampleProfileFile; PMBuilder.populateFunctionPassManager(FPM); PMBuilder.populateModulePassManager(MPM); } static void setCommandLineOpts(const CodeGenOptions &CodeGenOpts) { SmallVector BackendArgs; BackendArgs.push_back("clang"); // Fake program name. if (!CodeGenOpts.DebugPass.empty()) { BackendArgs.push_back("-debug-pass"); BackendArgs.push_back(CodeGenOpts.DebugPass.c_str()); } if (!CodeGenOpts.LimitFloatPrecision.empty()) { BackendArgs.push_back("-limit-float-precision"); BackendArgs.push_back(CodeGenOpts.LimitFloatPrecision.c_str()); } BackendArgs.push_back(nullptr); llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1, BackendArgs.data()); } void EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { // Create the TargetMachine for generating code. std::string Error; std::string Triple = TheModule->getTargetTriple(); const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error); if (!TheTarget) { if (MustCreateTM) Diags.Report(diag::err_fe_unable_to_create_target) << Error; return; } Optional CM = getCodeModel(CodeGenOpts); std::string FeaturesStr = llvm::join(TargetOpts.Features.begin(), TargetOpts.Features.end(), ","); llvm::Reloc::Model RM = CodeGenOpts.RelocationModel; CodeGenOpt::Level OptLevel = getCGOptLevel(CodeGenOpts); llvm::TargetOptions Options; initTargetOptions(Options, CodeGenOpts, TargetOpts, LangOpts, HSOpts); TM.reset(TheTarget->createTargetMachine(Triple, TargetOpts.CPU, FeaturesStr, Options, RM, CM, OptLevel)); } bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses, BackendAction Action, raw_pwrite_stream &OS, raw_pwrite_stream *DwoOS) { // Add LibraryInfo. llvm::Triple TargetTriple(TheModule->getTargetTriple()); std::unique_ptr TLII( createTLII(TargetTriple, CodeGenOpts)); CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII)); // Normal mode, emit a .s or .o file by running the code generator. Note, // this also adds codegenerator level optimization passes. TargetMachine::CodeGenFileType CGFT = getCodeGenFileType(Action); // Add ObjC ARC final-cleanup optimizations. This is done as part of the // "codegen" passes so that it isn't run multiple times when there is // inlining happening. if (CodeGenOpts.OptimizationLevel > 0) CodeGenPasses.add(createObjCARCContractPass()); if (TM->addPassesToEmitFile(CodeGenPasses, OS, DwoOS, CGFT, /*DisableVerify=*/!CodeGenOpts.VerifyModule)) { Diags.Report(diag::err_fe_unable_to_interface_with_target); return false; } return true; } void EmitAssemblyHelper::EmitAssembly(BackendAction Action, std::unique_ptr OS) { TimeRegion Region(FrontendTimesIsEnabled ? &CodeGenerationTime : nullptr); setCommandLineOpts(CodeGenOpts); bool UsesCodeGen = (Action != Backend_EmitNothing && Action != Backend_EmitBC && Action != Backend_EmitLL); CreateTargetMachine(UsesCodeGen); if (UsesCodeGen && !TM) return; if (TM) TheModule->setDataLayout(TM->createDataLayout()); legacy::PassManager PerModulePasses; PerModulePasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); legacy::FunctionPassManager PerFunctionPasses(TheModule); PerFunctionPasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); CreatePasses(PerModulePasses, PerFunctionPasses); legacy::PassManager CodeGenPasses; CodeGenPasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); std::unique_ptr ThinLinkOS, DwoOS; switch (Action) { case Backend_EmitNothing: break; case Backend_EmitBC: if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) { if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) { ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile); if (!ThinLinkOS) return; } TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", CodeGenOpts.EnableSplitLTOUnit); PerModulePasses.add(createWriteThinLTOBitcodePass( *OS, ThinLinkOS ? &ThinLinkOS->os() : nullptr)); } else { // Emit a module summary by default for Regular LTO except for ld64 // targets bool EmitLTOSummary = (CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses && llvm::Triple(TheModule->getTargetTriple()).getVendor() != llvm::Triple::Apple); if (EmitLTOSummary) { if (!TheModule->getModuleFlag("ThinLTO")) TheModule->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", CodeGenOpts.EnableSplitLTOUnit); } PerModulePasses.add(createBitcodeWriterPass( *OS, CodeGenOpts.EmitLLVMUseLists, EmitLTOSummary)); } break; case Backend_EmitLL: PerModulePasses.add( createPrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); break; default: if (!CodeGenOpts.SplitDwarfFile.empty() && (CodeGenOpts.getSplitDwarfMode() == CodeGenOptions::SplitFileFission)) { DwoOS = openOutputFile(CodeGenOpts.SplitDwarfFile); if (!DwoOS) return; } if (!AddEmitPasses(CodeGenPasses, Action, *OS, DwoOS ? &DwoOS->os() : nullptr)) return; } // Before executing passes, print the final values of the LLVM options. cl::PrintOptionValues(); // Run passes. For now we do all passes at once, but eventually we // would like to have the option of streaming code generation. { PrettyStackTraceString CrashInfo("Per-function optimization"); PerFunctionPasses.doInitialization(); for (Function &F : *TheModule) if (!F.isDeclaration()) PerFunctionPasses.run(F); PerFunctionPasses.doFinalization(); } { PrettyStackTraceString CrashInfo("Per-module optimization passes"); PerModulePasses.run(*TheModule); } { PrettyStackTraceString CrashInfo("Code generation"); CodeGenPasses.run(*TheModule); } if (ThinLinkOS) ThinLinkOS->keep(); if (DwoOS) DwoOS->keep(); } static PassBuilder::OptimizationLevel mapToLevel(const CodeGenOptions &Opts) { switch (Opts.OptimizationLevel) { default: llvm_unreachable("Invalid optimization level!"); case 1: return PassBuilder::O1; case 2: switch (Opts.OptimizeSize) { default: llvm_unreachable("Invalid optimization level for size!"); case 0: return PassBuilder::O2; case 1: return PassBuilder::Os; case 2: return PassBuilder::Oz; } case 3: return PassBuilder::O3; } } /// A clean version of `EmitAssembly` that uses the new pass manager. /// /// Not all features are currently supported in this system, but where /// necessary it falls back to the legacy pass manager to at least provide /// basic functionality. /// /// This API is planned to have its functionality finished and then to replace /// `EmitAssembly` at some point in the future when the default switches. void EmitAssemblyHelper::EmitAssemblyWithNewPassManager( BackendAction Action, std::unique_ptr OS) { TimeRegion Region(FrontendTimesIsEnabled ? &CodeGenerationTime : nullptr); setCommandLineOpts(CodeGenOpts); // The new pass manager always makes a target machine available to passes // during construction. CreateTargetMachine(/*MustCreateTM*/ true); if (!TM) // This will already be diagnosed, just bail. return; TheModule->setDataLayout(TM->createDataLayout()); Optional PGOOpt; if (CodeGenOpts.hasProfileIRInstr()) // -fprofile-generate. PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() ? DefaultProfileGenName : CodeGenOpts.InstrProfileOutput, "", "", "", true, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.hasProfileIRUse()) // -fprofile-use. PGOOpt = PGOOptions("", CodeGenOpts.ProfileInstrumentUsePath, "", CodeGenOpts.ProfileRemappingFile, false, CodeGenOpts.DebugInfoForProfiling); else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use PGOOpt = PGOOptions("", "", CodeGenOpts.SampleProfileFile, CodeGenOpts.ProfileRemappingFile, false, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling PGOOpt = PGOOptions("", "", "", "", false, true); PassBuilder PB(TM.get(), PGOOpt); LoopAnalysisManager LAM(CodeGenOpts.DebugPassManager); FunctionAnalysisManager FAM(CodeGenOpts.DebugPassManager); CGSCCAnalysisManager CGAM(CodeGenOpts.DebugPassManager); ModuleAnalysisManager MAM(CodeGenOpts.DebugPassManager); // Register the AA manager first so that our version is the one used. FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); // Register the target library analysis directly and give it a customized // preset TLI. Triple TargetTriple(TheModule->getTargetTriple()); std::unique_ptr TLII( createTLII(TargetTriple, CodeGenOpts)); FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); MAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); // Register all the basic analyses with the managers. PB.registerModuleAnalyses(MAM); PB.registerCGSCCAnalyses(CGAM); PB.registerFunctionAnalyses(FAM); PB.registerLoopAnalyses(LAM); PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); ModulePassManager MPM(CodeGenOpts.DebugPassManager); if (!CodeGenOpts.DisableLLVMPasses) { bool IsThinLTO = CodeGenOpts.PrepareForThinLTO; bool IsLTO = CodeGenOpts.PrepareForLTO; if (CodeGenOpts.OptimizationLevel == 0) { if (Optional Options = getGCOVOptions(CodeGenOpts)) MPM.addPass(GCOVProfilerPass(*Options)); // Build a minimal pipeline based on the semantics required by Clang, // which is just that always inlining occurs. MPM.addPass(AlwaysInlinerPass()); // At -O0 we directly run necessary sanitizer passes. if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) MPM.addPass(createModuleToFunctionPassAdaptor(BoundsCheckingPass())); // Lastly, add semantically necessary passes for LTO. if (IsLTO || IsThinLTO) { MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); } } else { // Map our optimization levels into one of the distinct levels used to // configure the pipeline. PassBuilder::OptimizationLevel Level = mapToLevel(CodeGenOpts); // Register callbacks to schedule sanitizer passes at the appropriate part of // the pipeline. if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) PB.registerScalarOptimizerLateEPCallback( [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { FPM.addPass(BoundsCheckingPass()); }); if (Optional Options = getGCOVOptions(CodeGenOpts)) PB.registerPipelineStartEPCallback([Options](ModulePassManager &MPM) { MPM.addPass(GCOVProfilerPass(*Options)); }); if (IsThinLTO) { MPM = PB.buildThinLTOPreLinkDefaultPipeline( Level, CodeGenOpts.DebugPassManager); MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); } else if (IsLTO) { MPM = PB.buildLTOPreLinkDefaultPipeline(Level, CodeGenOpts.DebugPassManager); MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); } else { MPM = PB.buildPerModuleDefaultPipeline(Level, CodeGenOpts.DebugPassManager); } } } // FIXME: We still use the legacy pass manager to do code generation. We // create that pass manager here and use it as needed below. legacy::PassManager CodeGenPasses; bool NeedCodeGen = false; std::unique_ptr ThinLinkOS, DwoOS; // Append any output we need to the pass manager. switch (Action) { case Backend_EmitNothing: break; case Backend_EmitBC: if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) { if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) { ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile); if (!ThinLinkOS) return; } TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", CodeGenOpts.EnableSplitLTOUnit); MPM.addPass(ThinLTOBitcodeWriterPass(*OS, ThinLinkOS ? &ThinLinkOS->os() : nullptr)); } else { // Emit a module summary by default for Regular LTO except for ld64 // targets bool EmitLTOSummary = (CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses && llvm::Triple(TheModule->getTargetTriple()).getVendor() != llvm::Triple::Apple); if (EmitLTOSummary) { if (!TheModule->getModuleFlag("ThinLTO")) TheModule->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", CodeGenOpts.EnableSplitLTOUnit); } MPM.addPass( BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists, EmitLTOSummary)); } break; case Backend_EmitLL: MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); break; case Backend_EmitAssembly: case Backend_EmitMCNull: case Backend_EmitObj: NeedCodeGen = true; CodeGenPasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); if (!CodeGenOpts.SplitDwarfFile.empty()) { DwoOS = openOutputFile(CodeGenOpts.SplitDwarfFile); if (!DwoOS) return; } if (!AddEmitPasses(CodeGenPasses, Action, *OS, DwoOS ? &DwoOS->os() : nullptr)) // FIXME: Should we handle this error differently? return; break; } // Before executing passes, print the final values of the LLVM options. cl::PrintOptionValues(); // Now that we have all of the passes ready, run them. { PrettyStackTraceString CrashInfo("Optimizer"); MPM.run(*TheModule, MAM); } // Now if needed, run the legacy PM for codegen. if (NeedCodeGen) { PrettyStackTraceString CrashInfo("Code generation"); CodeGenPasses.run(*TheModule); } if (ThinLinkOS) ThinLinkOS->keep(); if (DwoOS) DwoOS->keep(); } Expected clang::FindThinLTOModule(MemoryBufferRef MBRef) { Expected> BMsOrErr = getBitcodeModuleList(MBRef); if (!BMsOrErr) return BMsOrErr.takeError(); // The bitcode file may contain multiple modules, we want the one that is // marked as being the ThinLTO module. if (const BitcodeModule *Bm = FindThinLTOModule(*BMsOrErr)) return *Bm; return make_error("Could not find module summary", inconvertibleErrorCode()); } BitcodeModule *clang::FindThinLTOModule(MutableArrayRef BMs) { for (BitcodeModule &BM : BMs) { Expected LTOInfo = BM.getLTOInfo(); if (LTOInfo && LTOInfo->IsThinLTO) return &BM; } return nullptr; } static void runThinLTOBackend(ModuleSummaryIndex *CombinedIndex, Module *M, const HeaderSearchOptions &HeaderOpts, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, const LangOptions &LOpts, std::unique_ptr OS, std::string SampleProfile, std::string ProfileRemapping, BackendAction Action) { StringMap> ModuleToDefinedGVSummaries; CombinedIndex->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); setCommandLineOpts(CGOpts); // We can simply import the values mentioned in the combined index, since // we should only invoke this using the individual indexes written out // via a WriteIndexesThinBackend. FunctionImporter::ImportMapTy ImportList; for (auto &GlobalList : *CombinedIndex) { // Ignore entries for undefined references. if (GlobalList.second.SummaryList.empty()) continue; auto GUID = GlobalList.first; for (auto &Summary : GlobalList.second.SummaryList) { // Skip the summaries for the importing module. These are included to // e.g. record required linkage changes. if (Summary->modulePath() == M->getModuleIdentifier()) continue; // Add an entry to provoke importing by thinBackend. ImportList[Summary->modulePath()].insert(GUID); } } std::vector> OwnedImports; MapVector ModuleMap; for (auto &I : ImportList) { ErrorOr> MBOrErr = llvm::MemoryBuffer::getFile(I.first()); if (!MBOrErr) { errs() << "Error loading imported file '" << I.first() << "': " << MBOrErr.getError().message() << "\n"; return; } Expected BMOrErr = FindThinLTOModule(**MBOrErr); if (!BMOrErr) { handleAllErrors(BMOrErr.takeError(), [&](ErrorInfoBase &EIB) { errs() << "Error loading imported file '" << I.first() << "': " << EIB.message() << '\n'; }); return; } ModuleMap.insert({I.first(), *BMOrErr}); OwnedImports.push_back(std::move(*MBOrErr)); } auto AddStream = [&](size_t Task) { return llvm::make_unique(std::move(OS)); }; lto::Config Conf; if (CGOpts.SaveTempsFilePrefix != "") { if (Error E = Conf.addSaveTemps(CGOpts.SaveTempsFilePrefix + ".", /* UseInputModulePath */ false)) { handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { errs() << "Error setting up ThinLTO save-temps: " << EIB.message() << '\n'; }); } } Conf.CPU = TOpts.CPU; Conf.CodeModel = getCodeModel(CGOpts); Conf.MAttrs = TOpts.Features; Conf.RelocModel = CGOpts.RelocationModel; Conf.CGOptLevel = getCGOptLevel(CGOpts); initTargetOptions(Conf.Options, CGOpts, TOpts, LOpts, HeaderOpts); Conf.SampleProfile = std::move(SampleProfile); Conf.ProfileRemapping = std::move(ProfileRemapping); Conf.UseNewPM = CGOpts.ExperimentalNewPassManager; Conf.DebugPassManager = CGOpts.DebugPassManager; Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness; Conf.RemarksFilename = CGOpts.OptRecordFile; Conf.DwoPath = CGOpts.SplitDwarfFile; switch (Action) { case Backend_EmitNothing: Conf.PreCodeGenModuleHook = [](size_t Task, const Module &Mod) { return false; }; break; case Backend_EmitLL: Conf.PreCodeGenModuleHook = [&](size_t Task, const Module &Mod) { M->print(*OS, nullptr, CGOpts.EmitLLVMUseLists); return false; }; break; case Backend_EmitBC: Conf.PreCodeGenModuleHook = [&](size_t Task, const Module &Mod) { WriteBitcodeToFile(*M, *OS, CGOpts.EmitLLVMUseLists); return false; }; break; default: Conf.CGFileType = getCodeGenFileType(Action); break; } if (Error E = thinBackend( Conf, -1, AddStream, *M, *CombinedIndex, ImportList, ModuleToDefinedGVSummaries[M->getModuleIdentifier()], ModuleMap)) { handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { errs() << "Error running ThinLTO backend: " << EIB.message() << '\n'; }); } } void clang::EmitBackendOutput(DiagnosticsEngine &Diags, const HeaderSearchOptions &HeaderOpts, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, const LangOptions &LOpts, const llvm::DataLayout &TDesc, Module *M, BackendAction Action, std::unique_ptr OS) { std::unique_ptr EmptyModule; if (!CGOpts.ThinLTOIndexFile.empty()) { // If we are performing a ThinLTO importing compile, load the function index // into memory and pass it into runThinLTOBackend, which will run the // function importer and invoke LTO passes. Expected> IndexOrErr = llvm::getModuleSummaryIndexForFile(CGOpts.ThinLTOIndexFile, /*IgnoreEmptyThinLTOIndexFile*/true); if (!IndexOrErr) { logAllUnhandledErrors(IndexOrErr.takeError(), errs(), "Error loading index file '" + CGOpts.ThinLTOIndexFile + "': "); return; } std::unique_ptr CombinedIndex = std::move(*IndexOrErr); // A null CombinedIndex means we should skip ThinLTO compilation // (LLVM will optionally ignore empty index files, returning null instead // of an error). if (CombinedIndex) { if (!CombinedIndex->skipModuleByDistributedBackend()) { runThinLTOBackend(CombinedIndex.get(), M, HeaderOpts, CGOpts, TOpts, LOpts, std::move(OS), CGOpts.SampleProfileFile, CGOpts.ProfileRemappingFile, Action); return; } // Distributed indexing detected that nothing from the module is needed // for the final linking. So we can skip the compilation. We sill need to // output an empty object file to make sure that a linker does not fail // trying to read it. Also for some features, like CFI, we must skip // the compilation as CombinedIndex does not contain all required // information. EmptyModule = llvm::make_unique("empty", M->getContext()); EmptyModule->setTargetTriple(M->getTargetTriple()); M = EmptyModule.get(); } } EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M); if (CGOpts.ExperimentalNewPassManager) AsmHelper.EmitAssemblyWithNewPassManager(Action, std::move(OS)); else AsmHelper.EmitAssembly(Action, std::move(OS)); // Verify clang's TargetInfo DataLayout against the LLVM TargetMachine's // DataLayout. if (AsmHelper.TM) { std::string DLDesc = M->getDataLayout().getStringRepresentation(); if (DLDesc != TDesc.getStringRepresentation()) { unsigned DiagID = Diags.getCustomDiagID( DiagnosticsEngine::Error, "backend data layout '%0' does not match " "expected target description '%1'"); Diags.Report(DiagID) << DLDesc << TDesc.getStringRepresentation(); } } } static const char* getSectionNameForBitcode(const Triple &T) { switch (T.getObjectFormat()) { case Triple::MachO: return "__LLVM,__bitcode"; case Triple::COFF: case Triple::ELF: case Triple::Wasm: case Triple::UnknownObjectFormat: return ".llvmbc"; } llvm_unreachable("Unimplemented ObjectFormatType"); } static const char* getSectionNameForCommandline(const Triple &T) { switch (T.getObjectFormat()) { case Triple::MachO: return "__LLVM,__cmdline"; case Triple::COFF: case Triple::ELF: case Triple::Wasm: case Triple::UnknownObjectFormat: return ".llvmcmd"; } llvm_unreachable("Unimplemented ObjectFormatType"); } // With -fembed-bitcode, save a copy of the llvm IR as data in the // __LLVM,__bitcode section. void clang::EmbedBitcode(llvm::Module *M, const CodeGenOptions &CGOpts, llvm::MemoryBufferRef Buf) { if (CGOpts.getEmbedBitcode() == CodeGenOptions::Embed_Off) return; // Save llvm.compiler.used and remote it. SmallVector UsedArray; SmallPtrSet UsedGlobals; Type *UsedElementType = Type::getInt8Ty(M->getContext())->getPointerTo(0); GlobalVariable *Used = collectUsedGlobalVariables(*M, UsedGlobals, true); for (auto *GV : UsedGlobals) { if (GV->getName() != "llvm.embedded.module" && GV->getName() != "llvm.cmdline") UsedArray.push_back( ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType)); } if (Used) Used->eraseFromParent(); // Embed the bitcode for the llvm module. std::string Data; ArrayRef ModuleData; Triple T(M->getTargetTriple()); // Create a constant that contains the bitcode. // In case of embedding a marker, ignore the input Buf and use the empty // ArrayRef. It is also legal to create a bitcode marker even Buf is empty. if (CGOpts.getEmbedBitcode() != CodeGenOptions::Embed_Marker) { if (!isBitcode((const unsigned char *)Buf.getBufferStart(), (const unsigned char *)Buf.getBufferEnd())) { // If the input is LLVM Assembly, bitcode is produced by serializing // the module. Use-lists order need to be perserved in this case. llvm::raw_string_ostream OS(Data); llvm::WriteBitcodeToFile(*M, OS, /* ShouldPreserveUseListOrder */ true); ModuleData = ArrayRef((const uint8_t *)OS.str().data(), OS.str().size()); } else // If the input is LLVM bitcode, write the input byte stream directly. ModuleData = ArrayRef((const uint8_t *)Buf.getBufferStart(), Buf.getBufferSize()); } llvm::Constant *ModuleConstant = llvm::ConstantDataArray::get(M->getContext(), ModuleData); llvm::GlobalVariable *GV = new llvm::GlobalVariable( *M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage, ModuleConstant); GV->setSection(getSectionNameForBitcode(T)); UsedArray.push_back( ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType)); if (llvm::GlobalVariable *Old = M->getGlobalVariable("llvm.embedded.module", true)) { assert(Old->hasOneUse() && "llvm.embedded.module can only be used once in llvm.compiler.used"); GV->takeName(Old); Old->eraseFromParent(); } else { GV->setName("llvm.embedded.module"); } // Skip if only bitcode needs to be embedded. if (CGOpts.getEmbedBitcode() != CodeGenOptions::Embed_Bitcode) { // Embed command-line options. ArrayRef CmdData(const_cast(CGOpts.CmdArgs.data()), CGOpts.CmdArgs.size()); llvm::Constant *CmdConstant = llvm::ConstantDataArray::get(M->getContext(), CmdData); GV = new llvm::GlobalVariable(*M, CmdConstant->getType(), true, llvm::GlobalValue::PrivateLinkage, CmdConstant); GV->setSection(getSectionNameForCommandline(T)); UsedArray.push_back( ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType)); if (llvm::GlobalVariable *Old = M->getGlobalVariable("llvm.cmdline", true)) { assert(Old->hasOneUse() && "llvm.cmdline can only be used once in llvm.compiler.used"); GV->takeName(Old); Old->eraseFromParent(); } else { GV->setName("llvm.cmdline"); } } if (UsedArray.empty()) return; // Recreate llvm.compiler.used. ArrayType *ATy = ArrayType::get(UsedElementType, UsedArray.size()); auto *NewUsed = new GlobalVariable( *M, ATy, false, llvm::GlobalValue::AppendingLinkage, llvm::ConstantArray::get(ATy, UsedArray), "llvm.compiler.used"); NewUsed->setSection("llvm.metadata"); }