Index: include/llvm/ExecutionEngine/Orc/Core.h =================================================================== --- include/llvm/ExecutionEngine/Orc/Core.h +++ include/llvm/ExecutionEngine/Orc/Core.h @@ -223,6 +223,9 @@ /// Return the set of symbols that this source provides. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Return mutable set of symbols that this source provides. + SymbolFlagsMap &editSymbols() { return SymbolFlags; } + /// Called by materialization dispatchers (see /// ExecutionSession::DispatchMaterializationFunction) to trigger /// materialization of this MaterializationUnit. @@ -371,6 +374,9 @@ // FIXME: Remove this when we remove the old ORC layers. friend class JITDylib; +protected: + virtual ~ExecutionSessionBase() {} + public: /// For reporting errors. using ErrorReporter = std::function; @@ -462,10 +468,10 @@ /// or an error occurs. If WaitUntilReady is false and an error occurs /// after resolution, the function will return a success value, but the /// error will be reported via reportErrors. - Expected lookup(const JITDylibList &JDs, - const SymbolNameSet &Symbols, - RegisterDependenciesFunction RegisterDependencies, - bool WaitUntilReady = true); + virtual Expected + lookup(const JITDylibList &JDs, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, + bool WaitUntilReady = true); /// Materialize the given unit. void dispatchMaterialization(JITDylib &JD, Index: include/llvm/ExecutionEngine/Orc/Layer.h =================================================================== --- include/llvm/ExecutionEngine/Orc/Layer.h +++ include/llvm/ExecutionEngine/Orc/Layer.h @@ -132,6 +132,25 @@ Expected getObjectSymbolFlags(ExecutionSession &ES, MemoryBufferRef ObjBuffer); +/// Interface for layers that accept file paths. +class FileLayer { +public: + FileLayer(ExecutionSession &ES); + virtual ~FileLayer(); + + /// Returns the ExecutionSession for this layer. + ExecutionSession &getExecutionSession() { return ES; } + + /// Add prefix paths for searching Module files. + void addSearchPath(std::string Dir) { SearchPaths.push_back(std::move(Dir)); } + +protected: + Expected findFullModulePath(std::string RelPath) const; + +private: + ExecutionSession &ES; + std::vector SearchPaths; +}; } // End namespace orc } // End namespace llvm Index: include/llvm/ExecutionEngine/Orc/ThinLTOLayer.h =================================================================== --- /dev/null +++ include/llvm/ExecutionEngine/Orc/ThinLTOLayer.h @@ -0,0 +1,174 @@ +//===- ThinLTOLayer.h - ThinLTO Module Summaries for JITing -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_THINLTOLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_THINLTOLAYER_H + +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO/FunctionImport.h" + +#include +#include +#include + +namespace llvm { +namespace orc { + +using FnSetTy = FunctionImporter::FunctionsToImportTy; +using FnSetByModuleTy = FunctionImporter::ImportMapTy; +using FnSummaryMapTy = DenseMap; +using GetAvailableContextFunction = std::function; + +//===----------------------------------------------------------------------===// + +class ThinLTOLayer2; + +/// An ExecutionSession represents a running JIT program. This one broadcasts +/// symbol names to collect relevant JITDylibs from all registered providers on +/// lookup. +class ThinLTOExecutionSession : public ExecutionSession { +public: + // TODO Add an actual abstraction? + using JITDylibProvider = ThinLTOLayer2; + + ThinLTOExecutionSession(std::shared_ptr SSP = nullptr) + : ExecutionSession(std::move(SSP)) {} + + // TODO Requires ExecutionSessionBase::lookup() to be virtual. + Expected lookup(const JITDylibList &JDs, + const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, + bool WaitUntilReady = true) override; + + void addJITDylibProvider(JITDylibProvider *Provider) { + std::lock_guard Lock(JITDylibsLock); + JITDylibProviders.insert(Provider); + } + + void removeJITDylibProvider(JITDylibProvider *Provider) { + std::lock_guard Lock(JITDylibsLock); + JITDylibProviders.erase(Provider); + } + +private: + std::unordered_set JITDylibProviders; + std::mutex JITDylibsLock; +}; + +//===----------------------------------------------------------------------===// + +class ThinLTOLayer2 : public FileLayer { + constexpr static bool PerformingIRModuleAnalysis = false; + + struct StagedModule { + JITDylib *TargetDylib; + std::string Path; + }; + +public: + ThinLTOLayer2(ThinLTOExecutionSession &ES, const DataLayout &DL, + IRLayer &CompileLayer, GetAvailableContextFunction GetContext) + : FileLayer(ES), BaseLayer(CompileLayer), + CombinedIndex(PerformingIRModuleAnalysis), + GetContextForModule(GetContext), MandI(ES, DL) { + ES.addJITDylibProvider(this); + } + + ~ThinLTOLayer2() override { + auto &ES = static_cast(getExecutionSession()); + ES.removeJITDylibProvider(this); + } + + /// Called from client code. + Expected add(JITDylib &JD, std::string Path); + + /// Called from ThinLTOLayerMaterializationUnit. + void emit(MaterializationResponsibility R, const VModuleKey &K, + StringRef Path, const GVSummaryMapTy &AllDefinedFns, + const FnSetTy &DiscoveredGuids); + + /// Called from ThinLTOExecutionSession. + JITDylibList matchDylibs(const SymbolNameSet &Symbols, + ExecutionSession::ErrorReporter Report = nullptr); + +private: + IRLayer &BaseLayer; + ModuleSummaryIndex CombinedIndex; + std::map PendingModules; + JITDylibList SubmittedDylibs; + GetAvailableContextFunction GetContextForModule; + MangleAndInterner MandI; + std::mutex JITDylibsLock; + + /// Find modules and GUIDs for all callees from the given DiscoveryRoots. + /// DefinedGVSummaries has summaries for all entities defined in the module at + /// Path. DiscoveryRoots is a subset of functions. + FnSetByModuleTy discover(StringRef Path, const FnSummaryMapTy &DiscoveryRoots, + const GVSummaryMapTy &DefinedGVSummaries); + + /// Prepare the module at Path for materialization. At this point we haven't + /// parsed the module's actual IR code, so we still don't know the real names + /// of its symbols. Instead the MaterializationUnit stores only GUIDs. Real + /// names will replace them once the symbols are looked up. + /// + /// TODO Currently requires a few hacks in JITDylib::lodgeQueryImpl. + /// TODO Instead of actual names could we even resolve GUIDs only? Everywhere? + Error stage(std::string Path, FnSetTy Roots); + + /// Parse a module's actual IR code. + Expected> parseModule(VModuleKey Key, StringRef Path); +}; + +//===----------------------------------------------------------------------===// + +class ThinLTOLayerMaterializationUnit : public MaterializationUnit { +public: + static std::unique_ptr + Create(ThinLTOLayer2 &L, VModuleKey K, std::string Path, + GVSummaryMapTy AllDefinedFns, FnSetTy PotentialCallees, + ExecutionSession &ES); + +private: + ThinLTOLayer2 &L; + VModuleKey K; + std::string Path; + GVSummaryMapTy AllDefinedFns; + FnSetTy PotentialCallees; + + ThinLTOLayerMaterializationUnit(ThinLTOLayer2 &L, VModuleKey K, + std::string Path, SymbolFlagsMap SymbolFlags, + GVSummaryMapTy AllDefinedFns, + FnSetTy PotentialCallees) + : MaterializationUnit(std::move(SymbolFlags)), L(L), K(std::move(K)), + Path(std::move(Path)), AllDefinedFns(std::move(AllDefinedFns)), + PotentialCallees(std::move(PotentialCallees)) {} + + /// IIUC data members shouldn't be moved into emit(), because emission may + /// fail this time and materialization tried again later. + void materialize(MaterializationResponsibility R) override { + L.emit(std::move(R), K, Path, AllDefinedFns, PotentialCallees); + } + + void discard(const JITDylib &JD, SymbolStringPtr Name) override { + // TODO Discard the given symbol + assert(false); + }; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_THINLTOLAYER_H Index: lib/ExecutionEngine/Orc/CMakeLists.txt =================================================================== --- lib/ExecutionEngine/Orc/CMakeLists.txt +++ lib/ExecutionEngine/Orc/CMakeLists.txt @@ -8,6 +8,7 @@ Legacy.cpp Layer.cpp LLJIT.cpp + ThinLTOLayer.cpp NullResolver.cpp ObjectTransformLayer.cpp OrcABISupport.cpp @@ -28,4 +29,5 @@ PRIVATE LLVMBitReader LLVMBitWriter + LLVMipo ) Index: lib/ExecutionEngine/Orc/Core.cpp =================================================================== --- lib/ExecutionEngine/Orc/Core.cpp +++ lib/ExecutionEngine/Orc/Core.cpp @@ -42,6 +42,11 @@ if (!Flags.isExported()) OS << "[Hidden]"; + if (Flags.isLazy()) + OS << "[Lazy]"; + if (Flags.isMaterializing()) + OS << "[Materializing]"; + return OS; } @@ -606,7 +611,7 @@ void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { #ifndef NDEBUG - for (auto &KV : Symbols) { + for (const auto &KV : Symbols) { auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); @@ -1316,6 +1321,36 @@ } } +static StringRef unmangled(SymbolStringPtr SymbolName) { + StringRef PoolStr = *SymbolName; + if (PoolStr.startswith("_")) + return PoolStr.substr(1); + + // TODO Consider other linker manglings. + return PoolStr; +} + +template +static typename std::map::iterator +hackReplaceGuidWithNameIn(std::map &Map, + SymbolStringPtr SymName, ExecutionSessionBase &ES) { + StringRef FunctionName = unmangled(SymName); + auto Guid = GlobalValue::getGUID(FunctionName); + auto GuidStrPtr = ES.getSymbolStringPool().intern(std::to_string(Guid)); + + auto It = Map.find(GuidStrPtr); + if (It != Map.end()) { + // Add the actual name to the map and remove the placeholder GUID + auto PairNewItSuccess = Map.emplace(SymName, It->second); + assert(PairNewItSuccess.second && "Name must not exist"); + + Map.erase(It); + It = PairNewItSuccess.first; + } + + return It; +} + void JITDylib::lodgeQueryImpl( std::shared_ptr &Q, SymbolNameSet &Unresolved, std::vector> &MUs) { @@ -1325,8 +1360,14 @@ // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - continue; + if (SymI == Symbols.end()) { + // FIXME ThinLTO Hack + SymI = hackReplaceGuidWithNameIn(Symbols, Name, getExecutionSession()); + + // Skip it if neither name nor GUID found. + if (SymI == Symbols.end()) + continue; + } // If we found Name in JD, remove it frome the Unresolved set and add it // to the added set. @@ -1336,21 +1377,30 @@ if (SymI->second.getAddress() != 0) Q->resolve(Name, SymI->second); - // If the symbol is lazy, get the MaterialiaztionUnit for it. + // If the symbol is lazy, get the MaterializationUnit for it. if (SymI->second.getFlags().isLazy()) { assert(SymI->second.getAddress() == 0 && "Lazy symbol should not have a resolved address"); assert(!SymI->second.getFlags().isMaterializing() && "Materializing and lazy should not both be set"); auto UMII = UnmaterializedInfos.find(Name); + + if (UMII == UnmaterializedInfos.end()) + UMII = hackReplaceGuidWithNameIn(UnmaterializedInfos, Name, + getExecutionSession()); + assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); auto MU = std::move(UMII->second->MU); assert(MU != nullptr && "Materializer should not be null"); + if (MU->getSymbols().find(Name) == MU->getSymbols().end()) + hackReplaceGuidWithNameIn(MU->editSymbols(), Name, + getExecutionSession()); + // Move all symbols associated with this MaterializationUnit into // materializing state. - for (auto &KV : MU->getSymbols()) { + for (const auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); auto Flags = SymK->second.getFlags(); Flags &= ~JITSymbolFlags::Lazy; Index: lib/ExecutionEngine/Orc/Layer.cpp =================================================================== --- lib/ExecutionEngine/Orc/Layer.cpp +++ lib/ExecutionEngine/Orc/Layer.cpp @@ -8,7 +8,10 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Layer.h" + +#include "llvm/ADT/StringExtras.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Support/FileSystem.h" namespace llvm { namespace orc { @@ -136,5 +139,25 @@ return SymbolFlags; } +FileLayer::FileLayer(ExecutionSession &ES) : ES(ES) {} + +FileLayer::~FileLayer() {} + +Expected FileLayer::findFullModulePath(std::string RelPath) const { + if (sys::fs::exists(RelPath)) + return RelPath; + + for (const std::string &SearchPath : SearchPaths) { + std::string FullPath = SearchPath + RelPath; + if (sys::fs::exists(FullPath)) + return FullPath; + } + + auto SearchPathsStr = join(SearchPaths.begin(), SearchPaths.end(), ", "); + return createStringError(inconvertibleErrorCode(), + "Can't find file %s.\nModule search paths are: %s", + RelPath.c_str(), SearchPathsStr.c_str()); +} + } // End namespace orc. } // End namespace llvm. Index: lib/ExecutionEngine/Orc/ThinLTOLayer.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/Orc/ThinLTOLayer.cpp @@ -0,0 +1,422 @@ +//===-------- ThinLTOLayer.cpp - ThinLTO Module Summaries for JITing ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ThinLTOLayer.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "orc-thinlto-layer" + +namespace llvm { +namespace orc { + +//===----------------------------------------------------------------------===// + +raw_ostream &operator<<(raw_ostream &OS, const FnSetTy &Gs) { + if (Gs.empty()) { + OS << "{ }"; + } else { + OS << *Gs.begin(); + for (const auto &G : make_range(std::next(Gs.begin()), Gs.end())) + OS << ", " << G; + } + return OS; +} + +static raw_ostream &DumpDiscoveredCallees(const FnSetByModuleTy &Cs, + const ModuleSummaryIndex &Idx, + raw_ostream &OS) { + if (Cs.empty()) { + OS << "{ }"; + } else { + // Dump all map values with prefixed VModuleKey + auto DumpVal = [&OS, &Idx](const FnSetByModuleTy::MapEntryTy &E) { + OS << "(" << Idx.getModuleId(E.first()) << ") " << E.second; + }; + + DumpVal(*Cs.begin()); + for (const auto &C : make_range(std::next(Cs.begin()), Cs.end())) + DumpVal(C); + + // Dump all map keys with prefixed VModuleKey + for (const auto &C : Cs) + OS << "\n(" << Idx.getModuleId(C.first()) << ") " << C.first(); + } + + return OS; +} + +//===----------------------------------------------------------------------===// + +Expected ThinLTOLayer2::add(JITDylib &JD, std::string Path) { + LLVM_DEBUG(dbgs() << "| Add | "); + + if (auto FullPath = findFullModulePath(std::move(Path))) + Path = std::move(*FullPath); + else + return FullPath.takeError(); + + // Don't try and update added Modules. Consider files on disk immutable. + if (CombinedIndex.getModuleId(Path)) + return createStringError(inconvertibleErrorCode(), + "Cannot add module twice: %s", Path.c_str()); + + // Memory-map bitcode file. Don't cache anything. + auto BufferOrErr = MemoryBuffer::getFile(Path); + if (!BufferOrErr) + return errorCodeToError(BufferOrErr.getError()); + + // Read summary section and merge it into the global CombinedIndex. + ExecutionSession &ES = getExecutionSession(); + VModuleKey Key = ES.allocateVModule(); + auto BufferRef = (*BufferOrErr)->getMemBufferRef(); + if (Error Err = readModuleSummaryIndex(BufferRef, CombinedIndex, Key)) + return std::move(Err); + + // VModuleKey is the Orc equivalent for ThinLTO's ModuleID + assert(CombinedIndex.getModuleId(Path) == Key); + LLVM_DEBUG(dbgs() << "(" << Key << ") " << Path << "\n"); + + // Remember the module's target dylib. + PendingModules[Key] = &JD; + return Key; +} + +static StringRef unmangled(SymbolStringPtr SymbolName) { + StringRef PoolStr = *SymbolName; + if (PoolStr.startswith("_")) + return PoolStr.substr(1); + + // TODO Consider other linker manglings. + return PoolStr; +} + +JITDylibList +ThinLTOLayer2::matchDylibs(const SymbolNameSet &Symbols, + ExecutionSession::ErrorReporter Report) { + std::lock_guard Lock(JITDylibsLock); + bool AddSubmittedDylibs = false; + + JITDylibList Dylibs; + Dylibs.reserve(PendingModules.size() + SubmittedDylibs.size()); + + for (SymbolStringPtr NamePtr : Symbols) { + // ValueInfo tells us where the symbol is defined. + // If we don't find one, we don't know this symbol. + auto FnGuid = GlobalValue::getGUID(unmangled(NamePtr)); + ValueInfo VI = CombinedIndex.getValueInfo(FnGuid); + if (!VI || VI.getSummaryList().empty()) + continue; + + StringRef Path = VI.getSummaryList().front()->modulePath(); + VModuleKey Key = CombinedIndex.getModuleId(Path); + + // If the module is not pending, all submitted dylibs are candidates. + // We don't currently record past submit to the base layer. + auto It = PendingModules.find(Key); + if (It == PendingModules.end()) { + AddSubmittedDylibs = true; + if (PendingModules.empty()) + break; + else + continue; + } + + // Otherwise lookup only in the one dylib that we stage now. + JITDylib *TargetDylib = It->second; + if (auto Err = stage(Path, {FnGuid})) { + Report(std::move(Err)); + continue; + } + + Dylibs.push_back(TargetDylib); + } + + // We may get few duplicates here. ThinLTO-ES will sort and unique. + if (AddSubmittedDylibs) { + Dylibs.insert(Dylibs.end(), SubmittedDylibs.begin(), SubmittedDylibs.end()); + } + + return Dylibs; +} + +template static void InsertUniqueSorted(std::vector &V, T Item) { + auto It = std::lower_bound(V.begin(), V.end(), Item); + if (It == V.end() || *It != Item) + V.insert(It, Item); +} + +Error ThinLTOLayer2::stage(std::string Path, FnSetTy Roots) { + // We can only discover functions after their modules were passed to add(). + VModuleKey Key = CombinedIndex.getModuleId(Path); + if (!Key) + return createStringError(inconvertibleErrorCode(), "Unknown module: %s", + Path.c_str()); + + LLVM_DEBUG(dbgs() << "| Stage | (" << Key << ") " << Path << "\n"); + + // We expect a module that was added and not staged yet. + // TODO Otherwise, is silent success the best we can do here? + auto It = PendingModules.find(Key); + if (LLVM_UNLIKELY(It == PendingModules.end())) { + LLVM_DEBUG(dbgs() << "Warning: Module staged already!\n"); + } else { + // Collect all function definitions from the given file. + GVSummaryMapTy DefinedFnsMap; + CombinedIndex.collectDefinedFunctionsForModule(Path, DefinedFnsMap); + + // Store everything in a MaterializationUnit. + auto MU = ThinLTOLayerMaterializationUnit::Create( + *this, Key, std::move(Path), std::move(DefinedFnsMap), std::move(Roots), + getExecutionSession()); + + // Schedule materialization into the dylib that was passed to add(). + if (Error Err = It->second->define(std::move(MU))) + return Err; + + // Update dylib tracking info. + LLVM_DEBUG(dbgs() << "Target JITDylib: " << It->second->getName() << "\n"); + InsertUniqueSorted(SubmittedDylibs, It->second); + PendingModules.erase(It); + } + + LLVM_DEBUG(dbgs() << "All submitted JITDylibs: " << SubmittedDylibs << "\n"); + return Error::success(); +} + +static bool hasNameForGuid(const SymbolNameSet &S, GlobalValue::GUID G) { + for (SymbolStringPtr NamePtr : S) + if (G == GlobalValue::getGUID(unmangled(NamePtr))) + return true; + + return false; +} + +void ThinLTOLayer2::emit(MaterializationResponsibility R, const VModuleKey &Key, + StringRef Path, const GVSummaryMapTy &AllDefinedFns, + const FnSetTy &DiscoveredGuids) { + assert(CombinedIndex.getModuleId(Path) == Key); + + SymbolNameSet MUReqSymbols = R.getRequestedSymbols(); + LLVM_DEBUG({ + dbgs() << "| Emit | Requested Symbols: " << MUReqSymbols << "\n"; + dbgs() << "| Emit | Target "; + R.getTargetJITDylib().dump(dbgs()); + dbgs() << "\n"; + }); + + // Collect discovery roots. + FnSummaryMapTy DiscoveryRoots; + ExecutionSession &ES = getExecutionSession(); + + // AllDefinedFns and RequestedSymbols are isomorphic. They contain all + // functions defined in the module. DiscoveredGuids is the subset of functions + // we discovered in the last step. Only these become new discovery roots. + for (GlobalValue::GUID FnGuid : DiscoveredGuids) { + // AllDefinedFns was obtained from collectDefinedFunctionsForModule(). + assert(AllDefinedFns.find(FnGuid) != AllDefinedFns.end() && + "Superset should have the GUID too"); + + // AllDefinedFns has been used to calculate SymbolFlags for the MU + // base class. SymbolFlags is used to provide RequestedSymbols, the + // collection of symbols we are responsible to emit. We pass this + // responsibility down to the base layers. In order to make sure this will + // work, check that it has the actual symbol for this GUID. + assert(hasNameForGuid(MUReqSymbols, FnGuid) && + "Required by base layers to emit the symbol for this GUID"); + + // This should only fail, if the index was corrupted since the MU creation. + ValueInfo VI = CombinedIndex.getValueInfo(FnGuid); + if (!VI || VI.getSummaryList().empty()) { + ES.reportError(createStringError(llvm::inconvertibleErrorCode(), + "Can't find ValueInfo for GUID %" PRIx64, + FnGuid)); + R.failMaterialization(); + return; + } + + // For now, assume properties of interest to be invariant across all + // summaries. Usually there is only one. + for (const auto &S : VI.getSummaryList()) { + assert(S->modulePath() == Path); + assert(S->getSummaryKind() == GlobalValueSummary::FunctionKind); + } + + FunctionSummary *Summary = + dyn_cast(VI.getSummaryList().front()->getBaseObject()); + + auto Res = DiscoveryRoots.try_emplace(FnGuid, Summary); + assert(Res.second && "Can't have duplicates in DiscoveredGuids"); + } + + // Discover callees from all roots. + FnSetByModuleTy CalleesByModule = + discover(Path, DiscoveryRoots, AllDefinedFns); + + // Stage all modules that define discovered functions. + for (auto &Entry : CalleesByModule) { + StringRef ModulePath = Entry.first(); + FnSetTy DiscoveredGuids = std::move(Entry.second); + if (auto Err = stage(ModulePath, std::move(DiscoveredGuids))) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + } + + // All went well, so we can finally emit the original module. + if (auto M = parseModule(Key, Path)) { + BaseLayer.emit(std::move(R), Key, std::move(*M)); + } else { + ES.reportError(M.takeError()); + R.failMaterialization(); + return; + } +} + +FnSetByModuleTy +ThinLTOLayer2::discover(StringRef Path, const FnSummaryMapTy &DiscoveryRoots, + const GVSummaryMapTy &DefinedGVSummaries) { + LLVM_DEBUG(dbgs() << "| Discover | "); + + // Limit on instruction count for edges we are following (ThinLTO default). + unsigned DiscoveryInstrLimit = 100; + FunctionImporter::ImportThresholdsTy ImportThresholds; + + // List of discovered functions that need to be processed. + SmallVector Worklist; + + // Resulting map of modules and discovered functions. + FnSetByModuleTy DiscoveredFns; + + for (const auto &Entry : DiscoveryRoots) { + const FunctionSummary &S = *Entry.second; + computeImportForFunction(S, CombinedIndex, DiscoveryInstrLimit, + DefinedGVSummaries, Worklist, DiscoveredFns, + nullptr, ImportThresholds); + } + + // Process the newly imported functions and add callees to the worklist. + while (!Worklist.empty()) { + auto FuncInfo = Worklist.pop_back_val(); + auto *Summary = std::get<0>(FuncInfo); + auto Threshold = std::get<1>(FuncInfo); + computeImportForFunction(*Summary, CombinedIndex, Threshold, + DefinedGVSummaries, Worklist, DiscoveredFns, + nullptr, ImportThresholds); + } + + LLVM_DEBUG({ + dbgs() << "Callees: "; + DumpDiscoveredCallees(DiscoveredFns, CombinedIndex, dbgs()) << "\n"; + }); + + return DiscoveredFns; +} + +Expected> ThinLTOLayer2::parseModule(VModuleKey Key, + StringRef Path) { + LLVM_DEBUG(dbgs() << "| Parse | (" << Key << ") " << Path << "\n"); + + auto F = MemoryBuffer::getFile(Path); + if (!F) + return errorCodeToError(F.getError()); + + MemoryBuffer &B = **F; + return parseBitcodeFile(B.getMemBufferRef(), GetContextForModule(Key)); +} + +//===----------------------------------------------------------------------===// + +template +static void AppendAll(std::vector &V, std::vector Items) { + V.insert(V.end(), std::make_move_iterator(Items.begin()), + std::make_move_iterator(Items.end())); +} + +Expected ThinLTOExecutionSession::lookup( + const JITDylibList &OriginalJDs, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, bool WaitUntilReady) { + LLVM_DEBUG(dbgs() << "| Lookup | " << Symbols << "\n"); + + // Basically, we can figure out the relevant Dylibs on our own. + JITDylibList JDs; + auto ErrorReporter = [this](Error Err) { reportError(std::move(Err)); }; + + for (JITDylibProvider *P : JITDylibProviders) { + JITDylibList ExtraJDs = P->matchDylibs(Symbols, ErrorReporter); + AppendAll(JDs, std::move(ExtraJDs)); + } + + // Still, keep the ones provided originally. + AppendAll(JDs, OriginalJDs); + + // The list must have no duplicates, so just do sort and unique-erase. + // TODO Would it be good to preserve the order? + std::sort(JDs.begin(), JDs.end()); + JDs.erase(std::unique(JDs.begin(), JDs.end()), JDs.end()); + + return ExecutionSession::lookup(JDs, Symbols, RegisterDependencies, + WaitUntilReady); +} + +//===----------------------------------------------------------------------===// + +static JITSymbolFlags fromGVSummary(const GlobalValueSummary &S, + GlobalValue::GUID Guid) { + JITSymbolFlags Flags = JITSymbolFlags::None; + + auto Linkage = S.flags().Linkage; + if (Linkage == GlobalValue::WeakAnyLinkage || + Linkage == GlobalValue::WeakODRLinkage) + Flags |= JITSymbolFlags::Weak; + + if (Linkage == GlobalValue::CommonLinkage) + Flags |= JITSymbolFlags::Common; + + // TODO Is this correct? Consider hidden if we had to import with new name. + bool hasHiddenVisibility = (Guid != S.getOriginalName()); + if (Linkage != GlobalValue::InternalLinkage && + Linkage != GlobalValue::PrivateLinkage && !hasHiddenVisibility) + Flags |= JITSymbolFlags::Exported; + + if (S.getSummaryKind() == GlobalValueSummary::FunctionKind) + Flags |= JITSymbolFlags::Callable; + + return Flags; +} + +std::unique_ptr +ThinLTOLayerMaterializationUnit::Create( + ThinLTOLayer2 &L, VModuleKey K, std::string Path, + GVSummaryMapTy AllDefinedFns, + std::unordered_set PotentialCallees, + ExecutionSession &ES) { + SymbolFlagsMap AllDefinedFnsFlags; + + // Make Orc JITFlags from ThinLTO GlobalValue flags + for (const auto &Entry : AllDefinedFns) { + GlobalValue::GUID G = Entry.getFirst(); + const GlobalValueSummary &S = *Entry.getSecond(); + + // Use the GUID as placeholder name for now. + SymbolStringPtr Name = ES.getSymbolStringPool().intern(std::to_string(G)); + JITSymbolFlags Flags = fromGVSummary(S, G); + + AllDefinedFnsFlags.emplace(Name, Flags); + } + + return std::unique_ptr( + new ThinLTOLayerMaterializationUnit( + L, K, std::move(Path), std::move(AllDefinedFnsFlags), + std::move(AllDefinedFns), std::move(PotentialCallees))); +} + +} // End namespace orc. +} // End namespace llvm. Index: unittests/ExecutionEngine/Orc/CMakeLists.txt =================================================================== --- unittests/ExecutionEngine/Orc/CMakeLists.txt +++ unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -25,6 +25,7 @@ RTDyldObjectLinkingLayerTest.cpp RTDyldObjectLinkingLayer2Test.cpp SymbolStringPoolTest.cpp + ThinLTOLayerTest.cpp ) target_link_libraries(OrcJITTests PRIVATE ${ORC_JIT_TEST_LIBS}) Index: unittests/ExecutionEngine/Orc/Inputs/Bar.ll =================================================================== --- /dev/null +++ unittests/ExecutionEngine/Orc/Inputs/Bar.ll @@ -0,0 +1,22 @@ +; ModuleID = 'Bar.o' +source_filename = "Bar.cpp" +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +; Function Attrs: noinline nounwind optnone ssp uwtable +define i32 @bar() #0 { +entry: + ret i32 1 +} + +attributes #0 = { noinline nounwind optnone ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"PIC Level", i32 2} +!2 = !{!"clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 23a0e9ffba00bb559554d64e2cfc17aed71a5fea) (llvm/trunk 341541)"} + +^0 = module: (path: "Bar.o", hash: (1372045330, 2056102272, 1752332163, 195788812, 3147837727)) +^1 = gv: (name: "bar", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 1, live: 0, dsoLocal: 0), insts: 1))) ; guid = 16434608426314478903 Index: unittests/ExecutionEngine/Orc/Inputs/Foo.ll =================================================================== --- /dev/null +++ unittests/ExecutionEngine/Orc/Inputs/Foo.ll @@ -0,0 +1,27 @@ +; ModuleID = 'Foo.o' +source_filename = "Foo.cpp" +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +; Function Attrs: noinline optnone ssp uwtable +define i32 @foo() #0 { +entry: + %call = call i32 @bar() + ret i32 %call +} + +declare i32 @bar() #1 + +attributes #0 = { noinline optnone ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"PIC Level", i32 2} +!2 = !{!"clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 23a0e9ffba00bb559554d64e2cfc17aed71a5fea) (llvm/trunk 341541)"} + +^0 = module: (path: "Foo.o", hash: (3339198734, 4079218373, 1835853161, 962078918, 457384650)) +^1 = gv: (name: "foo", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 1, live: 0, dsoLocal: 0), insts: 2, calls: ((callee: ^2))))) ; guid = 6699318081062747564 +^2 = gv: (name: "bar") ; guid = 16434608426314478903 Index: unittests/ExecutionEngine/Orc/ThinLTOLayerTest.cpp =================================================================== --- /dev/null +++ unittests/ExecutionEngine/Orc/ThinLTOLayerTest.cpp @@ -0,0 +1,188 @@ +//===- ThinLTOLayerTest.cpp - Unit tests for the ThinLTO layer ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ThinLTOLayer.h" +#include "OrcTestCommon.h" + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" + +#include "gtest/gtest.h" + +#include +#include +#include + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +//===----------------------------------------------------------------------===// +// +// Test input files must be assembled: +// unittests/ExecutionEngine/Orc/Inputs/Bar.ll +// unittests/ExecutionEngine/Orc/Inputs/Foo.ll +// +// Run llvm-as on each of them and put the .bc output file next to your +// OrcJITTests executable. +// +//===----------------------------------------------------------------------===// + +class ThinLTOLayer2Test : public testing::TestWithParam { + struct Init { + Init() { + InitializeNativeTarget(); + InitializeNativeTargetAsmParser(); + InitializeNativeTargetAsmPrinter(); + } + }; + Init StaticInit; + +public: + std::shared_ptr SharedStringPool; + std::unique_ptr TM; + LLVMContext Ctx; + DataLayout DL; + std::shared_ptr MM; + RTDyldObjectLinkingLayer2::GetMemoryManagerFunction GetMemManager; + CompileOnDemandLayer2::GetAvailableContextFunction GetContext; + GetAvailableContextFunction GetContextForMod; + + static int Exec(JITEvaluatedSymbol FuncSym) { + auto FuncPtr = (int (*)())FuncSym.getAddress(); + return FuncPtr(); + } + + static void JoinAll(std::vector &Ts) { + for (std::thread &T : Ts) + if (T.joinable()) + T.join(); + } + + SmallVector LookupOrder() { + SmallVector Names; + SplitString(GetParam(), Names, ","); + return Names; + } + + ThinLTOLayer2Test() + : StaticInit(), SharedStringPool(std::make_shared()), + TM(EngineBuilder().selectTarget()), DL(TM->createDataLayout()), + MM(std::make_shared()), + GetMemManager([this](VModuleKey) { return MM; }), + GetContext([this]() -> LLVMContext & { return Ctx; }), + GetContextForMod([this](VModuleKey) -> LLVMContext & { return Ctx; }) {} +}; + +// Run each test with these lookup order parameterizations. +INSTANTIATE_TEST_CASE_P(LookupOrders, ThinLTOLayer2Test, + ::testing::Values("bar", // 0 + "foo", // 1 + "bar,foo", // 2 + "foo,bar", // 3 + "bar,foo,foo", // 4 + "foo,bar,bar" // 5 + )); + +//===----------------------------------------------------------------------===// + +TEST_P(ThinLTOLayer2Test, UnthreadedSingleDylib) { + ThinLTOExecutionSession ES(SharedStringPool); + RTDyldObjectLinkingLayer2 LinkLayer(ES, GetMemManager); + IRCompileLayer2 CompileLayer(ES, LinkLayer, SimpleCompiler(*TM)); + ThinLTOLayer2 LookaheadLayer(ES, DL, CompileLayer, GetContextForMod); + + MangleAndInterner MandI(ES, DL); + JITDylib &Main = ES.createJITDylib("Main"); + cantFail(LookaheadLayer.add(Main, "Foo.bc")); + cantFail(LookaheadLayer.add(Main, "Bar.bc")); + + // We can pass any dylib, it's just for lookup to find the ES. + JITDylibList JD{&Main}; + + for (StringRef Name : LookupOrder()) { + if (auto Sym = lookup(JD, MandI(Name))) { + EXPECT_EQ(1, Exec(*Sym)); + } else { + FAIL() << toString(Sym.takeError()); + } + } +} + +//===----------------------------------------------------------------------===// + +TEST_P(ThinLTOLayer2Test, UnthreadedMultiDylib) { + ThinLTOExecutionSession ES(SharedStringPool); + RTDyldObjectLinkingLayer2 LinkLayer(ES, GetMemManager); + IRCompileLayer2 CompileLayer(ES, LinkLayer, SimpleCompiler(*TM)); + ThinLTOLayer2 LookaheadLayer(ES, DL, CompileLayer, GetContextForMod); + + MangleAndInterner MandI(ES, DL); + cantFail(LookaheadLayer.add(ES.createJITDylib("Foo"), "Foo.bc")); + cantFail(LookaheadLayer.add(ES.createJITDylib("Bar"), "Bar.bc")); + + // We can pass any dylib, it's just for lookup to find the ES. + JITDylibList JD{&ES.createJITDylib("Dummy")}; + + for (StringRef Name : LookupOrder()) { + if (auto Sym = lookup(JD, MandI(Name))) { + EXPECT_EQ(1, Exec(*Sym)); + } else { + FAIL() << toString(Sym.takeError()); + } + } +} + +//===----------------------------------------------------------------------===// + +TEST_P(ThinLTOLayer2Test, NaiveLazyEmit) { + ThinLTOExecutionSession ES(SharedStringPool); + RTDyldObjectLinkingLayer2 LinkLayer(ES, GetMemManager); + IRCompileLayer2 CompileLayer(ES, LinkLayer, SimpleCompiler(*TM)); + + const Triple &TT = TM->getTargetTriple(); + auto CCMgr = createLocalCompileCallbackManager(TT, ES, 0); + auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); + + IRTransformLayer2 TransformLayer(ES, CompileLayer); + CompileOnDemandLayer2 CODLayer(ES, TransformLayer, *CCMgr, + std::move(ISMBuilder), GetContext); + + ThinLTOLayer2 LookaheadLayer(ES, DL, CODLayer, GetContextForMod); + + MangleAndInterner MandI(ES, DL); + JITDylib &Main = ES.createJITDylib("Main"); + cantFail(LookaheadLayer.add(Main, "Foo.bc")); + cantFail(LookaheadLayer.add(Main, "Bar.bc")); + + // We can pass any dylib, it's just for lookup to find the ES. + JITDylibList JD{&Main}; + + for (StringRef Name : LookupOrder()) { + if (auto Sym = lookup(JD, MandI(Name))) { + EXPECT_EQ(1, Exec(*Sym)); + } else { + FAIL() << toString(Sym.takeError()); + } + } +} + +} // namespace