Index: llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h =================================================================== --- llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -287,6 +287,13 @@ Create(ObjectLayer &L, std::unique_ptr ArchiveBuffer, GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface()); + /// Returns a list of filenames of dynamic libraries that this archive has + /// imported. This class does not load these libraries by itself. User is + /// responsible for making sure these libraries are avaliable to the JITDylib. + const std::set &getImportedDynamicLibraries() const { + return ImportedDynamicLibraries; + } + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) override; @@ -297,10 +304,14 @@ GetObjectFileInterface GetObjFileInterface, Error &Err); + Error buildObjectFilesMap(); + ObjectLayer &L; GetObjectFileInterface GetObjFileInterface; + std::set ImportedDynamicLibraries; std::unique_ptr ArchiveBuffer; std::unique_ptr Archive; + DenseMap ObjectFilesMap; }; } // end namespace orc Index: llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp =================================================================== --- llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -364,16 +364,11 @@ for (const auto &KV : Symbols) { const auto &Name = KV.first; - auto Child = Archive->findSym(*Name); - if (!Child) - return Child.takeError(); - if (*Child == None) + if (!ObjectFilesMap.count(Name)) continue; - auto ChildBuffer = (*Child)->getMemoryBufferRef(); - if (!ChildBuffer) - return ChildBuffer.takeError(); + auto ChildBuffer = ObjectFilesMap[Name]; ChildBufferInfos.insert( - {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); + {ChildBuffer.getBuffer(), ChildBuffer.getBufferIdentifier()}); } for (auto ChildBufferInfo : ChildBufferInfos) { @@ -392,6 +387,36 @@ return Error::success(); } +Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() { + DenseMap MemoryBuffers; + DenseSet Visited; + DenseSet Excluded; + for (auto &S : Archive->symbols()) { + StringRef SymName = S.getName(); + auto Member = S.getMember(); + if (!Member) + return Member.takeError(); + auto DataOffset = Member->getDataOffset(); + if (!Visited.count(DataOffset)) { + Visited.insert(DataOffset); + auto Child = Member->getAsBinary(); + if (!Child) + return Child.takeError(); + if ((*Child)->isCOFFImportFile()) { + ImportedDynamicLibraries.insert((*Child)->getFileName().str()); + Excluded.insert(DataOffset); + continue; + } + MemoryBuffers[DataOffset] = (*Child)->getMemoryBufferRef(); + } + if (!Excluded.count(DataOffset)) + ObjectFilesMap[L.getExecutionSession().intern(SymName)] = + MemoryBuffers[DataOffset]; + } + + return Error::success(); +} + StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr ArchiveBuffer, GetObjectFileInterface GetObjFileInterface, Error &Err) @@ -401,6 +426,8 @@ if (!this->GetObjFileInterface) this->GetObjFileInterface = getObjectFileInterface; + if (!Err) + Err = buildObjectFilesMap(); } } // End namespace orc. Index: llvm/tools/llvm-jitlink/llvm-jitlink.h =================================================================== --- llvm/tools/llvm-jitlink/llvm-jitlink.h +++ llvm/tools/llvm-jitlink/llvm-jitlink.h @@ -77,6 +77,7 @@ StringSet<> HarnessExternals; StringSet<> HarnessDefinitions; DenseMap CanonicalWeakDefs; + DenseSet LoadedLibraries; private: Session(std::unique_ptr EPC, Error &Err); Index: llvm/tools/llvm-jitlink/llvm-jitlink.cpp =================================================================== --- llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -14,6 +14,7 @@ #include "llvm-jitlink.h" +#include "llvm/ADT/PriorityQueue.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" #include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" @@ -93,6 +94,11 @@ cl::desc("Link against library X with hidden visibility"), cl::cat(JITLinkCategory)); +static cl::opt SearchSystemDynLibrary( + "search-sys-dynlib", + cl::desc("Add system library paths to dynamic library search paths"), + cl::init(false), cl::cat(JITLinkCategory)); + static cl::opt NoExec("noexec", cl::desc("Do not execute loaded code"), cl::init(false), cl::cat(JITLinkCategory)); @@ -1144,7 +1150,7 @@ if (EPC.getTargetTriple().getObjectFormat() == Triple::MachO) return registerMachOGraphInfo(*this, G); - if (EPC.getTargetTriple().isOSWindows()) + if (EPC.getTargetTriple().getObjectFormat() == Triple::COFF) return registerCOFFGraphInfo(*this, G); return make_error("Unsupported object format for GOT/stub " @@ -1263,8 +1269,10 @@ auto Obj = ExitOnErr( object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())); Triple TT = Obj->makeTriple(); - if (Magic == file_magic::coff_object) + if (Magic == file_magic::coff_object) { + TT.setObjectFormat(Triple::COFF); TT.setOS(Triple::OSType::Win32); + } return TT; } default: @@ -1525,6 +1533,17 @@ return I; } +static SmallVector getSearchPathsFromEnvVar(Session &S) { + SmallVector PathVec; + auto TT = S.ES.getExecutorProcessControl().getTargetTriple(); + if (TT.isOSBinFormatCOFF()) + StringRef(getenv("PATH")).split(PathVec, ";"); + else if (TT.isOSBinFormatELF()) + StringRef(getenv("LD_LIBRARY_PATH")).split(PathVec, ":"); + + return PathVec; +} + static Error addLibraries(Session &S, const std::map &IdxToJD) { @@ -1562,13 +1581,19 @@ // 2. Collect library loads struct LibraryLoad { - StringRef LibName; + std::string LibName; bool IsPath = false; unsigned Position; StringRef *CandidateExtensions; enum { Standard, Hidden } Modifier; }; - std::vector LibraryLoads; + + // Queue to load library as in the order as it appears in the argument list. + auto Comp = [](const LibraryLoad &LHS, const LibraryLoad &RHS) -> bool { + return LHS.Position > RHS.Position; + }; + llvm::PriorityQueue, decltype(Comp)> + LibraryLoadQueue(Comp); // Add archive files from the inputs to LibraryLoads. for (auto InputFileItr = InputFiles.begin(), InputFileEnd = InputFiles.end(); InputFileItr != InputFileEnd; ++InputFileItr) { @@ -1576,12 +1601,12 @@ if (!InputFile.endswith(".a") && !InputFile.endswith(".lib")) continue; LibraryLoad LL; - LL.LibName = InputFile; + LL.LibName = InputFile.str(); LL.IsPath = true; LL.Position = InputFiles.getPosition(InputFileItr - InputFiles.begin()); LL.CandidateExtensions = nullptr; LL.Modifier = LibraryLoad::Standard; - LibraryLoads.push_back(std::move(LL)); + LibraryLoadQueue.push(std::move(LL)); } // Add -load_hidden arguments to LibraryLoads. @@ -1593,9 +1618,10 @@ LL.Position = LoadHidden.getPosition(LibItr - LoadHidden.begin()); LL.CandidateExtensions = nullptr; LL.Modifier = LibraryLoad::Hidden; - LibraryLoads.push_back(std::move(LL)); + LibraryLoadQueue.push(std::move(LL)); } StringRef StandardExtensions[] = {".so", ".dylib", ".dll", ".a", ".lib"}; + StringRef DynLibExtensionsOnly[] = {".so", ".dylib", ".dll"}; StringRef ArchiveExtensionsOnly[] = {".a", ".lib"}; // Add -lx arguments to LibraryLoads. @@ -1606,7 +1632,7 @@ LL.Position = Libraries.getPosition(LibItr - Libraries.begin()); LL.CandidateExtensions = StandardExtensions; LL.Modifier = LibraryLoad::Standard; - LibraryLoads.push_back(std::move(LL)); + LibraryLoadQueue.push(std::move(LL)); } // Add -hidden-lx arguments to LibraryLoads. @@ -1619,7 +1645,7 @@ LibrariesHidden.getPosition(LibHiddenItr - LibrariesHidden.begin()); LL.CandidateExtensions = ArchiveExtensionsOnly; LL.Modifier = LibraryLoad::Hidden; - LibraryLoads.push_back(std::move(LL)); + LibraryLoadQueue.push(std::move(LL)); } // If there are any load- options then turn on flag overrides @@ -1627,11 +1653,6 @@ if (!LibrariesHidden.empty() || !LoadHidden.empty()) S.ObjLayer.setOverrideObjectFlagsWithResponsibilityFlags(true); - // Sort library loads by position in the argument list. - llvm::sort(LibraryLoads, [](const LibraryLoad &LHS, const LibraryLoad &RHS) { - return LHS.Position < RHS.Position; - }); - // 3. Process library loads. auto AddArchive = [&](const char *Path, const LibraryLoad &LL) -> Expected> { @@ -1646,13 +1667,35 @@ GetObjFileInterface = getObjectFileInterfaceHidden; break; } - return StaticLibraryDefinitionGenerator::Load( + auto G = StaticLibraryDefinitionGenerator::Load( S.ObjLayer, Path, S.ES.getExecutorProcessControl().getTargetTriple(), std::move(GetObjFileInterface)); + if (!G) + return G.takeError(); + + // Push additional dynamic libraries to search. + // Note that this mechanism only happens in COFF. + for (auto FileName : (*G)->getImportedDynamicLibraries()) { + LibraryLoad NewLL; + auto FileNameRef = StringRef(FileName); + assert(FileNameRef.endswith_insensitive(".dll") && + "COFF Imported library not ending with dll extension?"); + NewLL.LibName = FileNameRef.drop_back(strlen(".dll")).str(); + NewLL.Position = LL.Position; + NewLL.CandidateExtensions = DynLibExtensionsOnly; + NewLL.Modifier = LibraryLoad::Standard; + LibraryLoadQueue.push(std::move(NewLL)); + } + return G; }; - for (auto &LL : LibraryLoads) { + SmallVector SystemSearchPaths; + if (SearchSystemDynLibrary.getValue()) + SystemSearchPaths = getSearchPathsFromEnvVar(S); + while (!LibraryLoadQueue.empty()) { bool LibFound = false; + auto LL = LibraryLoadQueue.top(); + LibraryLoadQueue.pop(); auto &JD = *std::prev(IdxToJD.lower_bound(LL.Position))->second; // If this is the name of a JITDylib then link against that. @@ -1662,7 +1705,7 @@ } if (LL.IsPath) { - auto G = AddArchive(LL.LibName.str().c_str(), LL); + auto G = AddArchive(LL.LibName.c_str(), LL); if (!G) return createFileError(LL.LibName, G.takeError()); JD.addGenerator(std::move(*G)); @@ -1674,9 +1717,9 @@ } // Otherwise look through the search paths. - auto JDSearchPathsItr = JDSearchPaths.find(&JD); - if (JDSearchPathsItr != JDSearchPaths.end()) { - for (StringRef SearchPath : JDSearchPathsItr->second) { + auto TraverseSearchPathList = + [&](ArrayRef SearchPaths) -> Error { + for (StringRef SearchPath : SearchPaths) { for (const char *LibExt : {".dylib", ".so", ".dll", ".a", ".lib"}) { SmallVector LibPath; LibPath.reserve(SearchPath.size() + strlen("lib") + @@ -1710,6 +1753,7 @@ // in the default case. LibFound = true; switch (Magic) { + case file_magic::pecoff_executable: case file_magic::elf_shared_object: case file_magic::macho_dynamically_linked_shared_lib: { // TODO: On first reference to LibPath this should create a JITDylib @@ -1754,13 +1798,18 @@ if (LibFound) break; } - } - if (!LibFound) - return make_error("While linking " + JD.getName() + - ", could not find library for -l" + - LL.LibName, - inconvertibleErrorCode()); + if (!LibFound) + return make_error("While linking " + JD.getName() + + ", could not find library for -l" + + LL.LibName, + inconvertibleErrorCode()); + return Error::success(); + }; + if (auto Err = TraverseSearchPathList(JDSearchPaths[&JD])) + return Err; + if (auto Err = TraverseSearchPathList(SystemSearchPaths)) + return Err; } return Error::success();