Index: llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h =================================================================== --- llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -287,6 +287,13 @@ Create(ObjectLayer &L, std::unique_ptr ArchiveBuffer, GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface()); + /// Returns a list of filenames of dynamic libraries that this archive has + /// imported. This class does not load these libraries by itself. User is + /// responsible for making sure these libraries are avaliable to the JITDylib. + const std::set &getImportedDynamicLibraries() const { + return ImportedDynamicLibraries; + } + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) override; @@ -297,10 +304,14 @@ GetObjectFileInterface GetObjFileInterface, Error &Err); + Error buildObjectFilesMap(); + ObjectLayer &L; GetObjectFileInterface GetObjFileInterface; + std::set ImportedDynamicLibraries; std::unique_ptr ArchiveBuffer; std::unique_ptr Archive; + DenseMap ObjectFilesMap; }; } // end namespace orc Index: llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp =================================================================== --- llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -364,16 +364,11 @@ for (const auto &KV : Symbols) { const auto &Name = KV.first; - auto Child = Archive->findSym(*Name); - if (!Child) - return Child.takeError(); - if (*Child == None) + if (!ObjectFilesMap.count(Name)) continue; - auto ChildBuffer = (*Child)->getMemoryBufferRef(); - if (!ChildBuffer) - return ChildBuffer.takeError(); + auto ChildBuffer = ObjectFilesMap[Name]; ChildBufferInfos.insert( - {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); + {ChildBuffer.getBuffer(), ChildBuffer.getBufferIdentifier()}); } for (auto ChildBufferInfo : ChildBufferInfos) { @@ -392,6 +387,36 @@ return Error::success(); } +Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() { + DenseMap MemoryBuffers; + DenseSet Visited; + DenseSet Excluded; + for (auto &S : Archive->symbols()) { + StringRef SymName = S.getName(); + auto Member = S.getMember(); + if (!Member) + return Member.takeError(); + auto DataOffset = Member->getDataOffset(); + if (!Visited.count(DataOffset)) { + Visited.insert(DataOffset); + auto Child = Member->getAsBinary(); + if (!Child) + return Child.takeError(); + if ((*Child)->isCOFFImportFile()) { + ImportedDynamicLibraries.insert((*Child)->getFileName().str()); + Excluded.insert(DataOffset); + continue; + } + MemoryBuffers[DataOffset] = (*Child)->getMemoryBufferRef(); + } + if (!Excluded.count(DataOffset)) + ObjectFilesMap[L.getExecutionSession().intern(SymName)] = + MemoryBuffers[DataOffset]; + } + + return Error::success(); +} + StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr ArchiveBuffer, GetObjectFileInterface GetObjFileInterface, Error &Err) @@ -401,6 +426,8 @@ if (!this->GetObjFileInterface) this->GetObjFileInterface = getObjectFileInterface; + if (!Err) + Err = buildObjectFilesMap(); } } // End namespace orc. Index: llvm/tools/llvm-jitlink/llvm-jitlink.h =================================================================== --- llvm/tools/llvm-jitlink/llvm-jitlink.h +++ llvm/tools/llvm-jitlink/llvm-jitlink.h @@ -77,6 +77,7 @@ StringSet<> HarnessExternals; StringSet<> HarnessDefinitions; DenseMap CanonicalWeakDefs; + DenseSet LoadedLibraries; private: Session(std::unique_ptr EPC, Error &Err); Index: llvm/tools/llvm-jitlink/llvm-jitlink.cpp =================================================================== --- llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -51,6 +51,7 @@ #include #include +#include #include #ifdef LLVM_ON_UNIX @@ -93,6 +94,11 @@ cl::desc("Link against library X with hidden visibility"), cl::cat(JITLinkCategory)); +static cl::opt SearchSystemDynLibrary( + "search-sys-dynlib", + cl::desc("Add system library paths to dynamic library search paths"), + cl::init(false), cl::cat(JITLinkCategory)); + static cl::opt NoExec("noexec", cl::desc("Do not execute loaded code"), cl::init(false), cl::cat(JITLinkCategory)); @@ -1144,7 +1150,7 @@ if (EPC.getTargetTriple().getObjectFormat() == Triple::MachO) return registerMachOGraphInfo(*this, G); - if (EPC.getTargetTriple().isOSWindows()) + if (EPC.getTargetTriple().getObjectFormat() == Triple::COFF) return registerCOFFGraphInfo(*this, G); return make_error("Unsupported object format for GOT/stub " @@ -1263,8 +1269,10 @@ auto Obj = ExitOnErr( object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())); Triple TT = Obj->makeTriple(); - if (Magic == file_magic::coff_object) + if (Magic == file_magic::coff_object) { + TT.setObjectFormat(Triple::COFF); TT.setOS(Triple::OSType::Win32); + } return TT; } default: @@ -1525,6 +1533,17 @@ return I; } +static SmallVector getSearchPathsFromEnvVar(Session &S) { + SmallVector PathVec; + auto TT = S.ES.getExecutorProcessControl().getTargetTriple(); + if (TT.isOSBinFormatCOFF()) + StringRef(getenv("PATH")).split(PathVec, ";"); + else if (TT.isOSBinFormatELF()) + StringRef(getenv("LD_LIBRARY_PATH")).split(PathVec, ":"); + + return PathVec; +} + static Error addLibraries(Session &S, const std::map &IdxToJD) { @@ -1562,7 +1581,7 @@ // 2. Collect library loads struct LibraryLoad { - StringRef LibName; + std::string LibName; bool IsPath = false; unsigned Position; StringRef *CandidateExtensions; @@ -1576,7 +1595,7 @@ if (!InputFile.endswith(".a") && !InputFile.endswith(".lib")) continue; LibraryLoad LL; - LL.LibName = InputFile; + LL.LibName = InputFile.str(); LL.IsPath = true; LL.Position = InputFiles.getPosition(InputFileItr - InputFiles.begin()); LL.CandidateExtensions = nullptr; @@ -1596,6 +1615,7 @@ LibraryLoads.push_back(std::move(LL)); } StringRef StandardExtensions[] = {".so", ".dylib", ".dll", ".a", ".lib"}; + StringRef DynLibExtensionsOnly[] = {".so", ".dylib", ".dll"}; StringRef ArchiveExtensionsOnly[] = {".a", ".lib"}; // Add -lx arguments to LibraryLoads. @@ -1632,6 +1652,12 @@ return LHS.Position < RHS.Position; }); + // Stack to load library one by one with an option to push another library + // load. + std::stack LibraryLoadStack; + for (auto It = LibraryLoads.rbegin(); It != LibraryLoads.rend(); ++It) + LibraryLoadStack.push(*It); + // 3. Process library loads. auto AddArchive = [&](const char *Path, const LibraryLoad &LL) -> Expected> { @@ -1646,13 +1672,32 @@ GetObjFileInterface = getObjectFileInterfaceHidden; break; } - return StaticLibraryDefinitionGenerator::Load( + auto G = StaticLibraryDefinitionGenerator::Load( S.ObjLayer, Path, S.ES.getExecutorProcessControl().getTargetTriple(), std::move(GetObjFileInterface)); + if (!G) + return G.takeError(); + + // Push additional dynamic libraries to search. + // Note that this mechanism only happens in COFF. + for (auto FileName : (*G)->getImportedDynamicLibraries()) { + LibraryLoad NewLL; + auto FileNameRef = StringRef(FileName); + assert(FileNameRef.endswith_insensitive(".dll") && + "COFF Imported library not ending with dll extension?"); + NewLL.LibName = FileNameRef.drop_back(strlen(".dll")).str(); + NewLL.Position = LL.Position; + NewLL.CandidateExtensions = DynLibExtensionsOnly; + NewLL.Modifier = LibraryLoad::Standard; + LibraryLoadStack.push(std::move(NewLL)); + } + return G; }; - for (auto &LL : LibraryLoads) { + while (!LibraryLoadStack.empty()) { bool LibFound = false; + auto LL = LibraryLoadStack.top(); + LibraryLoadStack.pop(); auto &JD = *std::prev(IdxToJD.lower_bound(LL.Position))->second; // If this is the name of a JITDylib then link against that. @@ -1662,7 +1707,7 @@ } if (LL.IsPath) { - auto G = AddArchive(LL.LibName.str().c_str(), LL); + auto G = AddArchive(LL.LibName.c_str(), LL); if (!G) return createFileError(LL.LibName, G.takeError()); JD.addGenerator(std::move(*G)); @@ -1674,86 +1719,85 @@ } // Otherwise look through the search paths. - auto JDSearchPathsItr = JDSearchPaths.find(&JD); - if (JDSearchPathsItr != JDSearchPaths.end()) { - for (StringRef SearchPath : JDSearchPathsItr->second) { - for (const char *LibExt : {".dylib", ".so", ".dll", ".a", ".lib"}) { - SmallVector LibPath; - LibPath.reserve(SearchPath.size() + strlen("lib") + - LL.LibName.size() + strlen(LibExt) + - 2); // +2 for pathsep, null term. - llvm::copy(SearchPath, std::back_inserter(LibPath)); - if (StringRef(LibExt) != ".lib" && StringRef(LibExt) != ".dll") - sys::path::append(LibPath, "lib" + LL.LibName + LibExt); - else - sys::path::append(LibPath, LL.LibName + LibExt); - LibPath.push_back('\0'); - - // Skip missing or non-regular paths. - if (sys::fs::get_file_type(LibPath.data()) != - sys::fs::file_type::regular_file) { - continue; - } - - file_magic Magic; - if (auto EC = identify_magic(LibPath, Magic)) { - // If there was an error loading the file then skip it. - LLVM_DEBUG({ - dbgs() << "Library search found \"" << LibPath - << "\", but could not identify file type (" << EC.message() - << "). Skipping.\n"; - }); - continue; - } - - // We identified the magic. Assume that we can load it -- we'll reset - // in the default case. - LibFound = true; - switch (Magic) { - case file_magic::elf_shared_object: - case file_magic::macho_dynamically_linked_shared_lib: { - // TODO: On first reference to LibPath this should create a JITDylib - // with a generator and add it to JD's links-against list. Subsquent - // references should use the JITDylib created on the first - // reference. - auto G = - EPCDynamicLibrarySearchGenerator::Load(S.ES, LibPath.data()); - if (!G) - return G.takeError(); - LLVM_DEBUG({ - dbgs() << "Adding generator for dynamic library " - << LibPath.data() << " to " << JD.getName() << "\n"; - }); - JD.addGenerator(std::move(*G)); - break; - } - case file_magic::archive: - case file_magic::macho_universal_binary: { - auto G = AddArchive(LibPath.data(), LL); - if (!G) - return G.takeError(); - JD.addGenerator(std::move(*G)); - LLVM_DEBUG({ - dbgs() << "Adding generator for static library " << LibPath.data() - << " to " << JD.getName() << "\n"; - }); - break; - } - default: - // This file isn't a recognized library kind. - LLVM_DEBUG({ - dbgs() << "Library search found \"" << LibPath - << "\", but file type is not supported. Skipping.\n"; - }); - LibFound = false; - break; - } - if (LibFound) - break; + auto SearchPaths = JDSearchPaths[&JD]; + if (SearchSystemDynLibrary.getValue()) + SearchPaths.append(getSearchPathsFromEnvVar(S)); + for (StringRef SearchPath : SearchPaths) { + for (const char *LibExt : {".dylib", ".so", ".dll", ".a", ".lib"}) { + SmallVector LibPath; + LibPath.reserve(SearchPath.size() + strlen("lib") + LL.LibName.size() + + strlen(LibExt) + 2); // +2 for pathsep, null term. + llvm::copy(SearchPath, std::back_inserter(LibPath)); + if (StringRef(LibExt) != ".lib" && StringRef(LibExt) != ".dll") + sys::path::append(LibPath, "lib" + LL.LibName + LibExt); + else + sys::path::append(LibPath, LL.LibName + LibExt); + LibPath.push_back('\0'); + + // Skip missing or non-regular paths. + if (sys::fs::get_file_type(LibPath.data()) != + sys::fs::file_type::regular_file) { + continue; + } + + file_magic Magic; + if (auto EC = identify_magic(LibPath, Magic)) { + // If there was an error loading the file then skip it. + LLVM_DEBUG({ + dbgs() << "Library search found \"" << LibPath + << "\", but could not identify file type (" << EC.message() + << "). Skipping.\n"; + }); + continue; + } + + // We identified the magic. Assume that we can load it -- we'll reset + // in the default case. + LibFound = true; + switch (Magic) { + case file_magic::pecoff_executable: + case file_magic::elf_shared_object: + case file_magic::macho_dynamically_linked_shared_lib: { + // TODO: On first reference to LibPath this should create a JITDylib + // with a generator and add it to JD's links-against list. Subsquent + // references should use the JITDylib created on the first + // reference. + auto G = EPCDynamicLibrarySearchGenerator::Load(S.ES, LibPath.data()); + if (!G) + return G.takeError(); + LLVM_DEBUG({ + dbgs() << "Adding generator for dynamic library " << LibPath.data() + << " to " << JD.getName() << "\n"; + }); + JD.addGenerator(std::move(*G)); + break; + } + case file_magic::archive: + case file_magic::macho_universal_binary: { + auto G = AddArchive(LibPath.data(), LL); + if (!G) + return G.takeError(); + JD.addGenerator(std::move(*G)); + LLVM_DEBUG({ + dbgs() << "Adding generator for static library " << LibPath.data() + << " to " << JD.getName() << "\n"; + }); + break; + } + default: + // This file isn't a recognized library kind. + LLVM_DEBUG({ + dbgs() << "Library search found \"" << LibPath + << "\", but file type is not supported. Skipping.\n"; + }); + LibFound = false; + break; } if (LibFound) break; } + if (LibFound) + break; } if (!LibFound)