Index: include/lld/Core/ArchiveLibraryFile.h =================================================================== --- include/lld/Core/ArchiveLibraryFile.h +++ include/lld/Core/ArchiveLibraryFile.h @@ -11,6 +11,7 @@ #define LLD_CORE_ARCHIVE_LIBRARY_FILE_H #include "lld/Core/File.h" +#include "lld/Core/Parallel.h" #include namespace lld { @@ -37,9 +38,15 @@ virtual std::error_code parseAllMembers(std::vector> &result) = 0; + // Parses a member file containing a given symbol, so that when you + // need the file find() can return that immediately. Calling this function + // has no side effect other than pre-instantiating a file. Calling this + // function doesn't affect correctness. + virtual void preload(TaskGroup &group, StringRef symbolName) {} + /// Returns a set of all defined symbols in the archive, i.e. all /// resolvable symbol using this file. - virtual std::set getDefinedSymbols() const { + virtual std::set getDefinedSymbols() { return std::set(); } Index: include/lld/Core/File.h =================================================================== --- include/lld/Core/File.h +++ include/lld/Core/File.h @@ -164,6 +164,14 @@ std::error_code parse(); + // This function is called just before the core linker tries to use + // a file. Currently the PECOFF reader uses this to trigger the + // driver to parse .drectve section (which contains command line options). + // If you want to do something having side effects, don't do that in + // doParse() because a file could be pre-loaded speculatively. + // Use this hook instead. + virtual void beforeLink() {} + // Usually each file owns a std::unique_ptr. // However, there's one special case. If a file is an archive file, // the archive file and its children all shares the same memory buffer. Index: include/lld/Core/Resolver.h =================================================================== --- include/lld/Core/Resolver.h +++ include/lld/Core/Resolver.h @@ -10,6 +10,7 @@ #ifndef LLD_CORE_RESOLVER_H #define LLD_CORE_RESOLVER_H +#include "lld/Core/ArchiveLibraryFile.h" #include "lld/Core/File.h" #include "lld/Core/SharedLibraryFile.h" #include "lld/Core/SymbolTable.h" @@ -62,6 +63,7 @@ void maybeAddSectionGroupOrGnuLinkOnce(const DefinedAtom &atom); /// \brief The main function that iterates over the files to resolve + void makePreloadArchiveMap(); bool resolveUndefines(); void updateReferences(); void deadStripOptimize(); @@ -72,6 +74,7 @@ void markLive(const Atom *atom); void addAtoms(const std::vector&); + void maybePreloadArchiveMember(StringRef sym); class MergedFile : public MutableFile { public: @@ -119,6 +122,9 @@ std::vector _files; std::map _newUndefinesAdded; size_t _fileIndex; + + // Preloading + std::map _archiveMap; }; } // namespace lld Index: lib/Core/Resolver.cpp =================================================================== --- lib/Core/Resolver.cpp +++ lib/Core/Resolver.cpp @@ -31,9 +31,12 @@ bool undefAdded = false; for (const DefinedAtom *atom : file.defined()) doDefinedAtom(*atom); - for (const UndefinedAtom *atom : file.undefined()) - if (doUndefinedAtom(*atom)) + for (const UndefinedAtom *atom : file.undefined()) { + if (doUndefinedAtom(*atom)) { undefAdded = true; + maybePreloadArchiveMember(atom->name()); + } + } for (const SharedLibraryAtom *atom : file.sharedLibrary()) doSharedLibraryAtom(*atom); for (const AbsoluteAtom *atom : file.absolute()) @@ -229,6 +232,17 @@ doDefinedAtom(*newAtom); } +// Instantiate an archive file member if there's a file containing a +// defined symbol for a given symbol name. Instantiation is done in a +// different worker thread and has no visible side effect. +void Resolver::maybePreloadArchiveMember(StringRef sym) { + auto it = _archiveMap.find(sym); + if (it == _archiveMap.end()) + return; + ArchiveLibraryFile *archive = it->second; + archive->preload(_context.getTaskGroup(), sym); +} + // Returns true if at least one of N previous files has created an // undefined symbol. bool Resolver::undefinesAdded(int begin, int end) { @@ -261,6 +275,16 @@ return cast(inputs[index++].get())->getFile(); } +// Make a map of Symbol -> ArchiveFile. +void Resolver::makePreloadArchiveMap() { + std::vector> &nodes = _context.getNodes(); + for (auto it = nodes.rbegin(), e = nodes.rend(); it != e; ++it) + if (auto *fnode = dyn_cast(it->get())) + if (auto *archive = dyn_cast(fnode->getFile())) + for (StringRef sym : archive->getDefinedSymbols()) + _archiveMap[sym] = archive; +} + // Keep adding atoms until _context.getNextFile() returns an error. This // function is where undefined atoms are resolved. bool Resolver::resolveUndefines() { @@ -277,6 +301,7 @@ << ": " << ec.message() << "\n"; return false; } + file->beforeLink(); switch (file->kind()) { case File::kindObject: if (groupLevel > 0) @@ -446,6 +471,7 @@ } bool Resolver::resolve() { + makePreloadArchiveMap(); if (!resolveUndefines()) return false; updateReferences(); Index: lib/ReaderWriter/FileArchive.cpp =================================================================== --- lib/ReaderWriter/FileArchive.cpp +++ lib/ReaderWriter/FileArchive.cpp @@ -17,7 +17,9 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" +#include #include +#include #include #include @@ -57,6 +59,17 @@ return nullptr; _membersInstantiated.insert(memberStart); + + // Check if a file is preloaded. + { + std::lock_guard lock(_mutex); + auto it = _preloaded.find(memberStart); + if (it != _preloaded.end()) { + std::future &future = it->second; + return future.get(); + } + } + std::unique_ptr result; if (instantiateMember(ci, result)) return nullptr; @@ -65,6 +78,37 @@ return result.release(); } + // Instantiate a member file containing a given symbol name. + void preload(TaskGroup &group, StringRef name) override { + auto member = _symbolMemberMap.find(name); + if (member == _symbolMemberMap.end()) + return; + Archive::child_iterator ci = member->second; + + // Do nothing if a member is already instantiated. + const char *memberStart = ci->getBuffer().data(); + if (_membersInstantiated.count(memberStart)) + return; + + std::lock_guard lock(_mutex); + if (_preloaded.find(memberStart) != _preloaded.end()) + return; + + // Instantiate the member + int index = _promises.size(); + _promises.emplace_back(); + _preloaded[memberStart] = _promises[index].get_future(); + + group.spawn([=] { + std::unique_ptr result; + if (instantiateMember(ci, result)) { + _promises[index].set_value(nullptr); + return; + } + _promises[index].set_value(result.release()); + }); + } + /// \brief parse each member std::error_code parseAllMembers(std::vector> &result) override { @@ -117,7 +161,8 @@ } /// Returns a set of all defined symbols in the archive. - std::set getDefinedSymbols() const override { + std::set getDefinedSymbols() override { + parse(); std::set ret; for (const auto &e : _symbolMemberMap) ret.insert(e.first); @@ -225,6 +270,9 @@ atom_collection_vector _absoluteAtoms; bool _logLoading; mutable std::vector> _memberBuffers; + mutable std::map> _preloaded; + mutable std::vector> _promises; + mutable std::mutex _mutex; }; class ArchiveReader : public Reader { Index: lib/ReaderWriter/PECOFF/ReaderCOFF.cpp =================================================================== --- lib/ReaderWriter/PECOFF/ReaderCOFF.cpp +++ lib/ReaderWriter/PECOFF/ReaderCOFF.cpp @@ -106,6 +106,8 @@ return _absoluteAtoms; } + void beforeLink() override; + void addDefinedAtom(AliasAtom *atom) { atom->setOrdinal(_ordinal++); _definedAtoms._atoms.push_back(atom); @@ -382,7 +384,10 @@ // The mapping for /alternatename is in the context object. This helper // function iterate over defined atoms and create alias atoms if needed. createAlternateNameAtoms(); + return std::error_code(); +} +void FileCOFF::beforeLink() { // Acquire the mutex to mutate _ctx. std::lock_guard lock(_ctx.getMutex()); @@ -392,10 +397,8 @@ _ctx.setSafeSEH(false); if (_ctx.deadStrip()) - for (StringRef sym : undefinedSymbols) - _ctx.addDeadStripRoot(sym); - - return std::error_code(); + for (const UndefinedAtom *undef : undefined()) + _ctx.addDeadStripRoot(undef->name()); } /// Iterate over the symbol table to retrieve all symbols.