Index: llvm/tools/dsymutil/BinaryHolder.h =================================================================== --- llvm/tools/dsymutil/BinaryHolder.h +++ llvm/tools/dsymutil/BinaryHolder.h @@ -14,6 +14,7 @@ #ifndef LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H #define LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Triple.h" #include "llvm/Object/Archive.h" #include "llvm/Object/Error.h" @@ -26,6 +27,94 @@ namespace llvm { namespace dsymutil { +class CachedBinaryHolder { +public: + using TimestampTy = sys::TimePoint; + + CachedBinaryHolder(bool Verbose = false) : Verbose(Verbose) {} + + class EntryBase { + protected: + std::unique_ptr MemoryBuffer; + std::unique_ptr FatBinary; + std::string FatBinaryName; + }; + + struct ObjectEntry : public EntryBase { + std::vector> Objects; + + Error load(StringRef Filename, bool Verbose = false); + + /// Access all owned ObjectFiles. + std::vector getObjects() { + std::vector Result; + Result.reserve(Objects.size()); + for (auto &Object : Objects) { + Result.push_back(Object.get()); + } + return Result; + } + + /// Access to a derived version of all the currently owned ObjectFiles. The + /// conversion might be invalid, in which case an Error is returned. + template + Expected> getObjectsAs() { + std::vector Result; + Result.reserve(Objects.size()); + for (auto &Object : Objects) { + const auto *Derived = dyn_cast(Object.get()); + if (!Derived) + return errorCodeToError(object::object_error::invalid_file_type); + Result.push_back(Derived); + } + return Result; + } + + /// Access the owned ObjectFile with architecture \p T. + Expected getObject(const Triple &T); + + /// Access to a derived version of the currently owned ObjectFile with + /// architecture \p T. The conversion must be known to be valid. + template + Expected getObjectAs(const Triple &T) { + auto Object = getObject(T); + if (!Object) + return Object.takeError(); + return cast(*Object); + } + }; + + struct ArchiveEntry : public EntryBase { + struct KeyTy { + StringRef Filename; + TimestampTy Timestamp; + }; + + Error load(StringRef Filename, TimestampTy Timestamp, bool Verbose = false); + + Expected getObjectEntry(StringRef Filename, + TimestampTy Timestamp, + bool Verbose = false); + + private: + std::vector> Archives; + DenseMap ObjectCache; + }; + + Expected getObjectEntry(StringRef Filename, + TimestampTy Timestamp); + +private: + /// Cache of static archives. Objects that are part of a static archive are + /// stored under this object, rather than in the map below. + StringMap ArchiveCache; + + /// Object entries for objects that are not in a static archive. + StringMap ObjectCache; + + bool Verbose; +}; + /// The BinaryHolder class is responsible for creating and owning ObjectFile /// objects and their underlying MemoryBuffer. This is different from a simple /// OwningBinary in that it handles accessing to archive members. @@ -134,5 +223,33 @@ } }; } // namespace dsymutil + +template <> +struct DenseMapInfo { + + static inline dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy + getEmptyKey() { + return dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy(); + } + + static inline dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy + getTombstoneKey() { + return dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy(); + } + + static unsigned + getHashValue(const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &K) { + return hash_combine(DenseMapInfo::getHashValue(K.Filename), + DenseMapInfo::getHashValue( + K.Timestamp.time_since_epoch().count())); + } + + static bool + isEqual(const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &LHS, + const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &RHS) { + return LHS.Filename == RHS.Filename && LHS.Timestamp == RHS.Timestamp; + } +}; + } // namespace llvm #endif Index: llvm/tools/dsymutil/BinaryHolder.cpp =================================================================== --- llvm/tools/dsymutil/BinaryHolder.cpp +++ llvm/tools/dsymutil/BinaryHolder.cpp @@ -14,11 +14,21 @@ #include "BinaryHolder.h" #include "llvm/Object/MachO.h" +#include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" namespace llvm { namespace dsymutil { +static std::pair +getArchiveAndObjectName(StringRef Filename) { + StringRef Archive = Filename.substr(0, Filename.find('(')); + StringRef Object = Filename.substr(Archive.size() + 1).drop_back(); + return {Archive, Object}; +} + +static bool isArchive(StringRef Filename) { return Filename.endswith(")"); } + static std::vector getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem, object::MachOUniversalBinary &Fat) { @@ -32,6 +42,195 @@ return Buffers; } +Error CachedBinaryHolder::ArchiveEntry::load(StringRef Filename, + TimestampTy Timestamp, + bool Verbose) { + StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first; + + // Try to load archive and force it to be memory mapped. + auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(ArchiveFilename, -1, false); + if (auto Err = ErrOrBuff.getError()) + return errorCodeToError(Err); + + MemoryBuffer = std::move(*ErrOrBuff); + + if (Verbose) + WithColor::note() << "opened archive '" << ArchiveFilename << "'\n"; + + // Load one or more archive buffers, depending on whether we're dealing with + // a fat binary. + std::vector ArchiveBuffers; + + auto ErrOrFat = + object::MachOUniversalBinary::create(MemoryBuffer->getMemBufferRef()); + if (!ErrOrFat) { + consumeError(ErrOrFat.takeError()); + ArchiveBuffers.push_back(MemoryBuffer->getMemBufferRef()); + } else { + FatBinary = std::move(*ErrOrFat); + FatBinaryName = ArchiveFilename; + ArchiveBuffers = + getMachOFatMemoryBuffers(FatBinaryName, *MemoryBuffer, *FatBinary); + } + + // Finally, try to load the archives. + Archives.reserve(ArchiveBuffers.size()); + for (auto MemRef : ArchiveBuffers) { + auto ErrOrArchive = object::Archive::create(MemRef); + if (!ErrOrArchive) + return ErrOrArchive.takeError(); + Archives.push_back(std::move(*ErrOrArchive)); + } + + return Error::success(); +} + +Error CachedBinaryHolder::ObjectEntry::load(StringRef Filename, bool Verbose) { + // Try to load regular binary and force it to be memory mapped. + auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(Filename, -1, false); + if (auto Err = ErrOrBuff.getError()) + return errorCodeToError(Err); + + MemoryBuffer = std::move(*ErrOrBuff); + + if (Verbose) + WithColor::note() << "opened object.\n"; + + // Load one or more object buffers, depending on whether we're dealing with a + // fat binary. + std::vector ObjectBuffers; + + auto ErrOrFat = + object::MachOUniversalBinary::create(MemoryBuffer->getMemBufferRef()); + if (!ErrOrFat) { + consumeError(ErrOrFat.takeError()); + ObjectBuffers.push_back(MemoryBuffer->getMemBufferRef()); + } else { + FatBinary = std::move(*ErrOrFat); + FatBinaryName = Filename; + ObjectBuffers = + getMachOFatMemoryBuffers(FatBinaryName, *MemoryBuffer, *FatBinary); + } + + Objects.reserve(ObjectBuffers.size()); + for (auto MemRef : ObjectBuffers) { + auto ErrOrObjectFile = object::ObjectFile::createObjectFile(MemRef); + if (!ErrOrObjectFile) + return ErrOrObjectFile.takeError(); + Objects.push_back(std::move(*ErrOrObjectFile)); + } + + return Error::success(); +} + +Expected +CachedBinaryHolder::ObjectEntry::getObject(const Triple &T) { + for (const auto &Obj : Objects) { + if (const auto *MachO = dyn_cast(Obj.get())) { + if (MachO->getArchTriple().str() == T.str()) + return *MachO; + } else if (Obj->getArch() == T.getArch()) + return *Obj; + } + return errorCodeToError(object::object_error::arch_not_found); +} + +Expected +CachedBinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename, + TimestampTy Timestamp, + bool Verbose) { + StringRef ArchiveFilename; + StringRef ObjectFilename; + std::tie(ArchiveFilename, ObjectFilename) = getArchiveAndObjectName(Filename); + + // Try the cache first. + KeyTy Key = {ObjectFilename, Timestamp}; + + if (ObjectCache.count(Key)) + return ObjectCache[Key]; + + // Create a new ObjectEntry, but don't add it to the cache yet as loading + // the archive members might fail. + ObjectEntry OE; + for (const auto &Archive : Archives) { + Error Err = Error::success(); + for (auto Child : Archive->children(Err)) { + if (auto NameOrErr = Child.getName()) { + if (*NameOrErr == ObjectFilename) { + auto ModTimeOrErr = Child.getLastModified(); + if (!ModTimeOrErr) + return ModTimeOrErr.takeError(); + + if (Timestamp != sys::TimePoint<>() && + Timestamp != ModTimeOrErr.get()) { + if (Verbose) + WithColor::warning() << "member has timestamp mismatch.\n"; + continue; + } + + if (Verbose) + WithColor::note() << "found member in current archive.\n"; + + auto ErrOrMem = Child.getMemoryBufferRef(); + if (!ErrOrMem) + return ErrOrMem.takeError(); + + auto ErrOrObjectFile = + object::ObjectFile::createObjectFile(*ErrOrMem); + if (!ErrOrObjectFile) + return ErrOrObjectFile.takeError(); + + OE.Objects.push_back(std::move(*ErrOrObjectFile)); + } + } + } + if (Err) + return std::move(Err); + } + + if (OE.Objects.empty()) + return errorCodeToError(errc::no_such_file_or_directory); + + ObjectCache.try_emplace(Key, std::move(OE)); + return ObjectCache[Key]; +} + +Expected +CachedBinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { + if (Verbose) + WithColor::note() << "trying to open '" << Filename << "'\n"; + + // If this is an archive, we might have either the object or the archive + // cached. In this case we can load it without accessing the file system. + if (isArchive(Filename)) { + StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first; + if (!ArchiveCache.count(ArchiveFilename)) { + // Create a new ArchiveEntry, but don't add it to the cache yet as + // loading might fail. + ArchiveEntry AE; + auto Err = AE.load(Filename, Timestamp, Verbose); + if (Err) + return std::move(Err); + ArchiveCache[ArchiveFilename] = std::move(AE); + } + return ArchiveCache[ArchiveFilename].getObjectEntry(Filename, Timestamp); + } + + // If this is an object, we might have it cached. If not we'll have to load + // it from the file system and cache it now. + if (!ObjectCache.count(Filename)) { + // Create a new ObjectEntry, but don't add it to the cache yet as loading + // might fail. + ObjectEntry OE; + auto Err = OE.load(Filename); + if (Err) + return std::move(Err); + ObjectCache.try_emplace(Filename, std::move(OE)); + } + + return ObjectCache[Filename]; +} + void BinaryHolder::changeBackingMemoryBuffer( std::unique_ptr &&Buf) { CurrentArchives.clear(); Index: llvm/tools/dsymutil/DwarfLinker.cpp =================================================================== --- llvm/tools/dsymutil/DwarfLinker.cpp +++ llvm/tools/dsymutil/DwarfLinker.cpp @@ -1471,7 +1471,7 @@ class DwarfLinker { public: DwarfLinker(raw_fd_ostream &OutFile, const LinkOptions &Options) - : OutFile(OutFile), Options(Options) {} + : OutFile(OutFile), Options(Options), BinHolder(Options.Verbose) {} /// Link the contents of the DebugMap. bool link(const DebugMap &); @@ -1553,7 +1553,6 @@ /// Keeps track of data associated with one object during linking. struct LinkContext { DebugMapObject &DMO; - BinaryHolder BinHolder; const object::ObjectFile *ObjectFile; RelocationManager RelocMgr; std::unique_ptr DwarfContext; @@ -1561,8 +1560,8 @@ UnitListTy CompileUnits; LinkContext(const DebugMap &Map, DwarfLinker &Linker, DebugMapObject &DMO, - bool Verbose = false) - : DMO(DMO), BinHolder(Verbose), RelocMgr(Linker) { + CachedBinaryHolder &BinHolder) + : DMO(DMO), RelocMgr(Linker) { // Swift ASTs are not object files. if (DMO.getType() == MachO::N_AST) { ObjectFile = nullptr; @@ -1855,14 +1854,15 @@ bool createStreamer(const Triple &TheTriple, raw_fd_ostream &OutFile); /// Attempt to load a debug object from disk. - ErrorOr loadObject(BinaryHolder &BinaryHolder, - const DebugMapObject &Obj, - const DebugMap &Map); + ErrorOr + loadObject(CachedBinaryHolder &BinaryHolder, const DebugMapObject &Obj, + const DebugMap &Map); /// @} raw_fd_ostream &OutFile; LinkOptions Options; std::unique_ptr Streamer; + CachedBinaryHolder BinHolder; uint64_t OutputDebugInfoSize; unsigned MaxDwarfVersion = 0; @@ -3960,18 +3960,26 @@ } ErrorOr -DwarfLinker::loadObject(BinaryHolder &BinaryHolder, const DebugMapObject &Obj, - const DebugMap &Map) { - auto ErrOrObjs = - BinaryHolder.GetObjectFiles(Obj.getObjectFilename(), Obj.getTimestamp()); - if (std::error_code EC = ErrOrObjs.getError()) { - reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message(), Obj); - return EC; +DwarfLinker::loadObject(CachedBinaryHolder &BinaryHolder, + const DebugMapObject &Obj, const DebugMap &Map) { + auto ObjectEntry = + BinaryHolder.getObjectEntry(Obj.getObjectFilename(), Obj.getTimestamp()); + if (!ObjectEntry) { + auto Err = ObjectEntry.takeError(); + reportWarning( + Twine(Obj.getObjectFilename()) + ": " + toString(std::move(Err)), Obj); + return errorToErrorCode(std::move(Err)); } - auto ErrOrObj = BinaryHolder.Get(Map.getTriple()); - if (std::error_code EC = ErrOrObj.getError()) - reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message(), Obj); - return ErrOrObj; + + auto Object = ObjectEntry->getObject(Map.getTriple()); + if (!Object) { + auto Err = Object.takeError(); + reportWarning( + Twine(Obj.getObjectFilename()) + ": " + toString(std::move(Err)), Obj); + return errorToErrorCode(std::move(Err)); + } + + return *Object; } Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath, @@ -3987,7 +3995,9 @@ sys::path::append(Path, ModulePath, Filename); else sys::path::append(Path, Filename); - BinaryHolder ObjHolder(Options.Verbose); + // Don't use the cached binary holder because we have no thread-safety + // guarantee and the lifetime is limited. + CachedBinaryHolder ObjHolder(Options.Verbose); auto &Obj = ModuleMap.addDebugMapObject( Path, sys::TimePoint(), MachO::N_OSO); auto ErrOrObj = loadObject(ObjHolder, Obj, ModuleMap); @@ -4216,7 +4226,7 @@ std::vector ObjectContexts; ObjectContexts.reserve(NumObjects); for (const auto &Obj : Map.objects()) - ObjectContexts.emplace_back(Map, *this, *Obj.get(), Options.Verbose); + ObjectContexts.emplace_back(Map, *this, *Obj.get(), BinHolder); // This Dwarf string pool which is only used for uniquing. This one should // never be used for offsets as its not thread-safe or predictable.