Index: llvm/tools/dsymutil/BinaryHolder.h
===================================================================
--- llvm/tools/dsymutil/BinaryHolder.h
+++ llvm/tools/dsymutil/BinaryHolder.h
@@ -14,6 +14,7 @@
#ifndef LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
#define LLVM_TOOLS_DSYMUTIL_BINARYHOLDER_H
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/Error.h"
@@ -26,6 +27,94 @@
namespace llvm {
namespace dsymutil {
+class CachedBinaryHolder {
+public:
+ using TimestampTy = sys::TimePoint<std::chrono::seconds>;
+
+ CachedBinaryHolder(bool Verbose = false) : Verbose(Verbose) {}
+
+ class EntryBase {
+ protected:
+ std::unique_ptr<MemoryBuffer> MemoryBuffer;
+ std::unique_ptr<object::MachOUniversalBinary> FatBinary;
+ std::string FatBinaryName;
+ };
+
+ struct ObjectEntry : public EntryBase {
+ std::vector<std::unique_ptr<object::ObjectFile>> Objects;
+
+ Error load(StringRef Filename, bool Verbose = false);
+
+ /// Access all owned ObjectFiles.
+ std::vector<const object::ObjectFile *> getObjects() {
+ std::vector<const object::ObjectFile *> Result;
+ Result.reserve(Objects.size());
+ for (auto &Object : Objects) {
+ Result.push_back(Object.get());
+ }
+ return Result;
+ }
+
+ /// Access to a derived version of all the currently owned ObjectFiles. The
+ /// conversion might be invalid, in which case an Error is returned.
+ template <typename ObjectFileType>
+ Expected<std::vector<const ObjectFileType *>> getObjectsAs() {
+ std::vector<const object::ObjectFile *> Result;
+ Result.reserve(Objects.size());
+ for (auto &Object : Objects) {
+ const auto *Derived = dyn_cast<ObjectFileType>(Object.get());
+ if (!Derived)
+ return errorCodeToError(object::object_error::invalid_file_type);
+ Result.push_back(Derived);
+ }
+ return Result;
+ }
+
+ /// Access the owned ObjectFile with architecture \p T.
+ Expected<const object::ObjectFile &> getObject(const Triple &T);
+
+ /// Access to a derived version of the currently owned ObjectFile with
+ /// architecture \p T. The conversion must be known to be valid.
+ template <typename ObjectFileType>
+ Expected<const ObjectFileType &> getObjectAs(const Triple &T) {
+ auto Object = getObject(T);
+ if (!Object)
+ return Object.takeError();
+ return cast<ObjectFileType>(*Object);
+ }
+ };
+
+ struct ArchiveEntry : public EntryBase {
+ struct KeyTy {
+ StringRef Filename;
+ TimestampTy Timestamp;
+ };
+
+ Error load(StringRef Filename, TimestampTy Timestamp, bool Verbose = false);
+
+ Expected<ObjectEntry &> getObjectEntry(StringRef Filename,
+ TimestampTy Timestamp,
+ bool Verbose = false);
+
+ private:
+ std::vector<std::unique_ptr<object::Archive>> Archives;
+ DenseMap<KeyTy, ObjectEntry> ObjectCache;
+ };
+
+ Expected<ObjectEntry &> getObjectEntry(StringRef Filename,
+ TimestampTy Timestamp);
+
+private:
+ /// Cache of static archives. Objects that are part of a static archive are
+ /// stored under this object, rather than in the map below.
+ StringMap<ArchiveEntry> ArchiveCache;
+
+ /// Object entries for objects that are not in a static archive.
+ StringMap<ObjectEntry> ObjectCache;
+
+ bool Verbose;
+};
+
/// The BinaryHolder class is responsible for creating and owning ObjectFile
/// objects and their underlying MemoryBuffer. This is different from a simple
/// OwningBinary in that it handles accessing to archive members.
@@ -134,5 +223,33 @@
}
};
} // namespace dsymutil
+
+template <>
+struct DenseMapInfo<dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy> {
+
+ static inline dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy
+ getEmptyKey() {
+ return dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy();
+ }
+
+ static inline dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy
+ getTombstoneKey() {
+ return dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy();
+ }
+
+ static unsigned
+ getHashValue(const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &K) {
+ return hash_combine(DenseMapInfo<StringRef>::getHashValue(K.Filename),
+ DenseMapInfo<unsigned>::getHashValue(
+ K.Timestamp.time_since_epoch().count()));
+ }
+
+ static bool
+ isEqual(const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &LHS,
+ const dsymutil::CachedBinaryHolder::ArchiveEntry::KeyTy &RHS) {
+ return LHS.Filename == RHS.Filename && LHS.Timestamp == RHS.Timestamp;
+ }
+};
+
} // namespace llvm
#endif
Index: llvm/tools/dsymutil/BinaryHolder.cpp
===================================================================
--- llvm/tools/dsymutil/BinaryHolder.cpp
+++ llvm/tools/dsymutil/BinaryHolder.cpp
@@ -14,11 +14,21 @@
#include "BinaryHolder.h"
#include "llvm/Object/MachO.h"
+#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
namespace llvm {
namespace dsymutil {
+static std::pair<StringRef, StringRef>
+getArchiveAndObjectName(StringRef Filename) {
+ StringRef Archive = Filename.substr(0, Filename.find('('));
+ StringRef Object = Filename.substr(Archive.size() + 1).drop_back();
+ return {Archive, Object};
+}
+
+static bool isArchive(StringRef Filename) { return Filename.endswith(")"); }
+
static std::vector<MemoryBufferRef>
getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem,
object::MachOUniversalBinary &Fat) {
@@ -32,6 +42,195 @@
return Buffers;
}
+Error CachedBinaryHolder::ArchiveEntry::load(StringRef Filename,
+ TimestampTy Timestamp,
+ bool Verbose) {
+ StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first;
+
+ // Try to load archive and force it to be memory mapped.
+ auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(ArchiveFilename, -1, false);
+ if (auto Err = ErrOrBuff.getError())
+ return errorCodeToError(Err);
+
+ MemoryBuffer = std::move(*ErrOrBuff);
+
+ if (Verbose)
+ WithColor::note() << "opened archive '" << ArchiveFilename << "'\n";
+
+ // Load one or more archive buffers, depending on whether we're dealing with
+ // a fat binary.
+ std::vector<MemoryBufferRef> ArchiveBuffers;
+
+ auto ErrOrFat =
+ object::MachOUniversalBinary::create(MemoryBuffer->getMemBufferRef());
+ if (!ErrOrFat) {
+ consumeError(ErrOrFat.takeError());
+ ArchiveBuffers.push_back(MemoryBuffer->getMemBufferRef());
+ } else {
+ FatBinary = std::move(*ErrOrFat);
+ FatBinaryName = ArchiveFilename;
+ ArchiveBuffers =
+ getMachOFatMemoryBuffers(FatBinaryName, *MemoryBuffer, *FatBinary);
+ }
+
+ // Finally, try to load the archives.
+ Archives.reserve(ArchiveBuffers.size());
+ for (auto MemRef : ArchiveBuffers) {
+ auto ErrOrArchive = object::Archive::create(MemRef);
+ if (!ErrOrArchive)
+ return ErrOrArchive.takeError();
+ Archives.push_back(std::move(*ErrOrArchive));
+ }
+
+ return Error::success();
+}
+
+Error CachedBinaryHolder::ObjectEntry::load(StringRef Filename, bool Verbose) {
+ // Try to load regular binary and force it to be memory mapped.
+ auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(Filename, -1, false);
+ if (auto Err = ErrOrBuff.getError())
+ return errorCodeToError(Err);
+
+ MemoryBuffer = std::move(*ErrOrBuff);
+
+ if (Verbose)
+ WithColor::note() << "opened object.\n";
+
+ // Load one or more object buffers, depending on whether we're dealing with a
+ // fat binary.
+ std::vector<MemoryBufferRef> ObjectBuffers;
+
+ auto ErrOrFat =
+ object::MachOUniversalBinary::create(MemoryBuffer->getMemBufferRef());
+ if (!ErrOrFat) {
+ consumeError(ErrOrFat.takeError());
+ ObjectBuffers.push_back(MemoryBuffer->getMemBufferRef());
+ } else {
+ FatBinary = std::move(*ErrOrFat);
+ FatBinaryName = Filename;
+ ObjectBuffers =
+ getMachOFatMemoryBuffers(FatBinaryName, *MemoryBuffer, *FatBinary);
+ }
+
+ Objects.reserve(ObjectBuffers.size());
+ for (auto MemRef : ObjectBuffers) {
+ auto ErrOrObjectFile = object::ObjectFile::createObjectFile(MemRef);
+ if (!ErrOrObjectFile)
+ return ErrOrObjectFile.takeError();
+ Objects.push_back(std::move(*ErrOrObjectFile));
+ }
+
+ return Error::success();
+}
+
+Expected<const object::ObjectFile &>
+CachedBinaryHolder::ObjectEntry::getObject(const Triple &T) {
+ for (const auto &Obj : Objects) {
+ if (const auto *MachO = dyn_cast<object::MachOObjectFile>(Obj.get())) {
+ if (MachO->getArchTriple().str() == T.str())
+ return *MachO;
+ } else if (Obj->getArch() == T.getArch())
+ return *Obj;
+ }
+ return errorCodeToError(object::object_error::arch_not_found);
+}
+
+Expected<CachedBinaryHolder::ObjectEntry &>
+CachedBinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename,
+ TimestampTy Timestamp,
+ bool Verbose) {
+ StringRef ArchiveFilename;
+ StringRef ObjectFilename;
+ std::tie(ArchiveFilename, ObjectFilename) = getArchiveAndObjectName(Filename);
+
+ // Try the cache first.
+ KeyTy Key = {ObjectFilename, Timestamp};
+
+ if (ObjectCache.count(Key))
+ return ObjectCache[Key];
+
+ // Create a new ObjectEntry, but don't add it to the cache yet as loading
+ // the archive members might fail.
+ ObjectEntry OE;
+ for (const auto &Archive : Archives) {
+ Error Err = Error::success();
+ for (auto Child : Archive->children(Err)) {
+ if (auto NameOrErr = Child.getName()) {
+ if (*NameOrErr == ObjectFilename) {
+ auto ModTimeOrErr = Child.getLastModified();
+ if (!ModTimeOrErr)
+ return ModTimeOrErr.takeError();
+
+ if (Timestamp != sys::TimePoint<>() &&
+ Timestamp != ModTimeOrErr.get()) {
+ if (Verbose)
+ WithColor::warning() << "member has timestamp mismatch.\n";
+ continue;
+ }
+
+ if (Verbose)
+ WithColor::note() << "found member in current archive.\n";
+
+ auto ErrOrMem = Child.getMemoryBufferRef();
+ if (!ErrOrMem)
+ return ErrOrMem.takeError();
+
+ auto ErrOrObjectFile =
+ object::ObjectFile::createObjectFile(*ErrOrMem);
+ if (!ErrOrObjectFile)
+ return ErrOrObjectFile.takeError();
+
+ OE.Objects.push_back(std::move(*ErrOrObjectFile));
+ }
+ }
+ }
+ if (Err)
+ return std::move(Err);
+ }
+
+ if (OE.Objects.empty())
+ return errorCodeToError(errc::no_such_file_or_directory);
+
+ ObjectCache.try_emplace(Key, std::move(OE));
+ return ObjectCache[Key];
+}
+
+Expected<CachedBinaryHolder::ObjectEntry &>
+CachedBinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) {
+ if (Verbose)
+ WithColor::note() << "trying to open '" << Filename << "'\n";
+
+ // If this is an archive, we might have either the object or the archive
+ // cached. In this case we can load it without accessing the file system.
+ if (isArchive(Filename)) {
+ StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first;
+ if (!ArchiveCache.count(ArchiveFilename)) {
+ // Create a new ArchiveEntry, but don't add it to the cache yet as
+ // loading might fail.
+ ArchiveEntry AE;
+ auto Err = AE.load(Filename, Timestamp, Verbose);
+ if (Err)
+ return std::move(Err);
+ ArchiveCache[ArchiveFilename] = std::move(AE);
+ }
+ return ArchiveCache[ArchiveFilename].getObjectEntry(Filename, Timestamp);
+ }
+
+ // If this is an object, we might have it cached. If not we'll have to load
+ // it from the file system and cache it now.
+ if (!ObjectCache.count(Filename)) {
+ // Create a new ObjectEntry, but don't add it to the cache yet as loading
+ // might fail.
+ ObjectEntry OE;
+ auto Err = OE.load(Filename);
+ if (Err)
+ return std::move(Err);
+ ObjectCache.try_emplace(Filename, std::move(OE));
+ }
+
+ return ObjectCache[Filename];
+}
+
void BinaryHolder::changeBackingMemoryBuffer(
std::unique_ptr<MemoryBuffer> &&Buf) {
CurrentArchives.clear();
Index: llvm/tools/dsymutil/DwarfLinker.cpp
===================================================================
--- llvm/tools/dsymutil/DwarfLinker.cpp
+++ llvm/tools/dsymutil/DwarfLinker.cpp
@@ -1471,7 +1471,7 @@
class DwarfLinker {
public:
DwarfLinker(raw_fd_ostream &OutFile, const LinkOptions &Options)
- : OutFile(OutFile), Options(Options) {}
+ : OutFile(OutFile), Options(Options), BinHolder(Options.Verbose) {}
/// Link the contents of the DebugMap.
bool link(const DebugMap &);
@@ -1553,7 +1553,6 @@
/// Keeps track of data associated with one object during linking.
struct LinkContext {
DebugMapObject &DMO;
- BinaryHolder BinHolder;
const object::ObjectFile *ObjectFile;
RelocationManager RelocMgr;
std::unique_ptr<DWARFContext> DwarfContext;
@@ -1561,8 +1560,8 @@
UnitListTy CompileUnits;
LinkContext(const DebugMap &Map, DwarfLinker &Linker, DebugMapObject &DMO,
- bool Verbose = false)
- : DMO(DMO), BinHolder(Verbose), RelocMgr(Linker) {
+ CachedBinaryHolder &BinHolder)
+ : DMO(DMO), RelocMgr(Linker) {
// Swift ASTs are not object files.
if (DMO.getType() == MachO::N_AST) {
ObjectFile = nullptr;
@@ -1855,14 +1854,15 @@
bool createStreamer(const Triple &TheTriple, raw_fd_ostream &OutFile);
/// Attempt to load a debug object from disk.
- ErrorOr<const object::ObjectFile &> loadObject(BinaryHolder &BinaryHolder,
- const DebugMapObject &Obj,
- const DebugMap &Map);
+ ErrorOr<const object::ObjectFile &>
+ loadObject(CachedBinaryHolder &BinaryHolder, const DebugMapObject &Obj,
+ const DebugMap &Map);
/// @}
raw_fd_ostream &OutFile;
LinkOptions Options;
std::unique_ptr<DwarfStreamer> Streamer;
+ CachedBinaryHolder BinHolder;
uint64_t OutputDebugInfoSize;
unsigned MaxDwarfVersion = 0;
@@ -3960,18 +3960,26 @@
}
ErrorOr<const object::ObjectFile &>
-DwarfLinker::loadObject(BinaryHolder &BinaryHolder, const DebugMapObject &Obj,
- const DebugMap &Map) {
- auto ErrOrObjs =
- BinaryHolder.GetObjectFiles(Obj.getObjectFilename(), Obj.getTimestamp());
- if (std::error_code EC = ErrOrObjs.getError()) {
- reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message(), Obj);
- return EC;
+DwarfLinker::loadObject(CachedBinaryHolder &BinaryHolder,
+ const DebugMapObject &Obj, const DebugMap &Map) {
+ auto ObjectEntry =
+ BinaryHolder.getObjectEntry(Obj.getObjectFilename(), Obj.getTimestamp());
+ if (!ObjectEntry) {
+ auto Err = ObjectEntry.takeError();
+ reportWarning(
+ Twine(Obj.getObjectFilename()) + ": " + toString(std::move(Err)), Obj);
+ return errorToErrorCode(std::move(Err));
}
- auto ErrOrObj = BinaryHolder.Get(Map.getTriple());
- if (std::error_code EC = ErrOrObj.getError())
- reportWarning(Twine(Obj.getObjectFilename()) + ": " + EC.message(), Obj);
- return ErrOrObj;
+
+ auto Object = ObjectEntry->getObject(Map.getTriple());
+ if (!Object) {
+ auto Err = Object.takeError();
+ reportWarning(
+ Twine(Obj.getObjectFilename()) + ": " + toString(std::move(Err)), Obj);
+ return errorToErrorCode(std::move(Err));
+ }
+
+ return *Object;
}
Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
@@ -3987,7 +3995,9 @@
sys::path::append(Path, ModulePath, Filename);
else
sys::path::append(Path, Filename);
- BinaryHolder ObjHolder(Options.Verbose);
+ // Don't use the cached binary holder because we have no thread-safety
+ // guarantee and the lifetime is limited.
+ CachedBinaryHolder ObjHolder(Options.Verbose);
auto &Obj = ModuleMap.addDebugMapObject(
Path, sys::TimePoint<std::chrono::seconds>(), MachO::N_OSO);
auto ErrOrObj = loadObject(ObjHolder, Obj, ModuleMap);
@@ -4216,7 +4226,7 @@
std::vector<LinkContext> ObjectContexts;
ObjectContexts.reserve(NumObjects);
for (const auto &Obj : Map.objects())
- ObjectContexts.emplace_back(Map, *this, *Obj.get(), Options.Verbose);
+ ObjectContexts.emplace_back(Map, *this, *Obj.get(), BinHolder);
// This Dwarf string pool which is only used for uniquing. This one should
// never be used for offsets as its not thread-safe or predictable.