[dsymutil] Thread the VFS through dsymutil (NFC)

[JDevlieghere]

Summary

This patch threads the virtual file system through dsymutil. Currently there is no good way to find out exactly what files are necessary in order to reproduce a dsymutil link, at least not without knowledge of how dsymutil works. My goal is add lightweight reproducer functionality that automatically gather the input object files using the FileCollectorFileSystem. We can then also use the VFS mapping to reproduce a dsymutil link without having to mess with the OSO prefix path.

[JDevlieghere]

Remove spurious line which I used for testing.

[friss]

I don't know much about the VFS itself, but those ,technical changes look good

[aprantl]

llvm/tools/dsymutil/BinaryHolder.h

140

Perhaps add ... which is used for generating reproducers.

llvm/tools/dsymutil/BinaryHolder.h

 #include "llvm/ADT/Triple.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 #include <mutex>
 
 namespace llvm {
 namespace dsymutil {
 
 /// The BinaryHolder class is responsible for creating and owning
 /// ObjectFiles and their underlying MemoryBuffers. It differs from a simple
 /// OwningBinary in that it handles accessing and caching of archives and its
 /// members.
 class BinaryHolder {
 public:
   using TimestampTy = sys::TimePoint<std::chrono::seconds>;
 
-  BinaryHolder(bool Verbose = false) : Verbose(Verbose) {}
+  BinaryHolder(IntrusiveRefCntPtr<vfs::FileSystem> VFS, bool Verbose = false)
+      : VFS(VFS), Verbose(Verbose) {}
 
   // Forward declarations for friend declaration.
   class ObjectEntry;
   class ArchiveEntry;
 
   /// Base class shared by cached entries, representing objects and archives.
   class EntryBase {
   protected:
     std::unique_ptr<MemoryBuffer> MemBuffer;
     std::unique_ptr<object::MachOUniversalBinary> FatBinary;
     std::string FatBinaryName;
   };
 
   /// Cached entry holding one or more (in case of a fat binary) object files.
   class ObjectEntry : public EntryBase {
   public:
     /// Load the given object binary in memory.
-    Error load(StringRef Filename, bool Verbose = false);
+    Error load(IntrusiveRefCntPtr<vfs::FileSystem> VFS, StringRef Filename,
+               bool Verbose = false);
 
     /// Access all owned ObjectFiles.
     std::vector<const object::ObjectFile *> getObjects() const;
 
     /// Access to a derived version of all the currently owned ObjectFiles. The
     /// conversion might be invalid, in which case an Error is returned.
     template <typename ObjectFileType>
     Expected<std::vector<const ObjectFileType *>> getObjectsAs() const {
struct KeyTy {
       TimestampTy Timestamp;
 
       KeyTy() : Filename(), Timestamp() {}
       KeyTy(StringRef Filename, TimestampTy Timestamp)
           : Filename(Filename.str()), Timestamp(Timestamp) {}
     };
 
     /// Load the given object binary in memory.
-    Error load(StringRef Filename, TimestampTy Timestamp, bool Verbose = false);
+    Error load(IntrusiveRefCntPtr<vfs::FileSystem> VFS, StringRef Filename,
+               TimestampTy Timestamp, bool Verbose = false);
 
     Expected<const ObjectEntry &> getObjectEntry(StringRef Filename,
                                                  TimestampTy Timestamp,
                                                  bool Verbose = false);
 
   private:
     std::vector<std::unique_ptr<object::Archive>> Archives;
     DenseMap<KeyTy, ObjectEntry> MemberCache;
private:
   /// stored under this object, rather than in the map below.
   StringMap<ArchiveEntry> ArchiveCache;
   std::mutex ArchiveCacheMutex;
 
   /// Object entries for objects that are not in a static archive.
   StringMap<ObjectEntry> ObjectCache;
   std::mutex ObjectCacheMutex;
 
+  /// Virtual File System instance.
+  IntrusiveRefCntPtr<vfs::FileSystem> VFS;
+
   bool Verbose;
 };
 
 } // namespace dsymutil
 
 template <> struct DenseMapInfo<dsymutil::BinaryHolder::ArchiveEntry::KeyTy> {
 
   static inline dsymutil::BinaryHolder::ArchiveEntry::KeyTy getEmptyKey() {

llvm/tools/dsymutil/BinaryHolder.cpp

getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem,
   for (auto It = Fat.begin_objects(), End = Fat.end_objects(); It != End;
        ++It) {
     StringRef ObjData = FatData.substr(It->getOffset(), It->getSize());
     Buffers.emplace_back(ObjData, Filename);
   }
   return Buffers;
 }
 
-Error BinaryHolder::ArchiveEntry::load(StringRef Filename,
+Error BinaryHolder::ArchiveEntry::load(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
+                                       StringRef Filename,
                                        TimestampTy Timestamp, bool Verbose) {
   StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first;
 
   // Try to load archive and force it to be memory mapped.
-  auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(ArchiveFilename, -1, false);
+  auto ErrOrBuff = (ArchiveFilename == "-")
+                       ? MemoryBuffer::getSTDIN()
+                       : VFS->getBufferForFile(ArchiveFilename, -1, false);
   if (auto Err = ErrOrBuff.getError())
     return errorCodeToError(Err);
 
   MemBuffer = std::move(*ErrOrBuff);
 
   if (Verbose)
     WithColor::note() << "loaded archive '" << ArchiveFilename << "'\n";
 
for (auto MemRef : ArchiveBuffers) {
     if (!ErrOrArchive)
       return ErrOrArchive.takeError();
     Archives.push_back(std::move(*ErrOrArchive));
   }
 
   return Error::success();
 }
 
-Error BinaryHolder::ObjectEntry::load(StringRef Filename, bool Verbose) {
+Error BinaryHolder::ObjectEntry::load(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
+                                      StringRef Filename, bool Verbose) {
   // Try to load regular binary and force it to be memory mapped.
-  auto ErrOrBuff = MemoryBuffer::getFileOrSTDIN(Filename, -1, false);
+  auto ErrOrBuff = (Filename == "-")
+                       ? MemoryBuffer::getSTDIN()
+                       : VFS->getBufferForFile(Filename, -1, false);
   if (auto Err = ErrOrBuff.getError())
     return errorCodeToError(Err);
 
   MemBuffer = std::move(*ErrOrBuff);
 
   if (Verbose)
     WithColor::note() << "loaded object.\n";
 
BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) {
   if (isArchive(Filename)) {
     StringRef ArchiveFilename = getArchiveAndObjectName(Filename).first;
     std::lock_guard<std::mutex> Lock(ArchiveCacheMutex);
     if (ArchiveCache.count(ArchiveFilename)) {
       return ArchiveCache[ArchiveFilename].getObjectEntry(Filename, Timestamp,
                                                           Verbose);
     } else {
       ArchiveEntry &AE = ArchiveCache[ArchiveFilename];
-      auto Err = AE.load(Filename, Timestamp, Verbose);
+      auto Err = AE.load(VFS, Filename, Timestamp, Verbose);
       if (Err) {
         ArchiveCache.erase(ArchiveFilename);
         // Don't return the error here: maybe the file wasn't an archive.
         llvm::consumeError(std::move(Err));
       } else {
         return ArchiveCache[ArchiveFilename].getObjectEntry(Filename, Timestamp,
                                                             Verbose);
       }
     }
   }
 
   // If this is an object, we might have it cached. If not we'll have to load
   // it from the file system and cache it now.
   std::lock_guard<std::mutex> Lock(ObjectCacheMutex);
   if (!ObjectCache.count(Filename)) {
     ObjectEntry &OE = ObjectCache[Filename];
-    auto Err = OE.load(Filename, Verbose);
+    auto Err = OE.load(VFS, Filename, Verbose);
     if (Err) {
       ObjectCache.erase(Filename);
       return std::move(Err);
     }
   }
 
   return ObjectCache[Filename];
 }

llvm/tools/dsymutil/DebugMap.cpp

MappingTraits<dsymutil::DebugMapObject>::YamlDMO::YamlDMO(
   Entries.reserve(Obj.Symbols.size());
   for (auto &Entry : Obj.Symbols)
     Entries.push_back(
         std::make_pair(std::string(Entry.getKey()), Entry.getValue()));
 }
 
 dsymutil::DebugMapObject
 MappingTraits<dsymutil::DebugMapObject>::YamlDMO::denormalize(IO &IO) {
-  BinaryHolder BinHolder(/* Verbose =*/false);
+  BinaryHolder BinHolder(vfs::getRealFileSystem(), /* Verbose =*/false);
   const auto &Ctxt = *reinterpret_cast<YAMLContext *>(IO.getContext());
   SmallString<80> Path(Ctxt.PrependPath);
   StringMap<uint64_t> SymbolAddresses;
 
   sys::path::append(Path, Filename);
 
   auto ObjectEntry = BinHolder.getObjectEntry(Path);
   if (!ObjectEntry) {

llvm/tools/dsymutil/DwarfLinkerForBinary.cpp

if (Options.ResourceDir && !ParseableSwiftInterfaces.empty()) {
     if (auto E =
             copySwiftInterfaces(ParseableSwiftInterfaces, ArchName, Options))
       return error(toString(std::move(E)));
   }
 
   if (Map.getTriple().isOSDarwin() && !Map.getBinaryPath().empty() &&
       Options.FileType == OutputFileType::Object)
     return MachOUtils::generateDsymCompanion(
-        Map, Options.Translator, *Streamer->getAsmPrinter().OutStreamer,
-        OutFile);
+        Options.VFS, Map, Options.Translator,
+        *Streamer->getAsmPrinter().OutStreamer, OutFile);
 
   Streamer->finish();
   return true;
 }
 
 static bool isMachOPairedReloc(uint64_t RelocType, uint64_t Arch) {
   switch (Arch) {
   case Triple::x86:

llvm/tools/dsymutil/LinkUtils.h

 //===- tools/dsymutil/LinkUtils.h - Dwarf linker utilities ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_DSYMUTIL_LINKOPTIONS_H
 #define LLVM_TOOLS_DSYMUTIL_LINKOPTIONS_H
 
 #include "SymbolMap.h"
 
 #include "llvm/ADT/Twine.h"
 #include "llvm/Remarks/RemarkFormat.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/WithColor.h"
 
 #include "llvm/DWARFLinker/DWARFLinker.h"
 #include "llvm/DWARFLinker/DWARFStreamer.h"
 #include <string>
 
 namespace llvm {
 namespace dsymutil {
struct LinkOptions {
   std::map<std::string, std::string> ObjectPrefixMap;
 
   /// The Resources directory in the .dSYM bundle.
   Optional<std::string> ResourceDir;
 
   /// Symbol map translator.
   SymbolMapTranslator Translator;
 
+  /// Virtual File System.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
+      vfs::getRealFileSystem();
+
   /// Fields used for linking and placing remarks into the .dSYM bundle.
   /// @{
 
   /// Number of debug maps processed in total.
   unsigned NumDebugMaps = 0;
 
   /// -remarks-prepend-path: prepend a path to all the external remark file
   /// paths found in remark metadata.

llvm/tools/dsymutil/MachODebugMapParser.cpp

 
 namespace {
 using namespace llvm;
 using namespace llvm::dsymutil;
 using namespace llvm::object;
 
 class MachODebugMapParser {
 public:
-  MachODebugMapParser(StringRef BinaryPath, ArrayRef<std::string> Archs,
+  MachODebugMapParser(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+                      StringRef BinaryPath, ArrayRef<std::string> Archs,
                       StringRef PathPrefix = "",
                       bool PaperTrailWarnings = false, bool Verbose = false)
       : BinaryPath(std::string(BinaryPath)), Archs(Archs.begin(), Archs.end()),
         PathPrefix(std::string(PathPrefix)),
-        PaperTrailWarnings(PaperTrailWarnings), BinHolder(Verbose),
+        PaperTrailWarnings(PaperTrailWarnings), BinHolder(VFS, Verbose),
         CurrentDebugMapObject(nullptr) {}
 
   /// Parses and returns the DebugMaps of the input binary. The binary contains
   /// multiple maps in case it is a universal binary.
   /// \returns an error in case the provided BinaryPath doesn't exist
   /// or isn't of a supported type.
   ErrorOr<std::vector<std::unique_ptr<DebugMap>>> parse();
 
static std::string getArchName(const object::MachOObjectFile &Obj) {
   return std::string(T.getArchName());
 }
 
 std::unique_ptr<DebugMap>
 MachODebugMapParser::parseOneBinary(const MachOObjectFile &MainBinary,
                                     StringRef BinaryPath) {
   loadMainBinarySymbols(MainBinary);
   ArrayRef<uint8_t> UUID = MainBinary.getUuid();
-  Result = std::make_unique<DebugMap>(MainBinary.getArchTriple(), BinaryPath, UUID);
+  Result =
+      std::make_unique<DebugMap>(MainBinary.getArchTriple(), BinaryPath, UUID);
   MainBinaryStrings = MainBinary.getStringTableData();
   for (const SymbolRef &Symbol : MainBinary.symbols()) {
     const DataRefImpl &DRI = Symbol.getRawDataRefImpl();
     if (MainBinary.is64Bit())
       handleStabDebugMapEntry(MainBinary.getSymbol64TableEntry(DRI));
     else
       handleStabDebugMapEntry(MainBinary.getSymbolTableEntry(DRI));
   }
for (const auto &Sym : MainBinary.symbols()) {
     else
       MainBinarySymbolAddresses.try_emplace(Name, Addr);
   }
 }
 
 namespace llvm {
 namespace dsymutil {
 llvm::ErrorOr<std::vector<std::unique_ptr<DebugMap>>>
-parseDebugMap(StringRef InputFile, ArrayRef<std::string> Archs,
+parseDebugMap(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+              StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath, bool PaperTrailWarnings, bool Verbose,
               bool InputIsYAML) {
   if (InputIsYAML)
     return DebugMap::parseYAMLDebugMap(InputFile, PrependPath, Verbose);
 
-  MachODebugMapParser Parser(InputFile, Archs, PrependPath, PaperTrailWarnings,
-                             Verbose);
+  MachODebugMapParser Parser(VFS, InputFile, Archs, PrependPath,
+                             PaperTrailWarnings, Verbose);
   return Parser.parse();
 }
 
-bool dumpStab(StringRef InputFile, ArrayRef<std::string> Archs,
+bool dumpStab(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+              StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath) {
-  MachODebugMapParser Parser(InputFile, Archs, PrependPath, false);
+  MachODebugMapParser Parser(VFS, InputFile, Archs, PrependPath, false);
   return Parser.dumpStab();
 }
 } // namespace dsymutil
 } // namespace llvm

llvm/tools/dsymutil/MachOUtils.h

 //===-- MachOUtils.h - Mach-o specific helpers for dsymutil  --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H
 #define LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H
 
 #include "SymbolMap.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 #include <string>
 
 namespace llvm {
 class MCStreamer;
 class raw_fd_ostream;
 namespace dsymutil {
 class DebugMap;
struct ArchAndFile {
   ArchAndFile &operator=(ArchAndFile &&A) = default;
   ~ArchAndFile();
 };
 
 bool generateUniversalBinary(SmallVectorImpl<ArchAndFile> &ArchFiles,
                              StringRef OutputFileName, const LinkOptions &,
                              StringRef SDKPath);
 
-bool generateDsymCompanion(const DebugMap &DM, SymbolMapTranslator &Translator,
+bool generateDsymCompanion(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+                           const DebugMap &DM, SymbolMapTranslator &Translator,
                            MCStreamer &MS, raw_fd_ostream &OutFile);
 
 std::string getArchName(StringRef Arch);
 } // namespace MachOUtils
 } // namespace dsymutil
 } // namespace llvm
 #endif // LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H

llvm/tools/dsymutil/MachOUtils.cpp

return sizeof(MachO::segment_command_64) +
            NumSections * sizeof(MachO::section_64);
 
   return sizeof(MachO::segment_command) + NumSections * sizeof(MachO::section);
 }
 
 // Stream a dSYM companion binary file corresponding to the binary referenced
 // by \a DM to \a OutFile. The passed \a MS MCStreamer is setup to write to
 // \a OutFile and it must be using a MachObjectWriter object to do so.
-bool generateDsymCompanion(const DebugMap &DM, SymbolMapTranslator &Translator,
+bool generateDsymCompanion(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+                           const DebugMap &DM, SymbolMapTranslator &Translator,
                            MCStreamer &MS, raw_fd_ostream &OutFile) {
   auto &ObjectStreamer = static_cast<MCObjectStreamer &>(MS);
   MCAssembler &MCAsm = ObjectStreamer.getAssembler();
   auto &Writer = static_cast<MachObjectWriter &>(MCAsm.getWriter());
 
   // Layout but don't emit.
   ObjectStreamer.flushPendingLabels();
   MCAsmLayout Layout(MCAsm);
   MCAsm.layout(Layout);
 
-  BinaryHolder InputBinaryHolder(false);
+  BinaryHolder InputBinaryHolder(VFS, false);
 
   auto ObjectEntry = InputBinaryHolder.getObjectEntry(DM.getBinaryPath());
   if (!ObjectEntry) {
     auto Err = ObjectEntry.takeError();
     return error(Twine("opening ") + DM.getBinaryPath() + ": " +
                      toString(std::move(Err)),
                  "output file streaming");
   }

llvm/tools/dsymutil/dsymutil.h

 namespace dsymutil {
 
 class BinaryHolder;
 
 /// Extract the DebugMaps from the given file.
 /// The file has to be a MachO object file. Multiple debug maps can be
 /// returned when the file is universal (aka fat) binary.
 ErrorOr<std::vector<std::unique_ptr<DebugMap>>>
-parseDebugMap(StringRef InputFile, ArrayRef<std::string> Archs,
+parseDebugMap(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+              StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath, bool PaperTrailWarnings, bool Verbose,
               bool InputIsYAML);
 
-/// Dump the symbol table
-bool dumpStab(StringRef InputFile, ArrayRef<std::string> Archs,
+/// Dump the symbol table.
+bool dumpStab(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+              StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath = "");
 
 /// Link the Dwarf debug info as directed by the passed DebugMap \p DM into a
 /// DwarfFile named \p OutputFilename. \returns false if the link failed.
 bool linkDwarf(raw_fd_ostream &OutFile, BinaryHolder &BinHolder,
                const DebugMap &DM, LinkOptions Options);
 
 } // end namespace dsymutil
 } // end namespace llvm
 
 #endif // LLVM_TOOLS_DSYMUTIL_DSYMUTIL_H

llvm/tools/dsymutil/dsymutil.cpp

if (Arch != "*" && Arch != "all" &&
       return 1;
     }
 
   SymbolMapLoader SymMapLoader(Options.SymbolMap);
 
   for (auto &InputFile : Options.InputFiles) {
     // Dump the symbol table for each input file and requested arch
     if (Options.DumpStab) {
-      if (!dumpStab(InputFile, Options.Archs, Options.LinkOpts.PrependPath))
+      if (!dumpStab(Options.LinkOpts.VFS, InputFile, Options.Archs,
+                    Options.LinkOpts.PrependPath))
         return 1;
       continue;
     }
 
     auto DebugMapPtrsOrErr =
-        parseDebugMap(InputFile, Options.Archs, Options.LinkOpts.PrependPath,
-                      Options.PaperTrailWarnings, Options.LinkOpts.Verbose,
-                      Options.InputIsYAMLDebugMap);
+        parseDebugMap(Options.LinkOpts.VFS, InputFile, Options.Archs,
+                      Options.LinkOpts.PrependPath, Options.PaperTrailWarnings,
+                      Options.LinkOpts.Verbose, Options.InputIsYAMLDebugMap);
 
     if (auto EC = DebugMapPtrsOrErr.getError()) {
       WithColor::error() << "cannot parse the debug map for '" << InputFile
                          << "': " << EC.message() << '\n';
       return 1;
     }
 
     // Remember the number of debug maps that are being processed to decide how
for (auto &InputFile : Options.InputFiles) {
 
     // Ensure that the debug map is not empty (anymore).
     if (DebugMapPtrsOrErr->empty()) {
       WithColor::error() << "no architecture to link\n";
       return 1;
     }
 
     // Shared a single binary holder for all the link steps.
-    BinaryHolder BinHolder;
+    BinaryHolder BinHolder(Options.LinkOpts.VFS);
 
     ThreadPoolStrategy S = hardware_concurrency(Options.LinkOpts.Threads);
     if (Options.LinkOpts.Threads == 0) {
       // If NumThreads is not specified, create one thread for each input, up to
       // the number of hardware threads.
       S.ThreadsRequested = DebugMapPtrsOrErr->size();
       S.Limit = true;
     }
if (NeedsTempFiles) {
       }
       if (!MachOUtils::generateUniversalBinary(TempFiles,
                                                OutputLocationOrErr->DWARFFile,
                                                Options.LinkOpts, SDKPath))
         return 1;
     }
   }
 
+  Collector->writeMapping("mapping.yaml");
+
   return 0;
 }