[lldb] Add an option to specify a VFS overlay

[JDevlieghere]

Summary

Add the ability to specify a VFS overlay to the command line driver. The motivation for this functionality is to facilitate testing. Jim has a patch where we want to simulate a remote debugging situation where a binary doesn't exist for locally (i.e. for LLDB) but does exist remotely (i.e. for debugserver).

LLDB already uses a virtual file system for the reproducers. This patch is mainly focussed on being able to initialize the FileSystem with a VFS specified on the command line.

One limitation here is that, unlike clang, you can only specify a single VFS overlay. This is caused by the FileSystem having to devitualize the path, which requires it to know whether the VFS is a RedirectingFileSystem. We wouldn't have this issue if LLDB were to only manipulate files through LLVM's memory buffer abstraction, but when the same issue came up for the reproducers, that was deemed too intrusive. As this functionality is really meant for development and testing, I'm not too concerned about that.

[jingham]

LGTM

lldb/source/API/SBDebugger.cpp

878

These two are just reformatting, right? I can't see any actual difference.

[JDevlieghere]

Looks like there's an easier way to test this, so no need to add another SB API.

lldb/include/lldb/API/SBDebugger.h

 //===-- SBDebugger.h --------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLDB_API_SBDEBUGGER_H
 #define LLDB_API_SBDEBUGGER_H
 
 #include <cstdio>
 
 #include "lldb/API/SBDefines.h"
 #include "lldb/API/SBPlatform.h"
 
+namespace lldb_private {
+struct SystemInitializerOptions;
+}
+
 namespace lldb {
 
 class LLDB_API SBInputReader {
 public:
   SBInputReader() = default;
   ~SBInputReader() = default;
 
   SBError Initialize(lldb::SBDebugger &sb_debugger,
                      unsigned long (*callback)(void *, lldb::SBInputReader *,
                                                lldb::InputReaderAction,
                                                char const *, unsigned long),
                      void *a, lldb::InputReaderGranularity b, char const *c,
                      char const *d, bool e);
   void SetIsDone(bool);
   bool IsActive() const;
 };
 
+class LLDB_API SBInitializationOptions {
+public:
+  SBInitializationOptions();
+  ~SBInitializationOptions();
+
+  void SetVFSOverlay(const char *overlay);
+
+private:
+  friend class lldb::SBDebugger;
+  std::unique_ptr<lldb_private::SystemInitializerOptions> m_opaque_up;
+};
+
 class LLDB_API SBDebugger {
 public:
   FLAGS_ANONYMOUS_ENUM(){eBroadcastBitProgress = (1 << 0)};
 
   SBDebugger();
 
   SBDebugger(const lldb::SBDebugger &rhs);
 
static const char *GetProgressFromEvent(const lldb::SBEvent &event,
                                           uint64_t &completed, uint64_t &total,
                                           bool &is_debugger_specific);
 
   lldb::SBDebugger &operator=(const lldb::SBDebugger &rhs);
 
   static void Initialize();
 
   static lldb::SBError InitializeWithErrorHandling();
+  static lldb::SBError
+  InitializeWithErrorHandling(const SBInitializationOptions &options);
 
   static void Terminate();
 
   // Deprecated, use the one that takes a source_init_files bool.
   static lldb::SBDebugger Create();
 
   static lldb::SBDebugger Create(bool source_init_files);
 

lldb/include/lldb/API/SBDefines.h

 class LLDB_API SBExecutionContext;
 class LLDB_API SBExpressionOptions;
 class LLDB_API SBFile;
 class LLDB_API SBFileSpec;
 class LLDB_API SBFileSpecList;
 class LLDB_API SBFrame;
 class LLDB_API SBFunction;
 class LLDB_API SBHostOS;
+class LLDB_API SBInitializationOptions;
 class LLDB_API SBInstruction;
 class LLDB_API SBInstructionList;
 class LLDB_API SBLanguageRuntime;
 class LLDB_API SBLaunchInfo;
 class LLDB_API SBLineEntry;
 class LLDB_API SBListener;
 class LLDB_API SBMemoryRegionInfo;
 class LLDB_API SBMemoryRegionInfoList;

lldb/include/lldb/Initialization/SystemInitializerCommon.h

 //===-- SystemInitializerCommon.h -------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLDB_INITIALIZATION_SYSTEMINITIALIZERCOMMON_H
 #define LLDB_INITIALIZATION_SYSTEMINITIALIZERCOMMON_H
 
 #include "SystemInitializer.h"
 #include "lldb/Host/HostInfo.h"
 
 namespace lldb_private {
+struct SystemInitializerOptions {
+  /// Overlay the virtual filesystem described by file over the real file
+  /// system.
+  FileSpec vfs_overlay;
+};
+
 /// Initializes common lldb functionality.
 ///
 /// This class is responsible for initializing a subset of lldb
 /// useful to both debug servers and debug clients.  Debug servers
 /// do not use all of LLDB and desire small binary sizes, so this
 /// functionality is separate.  This class is used by constructing
 /// an instance of SystemLifetimeManager with this class passed to
 /// the constructor.
 class SystemInitializerCommon : public SystemInitializer {
 public:
-  SystemInitializerCommon(HostInfo::SharedLibraryDirectoryHelper *helper);
+  SystemInitializerCommon(HostInfo::SharedLibraryDirectoryHelper *helper,
+                          SystemInitializerOptions = {});
   ~SystemInitializerCommon() override;
 
   llvm::Error Initialize() override;
   void Terminate() override;
 
 private:
   HostInfo::SharedLibraryDirectoryHelper *m_shlib_dir_helper;
+  SystemInitializerOptions m_options;
 };
 
 } // namespace lldb_private
 
 #endif // LLDB_INITIALIZATION_SYSTEMINITIALIZERCOMMON_H

lldb/source/API/SBDebugger.cpp

 }
 
 bool SBInputReader::IsActive() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBInputReader, IsActive);
 
   return false;
 }
 
+SBInitializationOptions::SBInitializationOptions()
+    : m_opaque_up(std::make_unique<SystemInitializerOptions>()) {
+  LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBInitializationOptions);
+}
+
+SBInitializationOptions::~SBInitializationOptions() {}
+
+// We can't use an SBFileSpec as the argument, because that will try to use
+// the FileSystem before it's initialized.
+void SBInitializationOptions::SetVFSOverlay(const char *overlay) {
+  LLDB_RECORD_METHOD(void, SBInitializationOptions, SetVFSOverlay,
+                     (const char *), overlay);
+
+  m_opaque_up->vfs_overlay = FileSpec(overlay);
+}
+
 SBDebugger::SBDebugger() { LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBDebugger); }
 
 SBDebugger::SBDebugger(const lldb::DebuggerSP &debugger_sp)
     : m_opaque_sp(debugger_sp) {
   LLDB_RECORD_CONSTRUCTOR(SBDebugger, (const lldb::DebuggerSP &), debugger_sp);
 }
 
 SBDebugger::SBDebugger(const SBDebugger &rhs) : m_opaque_sp(rhs.m_opaque_sp) {
lldb::SBError SBDebugger::InitializeWithErrorHandling() {
   SBError error;
   if (auto e = g_debugger_lifetime->Initialize(
           std::make_unique<SystemInitializerFull>(), LoadPlugin)) {
     error.SetError(Status(std::move(e)));
   }
   return LLDB_RECORD_RESULT(error);
 }
 
+lldb::SBError SBDebugger::InitializeWithErrorHandling(
+    const SBInitializationOptions &options) {
+  LLDB_RECORD_STATIC_METHOD(lldb::SBError, SBDebugger,
+                            InitializeWithErrorHandling,
+                            (const lldb::SBInitializationOptions &), options);
+
+  SBError error;
+  if (auto e = g_debugger_lifetime->Initialize(
+          std::make_unique<SystemInitializerFull>(*options.m_opaque_up),
+          LoadPlugin)) {
+    error.SetError(Status(std::move(e)));
+  }
+  return LLDB_RECORD_RESULT(error);
+}
+
 void SBDebugger::Terminate() {
   LLDB_RECORD_STATIC_METHOD_NO_ARGS(void, SBDebugger, Terminate);
 
   g_debugger_lifetime->Terminate();
 }
 
 void SBDebugger::Clear() {
   LLDB_RECORD_METHOD_NO_ARGS(void, SBDebugger, Clear);
SBTarget SBDebugger::CreateTargetWithFileAndArch(const char *filename,
   SBTarget sb_target;
   TargetSP target_sp;
   if (m_opaque_sp) {
     Status error;
     if (arch_cstr == nullptr) {
       // The version of CreateTarget that takes an ArchSpec won't accept an
       // empty ArchSpec, so when the arch hasn't been specified, we need to
       // call the target triple version.
-      error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, 
-          arch_cstr, eLoadDependentsYes, nullptr, target_sp);
+      error = m_opaque_sp->GetTargetList().CreateTarget(
+          *m_opaque_sp, filename, arch_cstr, eLoadDependentsYes, nullptr,
+          target_sp);
     } else {
       PlatformSP platform_sp = m_opaque_sp->GetPlatformList()
           .GetSelectedPlatform();
-      ArchSpec arch = Platform::GetAugmentedArchSpec(platform_sp.get(), 
-          arch_cstr);
+      ArchSpec arch =
+          Platform::GetAugmentedArchSpec(platform_sp.get(), arch_cstr);
       if (arch.IsValid())
-        error = m_opaque_sp->GetTargetList().CreateTarget(*m_opaque_sp, filename, 
-            arch, eLoadDependentsYes, platform_sp, target_sp);
+        error = m_opaque_sp->GetTargetList().CreateTarget(
+            *m_opaque_sp, filename, arch, eLoadDependentsYes, platform_sp,
+            target_sp);
       else
         error.SetErrorStringWithFormat("invalid arch_cstr: %s", arch_cstr);
     }
     if (error.Success())
       sb_target.SetSP(target_sp);
   }
 
   LLDB_LOGF(log,
LLDB_REGISTER_METHOD(lldb::SBTypeSynthetic, SBDebugger, GetSyntheticForType,
                        (lldb::SBTypeNameSpecifier));
   LLDB_REGISTER_METHOD(lldb::SBTypeFilter, SBDebugger, GetFilterForType,
                        (lldb::SBTypeNameSpecifier));
   LLDB_REGISTER_METHOD(bool, SBDebugger, EnableLog,
                        (const char *, const char **));
   LLDB_REGISTER_METHOD(lldb::SBCommandInterpreterRunResult, SBDebugger,
                        RunCommandInterpreter,
                        (const lldb::SBCommandInterpreterRunOptions &));
+  LLDB_REGISTER_CONSTRUCTOR(SBInitializationOptions, ());
+  LLDB_REGISTER_METHOD(void, SBInitializationOptions, SetVFSOverlay,
+                       (const char *));
+  LLDB_REGISTER_STATIC_METHOD(lldb::SBError, SBDebugger,
+                              InitializeWithErrorHandling,
+                              (const lldb::SBInitializationOptions &));
 }
 
 } // namespace repro
 } // namespace lldb_private

lldb/source/API/SystemInitializerFull.h

 /// Initializes lldb.
 ///
 /// This class is responsible for initializing all of lldb system
 /// services needed to use the full LLDB application.  This class is
 /// not intended to be used externally, but is instead used
 /// internally by SBDebugger to initialize the system.
 class SystemInitializerFull : public SystemInitializerCommon {
 public:
-  SystemInitializerFull();
+  SystemInitializerFull(SystemInitializerOptions options = {});
   ~SystemInitializerFull() override;
 
   llvm::Error Initialize() override;
   void Terminate() override;
 };
 
 } // namespace lldb_private
 
 #endif // LLDB_SOURCE_API_SYSTEMINITIALIZERFULL_H

lldb/source/API/SystemInitializerFull.cpp

 
 #else
 constexpr lldb_private::HostInfo::SharedLibraryDirectoryHelper
     *g_shlib_dir_helper = 0;
 #endif
 
 using namespace lldb_private;
 
-SystemInitializerFull::SystemInitializerFull()
-    : SystemInitializerCommon(g_shlib_dir_helper) {}
+SystemInitializerFull::SystemInitializerFull(SystemInitializerOptions options)
+    : SystemInitializerCommon(g_shlib_dir_helper, std::move(options)) {}
 SystemInitializerFull::~SystemInitializerFull() = default;
 
 llvm::Error SystemInitializerFull::Initialize() {
   llvm::Error error = SystemInitializerCommon::Initialize();
   if (error) {
     // During active replay, the ::Initialize call is replayed like any other
     // SB API call and the return value is ignored. Since we can't intercept
     // this, we terminate here before the uninitialized debugger inevitably

lldb/source/Initialization/SystemInitializerCommon.cpp

 #include "llvm/Support/TargetSelect.h"
 
 #include <string>
 
 using namespace lldb_private;
 using namespace lldb_private::repro;
 
 SystemInitializerCommon::SystemInitializerCommon(
-    HostInfo::SharedLibraryDirectoryHelper *helper)
-    : m_shlib_dir_helper(helper) {}
+    HostInfo::SharedLibraryDirectoryHelper *helper,
+    SystemInitializerOptions options)
+    : m_shlib_dir_helper(helper), m_options(std::move(options)) {}
 
 SystemInitializerCommon::~SystemInitializerCommon() = default;
 
 /// Initialize the FileSystem based on the current reproducer mode.
-static llvm::Error InitializeFileSystem() {
+static llvm::Error InitializeFileSystem(FileSpec vfs_overlay) {
   auto &r = repro::Reproducer::Instance();
   if (repro::Loader *loader = r.GetLoader()) {
     FileSpec vfs_mapping = loader->GetFile<FileProvider::Info>();
     if (vfs_mapping) {
       if (llvm::Error e = FileSystem::Initialize(vfs_mapping))
         return e;
     } else {
       FileSystem::Initialize();
if (repro::Generator *g = r.GetGenerator()) {
     fp.RecordInterestingDirectory(
         g->GetOrCreate<repro::WorkingDirectoryProvider>().GetDirectory());
     fp.RecordInterestingDirectory(
         g->GetOrCreate<repro::HomeDirectoryProvider>().GetDirectory());
 
     return llvm::Error::success();
   }
 
+  if (vfs_overlay) {
+    if (llvm::Error e = FileSystem::Initialize(vfs_overlay))
+      return e;
+    return llvm::Error::success();
+  }
+
   FileSystem::Initialize();
   return llvm::Error::success();
 }
 
 llvm::Error SystemInitializerCommon::Initialize() {
 #if defined(_WIN32)
   const char *disable_crash_dialog_var = getenv("LLDB_DISABLE_CRASH_DIALOG");
   if (disable_crash_dialog_var &&
#endif
 
   // If the reproducer wasn't initialized before, we can safely assume it's
   // off.
   if (!Reproducer::Initialized()) {
     if (auto e = Reproducer::Initialize(ReproducerMode::Off, llvm::None))
       return e;
   }
 
-  if (auto e = InitializeFileSystem())
+  if (auto e = InitializeFileSystem(m_options.vfs_overlay))
     return e;
 
   Log::Initialize();
   HostInfo::Initialize(m_shlib_dir_helper);
 
   llvm::Error error = Socket::Initialize();
   if (error)
     return error;

lldb/test/Shell/Driver/Inputs/create_overlay.py

+import sys
+
+_, virtual_path, real_path, overlay_path = sys.argv
+
+string = '''{
+  "use-external-names": false,
+  "version": 0,
+  "roots": [
+    {
+      "type": "file",
+      "name": "%s",
+      "external-contents": "%s"
+    }
+  ]
+}''' % (virtual_path, real_path)
+
+
+with open(overlay_path, 'w') as f:
+  f.write(string)
+  print(string)

lldb/test/Shell/Driver/TestOverlay.test

+# REQUIRES: python
+
+# RUN: mkdir -p %t/foo
+# RUN: %clang_host %p/Inputs/hello.c -g -o %t/foo/a.out
+
+# RUN: %python %p/Inputs/create_overlay.py %t/bar/a.out %t/foo/a.out %t.overlay
+
+# RUN: %lldb -vfs-overlay %t.overlay %t/bar/a.out 2>&1 | FileCheck %s
+# CHECK-NOT: error:
+
+# RUN: %lldb %t/bar/a.out 2>&1 | FileCheck %s --check-prefix FAILURE
+# FAILURE: unable to find executable

lldb/tools/driver/Driver.cpp

llvm::errs() << "Use '" << argv0
                  << " --help' for a complete list of options.\n";
     return 1;
   }
 
   if (auto exit_code = InitializeReproducer(argv[0], input_args)) {
     return *exit_code;
   }
 
-  SBError error = SBDebugger::InitializeWithErrorHandling();
+  SBInitializationOptions options;
+  if (auto *arg = input_args.getLastArg(OPT_vfs_overlay))
+    options.SetVFSOverlay(arg->getValue());
+  SBError error = SBDebugger::InitializeWithErrorHandling(options);
   if (error.Fail()) {
     WithColor::error() << "initialization failed: " << error.GetCString()
                        << '\n';
     return 1;
   }
   SBHostOS::ThreadCreated("<lldb.driver.main-thread>");
 
   signal(SIGINT, sigint_handler);

lldb/tools/driver/Options.td

def no_version_check: F<"reproducer-no-version-check">,
   HelpText<"Disable the reproducer version check.">;
 def no_verification: F<"reproducer-no-verify">,
   HelpText<"Disable the reproducer verification.">;
 def no_generate_on_signal: F<"reproducer-no-generate-on-signal">,
   HelpText<"Don't generate reproducer when a signal is received.">;
 def generate_on_exit: F<"reproducer-generate-on-exit">,
   HelpText<"Generate reproducer on exit.">;
 
+def vfs_overlay: Separate<["--", "-"], "vfs-overlay">,
+  MetaVarName<"<file>">,
+  HelpText<"Overlay the virtual filesystem described by file over the real file system">;
+
 def REM : R<["--"], "">;