[lldb/Reproducers] Collect files imported by command script import

[JDevlieghere]

Summary

Files imported by the script interpreter aren't opened by LLDB so they don't end up in the reproducer. The solution is to explicitly add them to the FileCollector.

[labath]

This seems fine with a (fairly big) caveat that this approach will not work transitively loaded modules (== if the "command script import"ed module imports another module on its own). Supporting this would be possible, but it would require hooking fairly deep into the module loading mechanism of respective language. In lua, I believe that could be achieved by overriding the builtin loadfile function (to save the file when recording, and substitute it during replay). In python, that might be achieved by overriding the __import__ function to record the module file name and by adjusting the module import mechanism (perhaps via import hooks) on replay.

However, all of that is going to be fairly tricky code, so I don't think that's needed if you just want to make simple imports work.

lldb/include/lldb/Host/FileSystem.h

public:
 
   llvm::ErrorOr<std::string> GetExternalPath(const llvm::Twine &path);
   llvm::ErrorOr<std::string> GetExternalPath(const FileSpec &file_spec);
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> GetVirtualFileSystem() {
     return m_fs;
   }
 
+  void Collect(const FileSpec &file_spec);
+  void Collect(const llvm::Twine &file);
+
 private:
-  void AddFile(const llvm::Twine &file);
   static llvm::Optional<FileSystem> &InstanceImpl();
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> m_fs;
   std::shared_ptr<llvm::FileCollector> m_collector;
   bool m_mapped;
 };
 } // namespace lldb_private
 
 #endif

lldb/source/Host/common/FileSystem.cpp

void FileSystem::Resolve(FileSpec &file_spec) {
   else
     file_spec.SetPath(path);
   file_spec.SetIsResolved(true);
 }
 
 std::shared_ptr<DataBufferLLVM>
 FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size,
                              uint64_t offset) {
-  AddFile(path);
+  Collect(path);
 
   const bool is_volatile = !IsLocal(path);
   const ErrorOr<std::string> external_path = GetExternalPath(path);
 
   if (!external_path)
     return nullptr;
 
   std::unique_ptr<llvm::WritableMemoryBuffer> buffer;
static mode_t GetOpenMode(uint32_t permissions) {
   if (permissions & lldb::eFilePermissionsWorldExecute)
     mode |= S_IXOTH;
   return mode;
 }
 
 Expected<FileUP> FileSystem::Open(const FileSpec &file_spec,
                                   File::OpenOptions options,
                                   uint32_t permissions, bool should_close_fd) {
-  AddFile(file_spec.GetPath());
+  Collect(file_spec.GetPath());
 
   const int open_flags = GetOpenFlags(options);
   const mode_t open_mode =
       (open_flags & O_CREAT) ? GetOpenMode(permissions) : 0;
 
   auto path = GetExternalPath(file_spec);
   if (!path)
     return errorCodeToError(path.getError());
ErrorOr<std::string> FileSystem::GetExternalPath(const llvm::Twine &path) {
 
   return F->getExternalContentsPath().str();
 }
 
 ErrorOr<std::string> FileSystem::GetExternalPath(const FileSpec &file_spec) {
   return GetExternalPath(file_spec.GetPath());
 }
 
-void FileSystem::AddFile(const llvm::Twine &file) {
+void FileSystem::Collect(const FileSpec &file_spec) {
+  Collect(file_spec.GetPath());
+}
+
+void FileSystem::Collect(const llvm::Twine &file) {
   if (m_collector && !llvm::sys::fs::is_directory(file)) {
     m_collector->addFile(file);
   }
 }

lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp

void ScriptInterpreterLua::ExecuteInterpreterLoop() {
   IOHandlerSP io_handler_sp(new IOHandlerLuaInterpreter(debugger, *this));
   debugger.RunIOHandlerAsync(io_handler_sp);
 }
 
 bool ScriptInterpreterLua::LoadScriptingModule(
     const char *filename, bool init_session, lldb_private::Status &error,
     StructuredData::ObjectSP *module_sp) {
 
+  FileSystem::Instance().Collect(filename);
   if (llvm::Error e = m_lua->LoadModule(filename)) {
     error.SetErrorStringWithFormatv("lua failed to import '{0}': {1}\n",
                                     filename, llvm::toString(std::move(e)));
     return false;
   }
   return true;
 }
 

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

if (!pathname || !pathname[0]) {
     return false;
   }
 
   lldb::DebuggerSP debugger_sp = m_debugger.shared_from_this();
 
   {
     FileSpec target_file(pathname);
     FileSystem::Instance().Resolve(target_file);
+    FileSystem::Instance().Collect(target_file);
     std::string basename(target_file.GetFilename().GetCString());
 
     StreamString command_stream;
 
     // Before executing Python code, lock the GIL.
     Locker py_lock(this,
                    Locker::AcquireLock |
                        (init_session ? Locker::InitSession : 0) |

lldb/test/Shell/Reproducer/Inputs/foo.lua

+print('95126')

lldb/test/Shell/Reproducer/Inputs/foo.py

+print('95126')

lldb/test/Shell/Reproducer/TestLuaImport.test

+# REQUIRES: lua
+# UNSUPPORTED: system-windows
+# Ensure that the reproducers know about imported Lua modules.
+
+# RUN: rm -rf %t.repro
+# RUN: %lldb -x -b --script-language lua --capture --capture-path %t.repro -o 'command script import  %S/Inputs/foo.lua' -o 'reproducer generate' | FileCheck %s --check-prefix CAPTURE
+
+# CAPTURE: 95126
+
+# RUN: %lldb -b -o 'reproducer dump -p files -f %t.repro' | FileCheck %s --check-prefix FILES
+# FILES: foo.lua

lldb/test/Shell/Reproducer/TestPythonImport.test

+# REQUIRES: python
+# UNSUPPORTED: system-windows
+# Ensure that the reproducers know about imported Python modules.
+
+# RUN: rm -rf %t.repro
+# RUN: %lldb -x -b --capture --capture-path %t.repro -o 'command script import  %S/Inputs/foo.py' -o 'reproducer generate' | FileCheck %s --check-prefix CAPTURE
+
+# CAPTURE: 95126
+
+# RUN: %lldb -b -o 'reproducer dump -p files -f %t.repro' | FileCheck %s --check-prefix FILES
+# FILES: foo.py