[lldb/Plugins] Add ability to load modules to Scripted Processes

[mib]

Summary

This patch introduces a new way to load modules programatically with
Scripted Processes. To do so, the scripted process blueprint holds a
list of dictionary describing the modules to load, which their path or
uuid, load address and eventually a slide offset.

LLDB will fetch that list after launching the ScriptedProcess, and
iterate over each entry to create the module that will be loaded in the
Scripted Process' target.

The patch also refactors the StackCoreScriptedProcess test to stop
inside the libbaz module and make sure it's loaded correctly and that
we can fetch some variables from it.

rdar://74520238

Signed-off-by: Med Ismail Bennani <[e-mail]>

[mib]

Re-format

[JDevlieghere]

lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp

413–416

Nit: you're calling it a key in the first error message, but field in the second. Let's pick one and be consistent.

lldb/test/API/functionalities/scripted_process/stack_core_scripted_process.py

51

What exactly is this value? Shouldn't we get the load address from the core file and report that?

[mib]

lldb/test/API/functionalities/scripted_process/stack_core_scripted_process.py

51

You're right, I missed that.

[mib]

Address @JDevlieghere comments:

• Fix error message
• Use corefile module load address for scripted process module

[JDevlieghere]

LGTM

lldb/examples/python/scripted_process/scripted_process.py

class ScriptedProcess:
 
     DISCLAIMER: THIS INTERFACE IS STILL UNDER DEVELOPMENT AND NOT STABLE.
                 THE METHODS EXPOSED MIGHT CHANGE IN THE FUTURE.
     """
 
     memory_regions = None
     stack_memory_dump = None
     loaded_images = None
-    threads = {}
+    threads = None
 
     @abstractmethod
     def __init__(self, target, args):
         """ Construct a scripted process.
 
         Args:
             target (lldb.SBTarget): The target launching the scripted process.
             args (lldb.SBStructuredData): A Dictionary holding arbitrary
                 key/value pairs used by the scripted process.
         """
         self.target = None
         self.args = None
         self.arch = None
         if isinstance(target, lldb.SBTarget) and target.IsValid():
             self.target = target
             triple = self.target.triple
             if triple:
                 self.arch = triple.split('-')[0]
             self.dbg = target.GetDebugger()
         if isinstance(args, lldb.SBStructuredData) and args.IsValid():
             self.args = args
+        self.threads = {}
+        self.loaded_images = []
 
     @abstractmethod
     def get_memory_region_containing_address(self, addr):
         """ Get the memory region for the scripted process, containing a
             specific address.
 
         Args:
             addr (int): Address to look for in the scripted process memory
def read_memory_at_address(self, addr, size):
         """
         pass
 
     def get_loaded_images(self):
         """ Get the list of loaded images for the scripted process.
 
         ```
         class ScriptedProcessImage:
-            def __init__(name, file_spec, uuid, load_address):
-              self.name = name
+            def __init__(file_spec, uuid, load_address):
               self.file_spec = file_spec
               self.uuid = uuid
               self.load_address = load_address
         ```
 
         Returns:
             List[ScriptedProcessImage]: A list of `ScriptedProcessImage`
                 containing for each entry, the name of the library, a UUID,

lldb/include/lldb/Interpreter/ScriptedProcessInterface.h

virtual StructuredData::DictionarySP GetRegistersForThread(lldb::tid_t tid) {
     return nullptr;
   }
 
   virtual lldb::DataExtractorSP
   ReadMemoryAtAddress(lldb::addr_t address, size_t size, Status &error) {
     return nullptr;
   }
 
-  virtual StructuredData::DictionarySP GetLoadedImages() { return nullptr; }
+  virtual StructuredData::ArraySP GetLoadedImages() { return nullptr; }
 
   virtual lldb::pid_t GetProcessID() { return LLDB_INVALID_PROCESS_ID; }
 
   virtual bool IsAlive() { return true; }
 
   virtual llvm::Optional<std::string> GetScriptedThreadPluginName() {
     return llvm::None;
   }

lldb/source/Plugins/Process/scripted/ScriptedProcess.h

public:
 
   ArchSpec GetArchitecture();
 
   Status
   GetMemoryRegions(lldb_private::MemoryRegionInfos &region_list) override;
 
   bool GetProcessInfo(ProcessInstanceInfo &info) override;
 
+  lldb_private::StructuredData::ObjectSP
+  GetLoadedDynamicLibrariesInfos() override;
+
 protected:
   Status DoStop();
 
   void Clear();
 
   bool DoUpdateThreadList(ThreadList &old_thread_list,
                           ThreadList &new_thread_list) override;
 

lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp

Status ScriptedProcess::DoLaunch(Module *exe_module,
   SetPrivateState(eStateStopped);
 
   return {};
 }
 
 void ScriptedProcess::DidLaunch() {
   CheckInterpreterAndScriptObject();
   m_pid = GetInterface().GetProcessID();
+  GetLoadedDynamicLibrariesInfos();
 }
 
 Status ScriptedProcess::DoResume() {
   CheckInterpreterAndScriptObject();
 
   Log *log = GetLog(LLDBLog::Process);
   // FIXME: Fetch data from thread.
   const StateType thread_resume_state = eStateRunning;
bool ScriptedProcess::GetProcessInfo(ProcessInstanceInfo &info) {
   if (module_sp) {
     const bool add_exe_file_as_first_arg = false;
     info.SetExecutableFile(GetTarget().GetExecutableModule()->GetFileSpec(),
                            add_exe_file_as_first_arg);
   }
   return true;
 }
 
+lldb_private::StructuredData::ObjectSP
+ScriptedProcess::GetLoadedDynamicLibrariesInfos() {
+  CheckInterpreterAndScriptObject();
+
+  Status error;
+  auto error_with_message = [&error](llvm::StringRef message) {
+    return ScriptedInterface::ErrorWithMessage<bool>(LLVM_PRETTY_FUNCTION,
+                                                     message.data(), error);
+  };
+
+  StructuredData::ArraySP loaded_images_sp = GetInterface().GetLoadedImages();
+
+  if (!loaded_images_sp || !loaded_images_sp->GetSize())
+    return GetInterface().ErrorWithMessage<StructuredData::ObjectSP>(
+        LLVM_PRETTY_FUNCTION, "No loaded images.", error);
+
+  ModuleList module_list;
+  Target &target = GetTarget();
+
+  auto reload_image = [&target, &module_list, &error_with_message](
+                          StructuredData::Object *obj) -> bool {
+    StructuredData::Dictionary *dict = obj->GetAsDictionary();
+
+    if (!dict)
+      return error_with_message("Couldn't cast image object into dictionary.");
+
+    ModuleSpec module_spec;
+    llvm::StringRef value;
+
+    bool has_path = dict->HasKey("path");
+    bool has_uuid = dict->HasKey("uuid");
+    if (!has_path && !has_uuid)
+      return error_with_message("Dictionary should have key 'path' or 'uuid'");
+    if (!dict->HasKey("load_addr"))
+      return error_with_message("Dictionary is missing key 'load_addr'");
+
+    if (has_path) {
+      dict->GetValueForKeyAsString("path", value);
+      module_spec.GetFileSpec().SetPath(value);
+    }
+
+    if (has_uuid) {
+      dict->GetValueForKeyAsString("uuid", value);
+      module_spec.GetUUID().SetFromStringRef(value);
+    }
+    module_spec.GetArchitecture() = target.GetArchitecture();
+
+    ModuleSP module_sp =
+        target.GetOrCreateModule(module_spec, true /* notify */);
+
+    if (!module_sp)
+      return error_with_message("Couldn't create or get module.");
+
+    lldb::addr_t load_addr = LLDB_INVALID_ADDRESS;
+    lldb::addr_t slide = LLDB_INVALID_OFFSET;
+    dict->GetValueForKeyAsInteger("load_addr", load_addr);
+    dict->GetValueForKeyAsInteger("slide", slide);
+    if (load_addr == LLDB_INVALID_ADDRESS)
+      return error_with_message(
+          "Couldn't get valid load address or slide offset.");
+
+    if (slide != LLDB_INVALID_OFFSET)
+      load_addr += slide;
+
+    bool changed = false;
+    module_sp->SetLoadAddress(target, load_addr, false /*=value_is_offset*/,
+                              changed);
+
+    if (!changed && !module_sp->GetObjectFile())
+      return error_with_message("Couldn't set the load address for module.");
+
+    dict->GetValueForKeyAsString("path", value);
+    FileSpec objfile(value);
+    module_sp->SetFileSpecAndObjectName(objfile, objfile.GetFilename());
+
+    return module_list.AppendIfNeeded(module_sp);
+  };
+
+  if (!loaded_images_sp->ForEach(reload_image))
+    return GetInterface().ErrorWithMessage<StructuredData::ObjectSP>(
+        LLVM_PRETTY_FUNCTION, "Couldn't reload all images.", error);
+
+  target.ModulesDidLoad(module_list);
+
+  return loaded_images_sp;
+}
+
 ScriptedProcessInterface &ScriptedProcess::GetInterface() const {
   return m_interpreter->GetScriptedProcessInterface();
 }

lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h

public:
 
   StructuredData::DictionarySP GetThreadWithID(lldb::tid_t tid) override;
 
   StructuredData::DictionarySP GetRegistersForThread(lldb::tid_t tid) override;
 
   lldb::DataExtractorSP ReadMemoryAtAddress(lldb::addr_t address, size_t size,
                                             Status &error) override;
 
-  StructuredData::DictionarySP GetLoadedImages() override;
+  StructuredData::ArraySP GetLoadedImages() override;
 
   lldb::pid_t GetProcessID() override;
 
   bool IsAlive() override;
 
   llvm::Optional<std::string> GetScriptedThreadPluginName() override;
 
 private:
   lldb::ScriptedThreadInterfaceSP CreateScriptedThreadInterface() override;
 };
 } // namespace lldb_private
 
 #endif // LLDB_ENABLE_PYTHON
 #endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTEDPROCESSPYTHONINTERFACE_H

lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp

 }
 
 lldb::DataExtractorSP ScriptedProcessPythonInterface::ReadMemoryAtAddress(
     lldb::addr_t address, size_t size, Status &error) {
   return Dispatch<lldb::DataExtractorSP>("read_memory_at_address", error,
                                          address, size);
 }
 
-StructuredData::DictionarySP ScriptedProcessPythonInterface::GetLoadedImages() {
-  // TODO: Implement
-  return {};
+StructuredData::ArraySP ScriptedProcessPythonInterface::GetLoadedImages() {
+  Status error;
+  StructuredData::ArraySP array =
+      Dispatch<StructuredData::ArraySP>("get_loaded_images", error);
+
+  if (!array || !array->IsValid() || error.Fail()) {
+    return ScriptedInterface::ErrorWithMessage<StructuredData::ArraySP>(
+        LLVM_PRETTY_FUNCTION,
+        llvm::Twine("Null or invalid object (" +
+                    llvm::Twine(error.AsCString()) + llvm::Twine(")."))
+            .str(),
+        error);
+  }
+
+  return array;
 }
 
 lldb::pid_t ScriptedProcessPythonInterface::GetProcessID() {
   Status error;
   StructuredData::ObjectSP obj = Dispatch("get_process_id", error);
 
   if (!CheckStructuredDataObject(LLVM_PRETTY_FUNCTION, obj, error))
     return LLDB_INVALID_PROCESS_ID;

lldb/test/API/functionalities/scripted_process/Makefile

 CXX_SOURCES := main.cpp
 ENABLE_THREADS := YES
-include Makefile.rules
+LD_EXTRAS := -L. -lbaz -I.
+
+override ARCH := $(shell uname -m)
+
+all: libbaz.dylib a.out
 
+libbaz.dylib: baz.c
+  $(MAKE) -f $(MAKEFILE_RULES) ARCH=$(ARCH) \
+    DYLIB_ONLY=YES DYLIB_NAME=baz DYLIB_C_SOURCES=baz.c
+
+include Makefile.rules

lldb/test/API/functionalities/scripted_process/TestStackCoreScriptedProcess.py

def setUp(self):
         TestBase.setUp(self)
 
     def tearDown(self):
         TestBase.tearDown(self)
 
     def create_stack_skinny_corefile(self, file):
         self.build()
         target, process, thread, _ = lldbutil.run_to_source_breakpoint(self, "// break here",
-                                                                       lldb.SBFileSpec("main.cpp"))
+                                                                       lldb.SBFileSpec("baz.c"))
         self.assertTrue(process.IsValid(), "Process is invalid.")
         # FIXME: Use SBAPI to save the process corefile.
         self.runCmd("process save-core -s stack  " + file)
         self.assertTrue(os.path.exists(file), "No stack-only corefile found.")
         self.assertTrue(self.dbg.DeleteTarget(target), "Couldn't delete target")
 
+    def get_module_with_name(self, target, name):
+        for module in target.modules:
+            if name in module.GetFileSpec().GetFilename():
+                return module
+        return None
+
     @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
     def test_launch_scripted_process_stack_frames(self):
         """Test that we can launch an lldb scripted process from the command
         line, check its process ID and read string from memory."""
         self.build()
         target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
         self.assertTrue(target, VALID_TARGET)
 
-        for module in target.modules:
-            if 'a.out' in module.GetFileSpec().GetFilename():
-                main_module = module
-                break
-
+        main_module = self.get_module_with_name(target, 'a.out')
         self.assertTrue(main_module, "Invalid main module.")
         error = target.SetModuleLoadAddress(main_module, 0)
         self.assertSuccess(error, "Reloading main module at offset 0 failed.")
 
+        scripted_dylib = self.get_module_with_name(target, 'libbaz.dylib')
+        self.assertTrue(scripted_dylib, "Dynamic library libbaz.dylib not found.")
+        self.assertEqual(scripted_dylib.GetObjectFileHeaderAddress().GetLoadAddress(target), 0xffffffffffffffff)
+
         os.environ['SKIP_SCRIPTED_PROCESS_LAUNCH'] = '1'
         def cleanup():
           del os.environ["SKIP_SCRIPTED_PROCESS_LAUNCH"]
         self.addTearDownHook(cleanup)
 
         scripted_process_example_relpath = 'stack_core_scripted_process.py'
         self.runCmd("command script import " + os.path.join(self.getSourceDir(),
                                                             scripted_process_example_relpath))
 
         corefile_process = None
         with tempfile.NamedTemporaryFile() as file:
             self.create_stack_skinny_corefile(file.name)
             corefile_target = self.dbg.CreateTarget(None)
             corefile_process = corefile_target.LoadCore(self.getBuildArtifact(file.name))
         self.assertTrue(corefile_process, PROCESS_IS_VALID)
 
         structured_data = lldb.SBStructuredData()
         structured_data.SetFromJSON(json.dumps({
-            "backing_target_idx" : self.dbg.GetIndexOfTarget(corefile_process.GetTarget())
+            "backing_target_idx" : self.dbg.GetIndexOfTarget(corefile_process.GetTarget()),
+            "libbaz_path" : self.getBuildArtifact("libbaz.dylib")
         }))
         launch_info = lldb.SBLaunchInfo(None)
         launch_info.SetProcessPluginName("ScriptedProcess")
         launch_info.SetScriptedProcessClassName("stack_core_scripted_process.StackCoreScriptedProcess")
         launch_info.SetScriptedProcessDictionary(structured_data)
 
         error = lldb.SBError()
         process = target.Launch(launch_info, error)
         self.assertSuccess(error)
         self.assertTrue(process, PROCESS_IS_VALID)
         self.assertEqual(process.GetProcessID(), 42)
 
-        self.assertEqual(process.GetNumThreads(), 3)
+        self.assertEqual(process.GetNumThreads(), 2)
         thread = process.GetSelectedThread()
         self.assertTrue(thread, "Invalid thread.")
-        self.assertEqual(thread.GetName(), "StackCoreScriptedThread.thread-2")
+        self.assertEqual(thread.GetName(), "StackCoreScriptedThread.thread-1")
 
         self.assertTrue(target.triple, "Invalid target triple")
         arch = target.triple.split('-')[0]
         supported_arch = ['x86_64', 'arm64', 'arm64e']
         self.assertIn(arch, supported_arch)
         # When creating a corefile of a arm process, lldb saves the exception
         # that triggers the breakpoint in the LC_NOTES of the corefile, so they
         # can be reloaded with the corefile on the next debug session.
         if arch in 'arm64e':
             self.assertTrue(thread.GetStopReason(), lldb.eStopReasonException)
         # However, it's architecture specific, and corefiles made from intel
         # process don't save any metadata to retrieve to stop reason.
         # To mitigate this, the StackCoreScriptedProcess will report a
         # eStopReasonSignal with a SIGTRAP, mimicking what debugserver does.
         else:
             self.assertTrue(thread.GetStopReason(), lldb.eStopReasonSignal)
 
-        self.assertEqual(thread.GetNumFrames(), 6)
+        self.assertEqual(thread.GetNumFrames(), 5)
         frame = thread.GetSelectedFrame()
         self.assertTrue(frame, "Invalid frame.")
-        self.assertIn("bar", frame.GetFunctionName())
-        self.assertEqual(int(frame.FindValue("i", lldb.eValueTypeVariableArgument).GetValue()), 42)
-        self.assertEqual(int(frame.FindValue("j", lldb.eValueTypeVariableLocal).GetValue()), 42 * 42)
+        func = frame.GetFunction()
+        self.assertTrue(func, "Invalid function.")
+
+        self.assertIn("baz", frame.GetFunctionName())
+        self.assertEqual(frame.vars.GetSize(), 2)
+        self.assertEqual(int(frame.vars.GetFirstValueByName('j').GetValue()), 42 * 42)
+        self.assertEqual(int(frame.vars.GetFirstValueByName('k').GetValue()), 42)
+
+        scripted_dylib = self.get_module_with_name(target, 'libbaz.dylib')
+        self.assertTrue(scripted_dylib, "Dynamic library libbaz.dylib not found.")
+        self.assertEqual(scripted_dylib.GetObjectFileHeaderAddress().GetLoadAddress(target), 0x1001e0000)

lldb/test/API/functionalities/scripted_process/baz.h

+#pragma once
+
+int baz(int j);

lldb/test/API/functionalities/scripted_process/baz.c

+#include "baz.h"
+
+#include <math.h>
+
+int baz(int j) {
+  int k = sqrt(j);
+  return k; // break here
+}

lldb/test/API/functionalities/scripted_process/main.cpp

 #include <iostream>
 #include <mutex>
 #include <thread>
 
+extern "C" {
+int baz(int);
+}
+
 int bar(int i) {
   int j = i * i;
-  return j; // break here
+  return j;
 }
 
 int foo(int i) { return bar(i); }
 
 void call_and_wait(int &n) {
   std::cout << "waiting for computation!" << std::endl;
-  while (n != 42 * 42)
+  while (baz(n) != 42)
     ;
   std::cout << "finished computation!" << std::endl;
 }
 
 void compute_pow(int &n) { n = foo(n); }
 
 int main() {
   int n = 42;

lldb/test/API/functionalities/scripted_process/stack_core_scripted_process.py

 import os,json,struct,signal
 
 from typing import Any, Dict
 
 import lldb
 from lldb.plugins.scripted_process import ScriptedProcess
 from lldb.plugins.scripted_process import ScriptedThread
 
 class StackCoreScriptedProcess(ScriptedProcess):
+    def get_module_with_name(self, target, name):
+        for module in target.modules:
+            if name in module.GetFileSpec().GetFilename():
+                return module
+        return None
+
     def __init__(self, target: lldb.SBTarget, args : lldb.SBStructuredData):
         super().__init__(target, args)
 
-        self.backing_target_idx = args.GetValueForKey("backing_target_idx")
-
         self.corefile_target = None
         self.corefile_process = None
+
+        self.backing_target_idx = args.GetValueForKey("backing_target_idx")
         if (self.backing_target_idx and self.backing_target_idx.IsValid()):
             if self.backing_target_idx.GetType() == lldb.eStructuredDataTypeInteger:
                 idx = self.backing_target_idx.GetIntegerValue(42)
             if self.backing_target_idx.GetType() == lldb.eStructuredDataTypeString:
                 idx = int(self.backing_target_idx.GetStringValue(100))
             self.corefile_target = target.GetDebugger().GetTargetAtIndex(idx)
             self.corefile_process = self.corefile_target.GetProcess()
             for corefile_thread in self.corefile_process:
                 structured_data = lldb.SBStructuredData()
                 structured_data.SetFromJSON(json.dumps({
                     "backing_target_idx" : idx,
                     "thread_idx" : corefile_thread.GetIndexID()
                 }))
 
                 self.threads[corefile_thread.GetThreadID()] = StackCoreScriptedThread(self, structured_data)
 
-        if len(self.threads) == 3:
+        if len(self.threads) == 2:
             self.threads[len(self.threads) - 1].is_stopped = True
 
+        corefile_module = self.get_module_with_name(self.corefile_target,
+                                                    "libbaz.dylib")
+        if not corefile_module or not corefile_module.IsValid():
+            return
+        module_path = os.path.join(corefile_module.GetFileSpec().GetDirectory(),
+                                   corefile_module.GetFileSpec().GetFilename())
+        if not os.path.exists(module_path):
+            return
+        module_load_addr = corefile_module.GetObjectFileHeaderAddress().GetLoadAddress(self.corefile_target)
+
+        self.loaded_images.append({"path": module_path,
+                                   "load_addr": module_load_addr})
+
     def get_memory_region_containing_address(self, addr: int) -> lldb.SBMemoryRegionInfo:
         mem_region = lldb.SBMemoryRegionInfo()
         error = self.corefile_process.GetMemoryRegionInfo(addr, mem_region)
         if error.Fail():
             return None
         return mem_region
 
     def get_thread_with_id(self, tid: int):
def read_memory_at_address(self, addr: int, size: int) -> lldb.SBData:
 
         data.SetDataWithOwnership(error, bytes_read,
                                   self.corefile_target.GetByteOrder(),
                                   self.corefile_target.GetAddressByteSize())
 
         return data
 
     def get_loaded_images(self):
-        # TODO: Iterate over corefile_target modules and build a data structure
-        # from it.
         return self.loaded_images
 
     def get_process_id(self) -> int:
         return 42
 
     def should_stop(self) -> bool:
         return True