On Darwin systems, follow transition from on-disk dyld to shared-cache dyld at process launch

[jasonmolenda]

Summary

In macOS Ventura et al, the dynamic linker, dyld, has moved into the shared cache image. A process starts with our normal on-disk /usr/lib/dyld, and the first thing it does is execute into an in-memory-only copy of dyld in the shared cache blob. lldb needs to track this transition as we hand off from one dyld to the other, to get notified about newly added binary images in the process.

All of the action happens in DynamicLoaderMacOS::NotifyBreakpointHit(), where we detect two new cases: A new dyld_image_dyld_moved which is called when the launch dyld is about to transition over to the shared cache dyld, and we can get the address of the new notification function out of all_image_infos and set an address breakpoint and we clear out the list of known binary images. And a new mode of dyld_notify_adding where we have no binary images registered (because it was cleared in the previous step), so we know that this is a new shared cache dyld doing its first notification, and treat it appropriately, setting our normal by-name notification breakpoint.

rdar://84222158

[JDevlieghere]

LGTM. This was also reviewed internally and we've been living on this patch for a while now. Testing is provided by running the existing test suite on macOS Ventura.

lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h

protected:
 
   // Clear method for classes derived from this one
   virtual void DoClear() = 0;
 
   void SetDYLDModule(lldb::ModuleSP &dyld_module_sp);
 
   lldb::ModuleSP GetDYLDModule();
 
+  void ClearDYLDModule();
+
   class Segment {
   public:
     Segment() : name() {}
 
     lldb_private::ConstString name;
     lldb::addr_t vmaddr = LLDB_INVALID_ADDRESS;
     lldb::addr_t vmsize = 0;
     lldb::addr_t fileoff = 0;

lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp

void DynamicLoaderDarwin::SetDYLDModule(lldb::ModuleSP &dyld_module_sp) {
   m_dyld_module_wp = dyld_module_sp;
 }
 
 ModuleSP DynamicLoaderDarwin::GetDYLDModule() {
   ModuleSP dyld_sp(m_dyld_module_wp.lock());
   return dyld_sp;
 }
 
+void DynamicLoaderDarwin::ClearDYLDModule() { m_dyld_module_wp.reset(); }
+
 bool DynamicLoaderDarwin::AddModulesUsingImageInfos(
     ImageInfo::collection &image_infos) {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
   // Now add these images to the main list.
   ModuleList loaded_module_list;
   Log *log = GetLog(LLDBLog::DynamicLoader);
   Target &target = m_process->GetTarget();
   ModuleList &target_images = target.GetImages();

lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.h

protected:
   void PutToLog(lldb_private::Log *log) const;
 
   void DoInitialImageFetch() override;
 
   bool NeedToDoInitialImageFetch() override;
 
   bool DidSetNotificationBreakpoint() override;
 
+  bool SetDYLDHandoverBreakpoint(lldb::addr_t notification_address);
+
+  void ClearDYLDHandoverBreakpoint();
+
   void AddBinaries(const std::vector<lldb::addr_t> &load_addresses);
 
   void DoClear() override;
 
   static bool
   NotifyBreakpointHit(void *baton,
                       lldb_private::StoppointCallbackContext *context,
                       lldb::user_id_t break_id, lldb::user_id_t break_loc_id);
 
   bool SetNotificationBreakpoint() override;
 
   void ClearNotificationBreakpoint() override;
 
   void UpdateImageInfosHeaderAndLoadCommands(ImageInfo::collection &image_infos,
                                              uint32_t infos_count,
                                              bool update_executable);
 
   lldb::addr_t
   GetDyldLockVariableAddressFromModule(lldb_private::Module *module);
 
   uint32_t m_image_infos_stop_id; // The Stop ID the last time we
                                   // loaded/unloaded images
   lldb::user_id_t m_break_id;
+  lldb::user_id_t m_dyld_handover_break_id;
   mutable std::recursive_mutex m_mutex;
   lldb::addr_t m_maybe_image_infos_address; // If dyld is still maintaining the
                                             // all_image_infos address, store it
                                             // here so we can use it to detect
                                             // exec's when talking to
                                             // debugservers that don't support
                                             // the "reason:exec" annotation.
 };
 
 #endif // LLDB_SOURCE_PLUGINS_DYNAMICLOADER_MACOSX_DYLD_DYNAMICLOADERMACOS_H

lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp

 //===-- DynamicLoaderMacOS.cpp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/SymbolVendor.h"
 #include "lldb/Target/ABI.h"
+#include "lldb/Target/SectionLoadList.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
 
 #include "DynamicLoaderDarwin.h"
DynamicLoader *DynamicLoaderMacOS::CreateInstance(Process *process,
   if (create)
     return new DynamicLoaderMacOS(process);
   return nullptr;
 }
 
 // Constructor
 DynamicLoaderMacOS::DynamicLoaderMacOS(Process *process)
     : DynamicLoaderDarwin(process), m_image_infos_stop_id(UINT32_MAX),
-      m_break_id(LLDB_INVALID_BREAK_ID), m_mutex(),
+      m_break_id(LLDB_INVALID_BREAK_ID),
+      m_dyld_handover_break_id(LLDB_INVALID_BREAK_ID), m_mutex(),
       m_maybe_image_infos_address(LLDB_INVALID_ADDRESS) {}
 
 // Destructor
 DynamicLoaderMacOS::~DynamicLoaderMacOS() {
   if (LLDB_BREAK_ID_IS_VALID(m_break_id))
     m_process->GetTarget().RemoveBreakpointByID(m_break_id);
+  if (LLDB_BREAK_ID_IS_VALID(m_dyld_handover_break_id))
+    m_process->GetTarget().RemoveBreakpointByID(m_dyld_handover_break_id);
 }
 
 bool DynamicLoaderMacOS::ProcessDidExec() {
   std::lock_guard<std::recursive_mutex> baseclass_guard(GetMutex());
   bool did_exec = false;
   if (m_process) {
     // If we are stopped after an exec, we will have only one thread...
     if (m_process->GetThreadList().GetSize() == 1) {

 }
 
 // Clear out the state of this class.
 void DynamicLoaderMacOS::DoClear() {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
 
   if (LLDB_BREAK_ID_IS_VALID(m_break_id))
     m_process->GetTarget().RemoveBreakpointByID(m_break_id);
+  if (LLDB_BREAK_ID_IS_VALID(m_dyld_handover_break_id))
+    m_process->GetTarget().RemoveBreakpointByID(m_dyld_handover_break_id);
 
   m_break_id = LLDB_INVALID_BREAK_ID;
+  m_dyld_handover_break_id = LLDB_INVALID_BREAK_ID;
 }
 
 // Check if we have found DYLD yet
 bool DynamicLoaderMacOS::DidSetNotificationBreakpoint() {
   return LLDB_BREAK_ID_IS_VALID(m_break_id);
 }
 
 void DynamicLoaderMacOS::ClearNotificationBreakpoint() {
if (abi->GetArgumentValues(exe_ctx.GetThreadRef(), argument_values)) {
               addr_t addr = process->ReadUnsignedIntegerFromMemory(
                   header_array + (8 * i), 8, LLDB_INVALID_ADDRESS, error);
               if (addr != LLDB_INVALID_ADDRESS) {
                 image_load_addresses.push_back(addr);
               }
             }
             if (dyld_mode == 0) {
               // dyld_notify_adding
+              if (process->GetTarget().GetImages().GetSize() == 0) {
+                // When all images have been removed, we're doing the
+                // dyld handover from a launch-dyld to a shared-cache-dyld,
+                // and we've just hit our one-shot address breakpoint in
+                // the sc-dyld.  Note that the image addresses passed to
+                // this function are inferior sizeof(void*) not uint64_t's
+                // like our normal notification, so don't even look at
+                // image_load_addresses.
+
+                dyld_instance->ClearDYLDHandoverBreakpoint();
+
+                dyld_instance->DoInitialImageFetch();
+                dyld_instance->SetNotificationBreakpoint();
+              } else {
                 dyld_instance->AddBinaries(image_load_addresses);
+              }
             } else if (dyld_mode == 1) {
               // dyld_notify_removing
               dyld_instance->UnloadImages(image_load_addresses);
             } else if (dyld_mode == 2) {
               // dyld_notify_remove_all
               dyld_instance->UnloadAllImages();
+            } else if (dyld_mode == 3 && image_infos_count == 1) {
+              // dyld_image_dyld_moved
+
+              dyld_instance->ClearNotificationBreakpoint();
+              dyld_instance->UnloadAllImages();
+              dyld_instance->ClearDYLDModule();
+              process->GetTarget().GetImages().Clear();
+              process->GetTarget().GetSectionLoadList().Clear();
+
+              addr_t all_image_infos = process->GetImageInfoAddress();
+              int addr_size =
+                  process->GetTarget().GetArchitecture().GetAddressByteSize();
+              addr_t notification_location = all_image_infos + 4 + // version
+                                             4 +        // infoArrayCount
+                                             addr_size; // infoArray
+              Status error;
+              addr_t notification_addr =
+                  process->ReadPointerFromMemory(notification_location, error);
+              if (ABISP abi_sp = process->GetABI())
+                notification_addr = abi_sp->FixCodeAddress(notification_addr);
+
+              dyld_instance->SetDYLDHandoverBreakpoint(notification_addr);
             }
           }
         }
       }
     }
   } else {
     Target &target = process->GetTarget();
     Debugger::ReportWarning(
if (dyld_sp) {
                               true);
       breakpoint->SetBreakpointKind("shared-library-event");
       m_break_id = breakpoint->GetID();
     }
   }
   return m_break_id != LLDB_INVALID_BREAK_ID;
 }
 
+bool DynamicLoaderMacOS::SetDYLDHandoverBreakpoint(
+    addr_t notification_address) {
+  if (m_dyld_handover_break_id == LLDB_INVALID_BREAK_ID) {
+    BreakpointSP dyld_handover_bp = m_process->GetTarget().CreateBreakpoint(
+        notification_address, true, false);
+    dyld_handover_bp->SetCallback(DynamicLoaderMacOS::NotifyBreakpointHit, this,
+                                  true);
+    dyld_handover_bp->SetOneShot(true);
+    m_dyld_handover_break_id = dyld_handover_bp->GetID();
+    return true;
+  }
+  return false;
+}
+
+void DynamicLoaderMacOS::ClearDYLDHandoverBreakpoint() {
+  if (LLDB_BREAK_ID_IS_VALID(m_dyld_handover_break_id))
+    m_process->GetTarget().RemoveBreakpointByID(m_dyld_handover_break_id);
+  m_dyld_handover_break_id = LLDB_INVALID_BREAK_ID;
+}
+
 addr_t
 DynamicLoaderMacOS::GetDyldLockVariableAddressFromModule(Module *module) {
   SymbolContext sc;
   Target &target = m_process->GetTarget();
   if (Symtab *symtab = module->GetSymtab()) {
     std::vector<uint32_t> match_indexes;
     ConstString g_symbol_name("_dyld_global_lock_held");
     uint32_t num_matches = 0;