[LLDB] Add SymbolVendorWasm plugin for WebAssembly debugging

[paolosev]

Summary

Add plugin class SymbolVendorWasm, with the logic to manage debug symbols for Wasm modules.

Depends on D71575.

[labath]

The patch looks pretty good. A reasonable way to test this would be again via lldb-test object-file . The command dumps the "unified section list" of the module, so if the debug info sections show up there, you know the symbol vendor has done it's job. You can look at test/Shell/ObjectFile/ELF/build-id-case.yaml for inspiration.

[paolosev]

Added "lldb-test object-file" test.

[paolosev]

In D72650#1819403, @labath wrote:

The patch looks pretty good. A reasonable way to test this would be again via lldb-test object-file . The command dumps the "unified section list" of the module, so if the debug info sections show up there, you know the symbol vendor has done it's job. You can look at test/Shell/ObjectFile/ELF/build-id-case.yaml for inspiration.

Thank you for the suggestion! I added a test.
I would have liked to use llvm-objcopy --strip-all in my test, but llvm-objcopy does not support wasm yet (I started working on this feature but I found out that there was already an ongoing effort: https://reviews.llvm.org/D70930, and https://reviews.llvm.org/D70970) .
So I created with two separated yaml files.

[paolosev]

Rebasing from D71575.

[labath]

In D72650#1820897, @paolosev wrote:

In D72650#1819403, @labath wrote:

The patch looks pretty good. A reasonable way to test this would be again via lldb-test object-file . The command dumps the "unified section list" of the module, so if the debug info sections show up there, you know the symbol vendor has done it's job. You can look at test/Shell/ObjectFile/ELF/build-id-case.yaml for inspiration.

Thank you for the suggestion! I added a test.
I would have liked to use llvm-objcopy --strip-all in my test, but llvm-objcopy does not support wasm yet (I started working on this feature but I found out that there was already an ongoing effort: https://reviews.llvm.org/D70930, and https://reviews.llvm.org/D70970) .
So I created with two separated yaml files.

That's fine. I think I actually prefer separate yaml representations, as that makes what goes into which file more explicit, and shields the test from changes in objcopy behavior. What you could consider is "inlining" the two yaml files into unified-debug-sections.yaml, and using the yaml2obj --docnum functionality (as in e.g., test/Shell/Minidump/memory-region-from-module.yaml) to create two wasm files.

[paolosev]

Modified test to have two "inlined" yaml files and use "yaml2obj --docnum".

[labath]

This looks good.

lldb/source/API/SystemInitializerFull.cpp

 #include "Plugins/Process/minidump/ProcessMinidump.h"
 #include "Plugins/ScriptInterpreter/None/ScriptInterpreterNone.h"
 #include "Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARF.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h"
 #include "Plugins/SymbolFile/PDB/SymbolFilePDB.h"
 #include "Plugins/SymbolFile/Symtab/SymbolFileSymtab.h"
 #include "Plugins/SymbolVendor/ELF/SymbolVendorELF.h"
+#include "Plugins/SymbolVendor/wasm/SymbolVendorWasm.h"
 #include "Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.h"
 #include "Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h"
 #include "Plugins/UnwindAssembly/x86/UnwindAssembly-x86.h"
 
 #if defined(__APPLE__)
 #include "Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.h"
 #include "Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h"
 #include "Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h"
#include "llvm/Config/Targets.def"
   UndefinedBehaviorSanitizerRuntime::Initialize();
   MainThreadCheckerRuntime::Initialize();
 
   SymbolVendorELF::Initialize();
   breakpad::SymbolFileBreakpad::Initialize();
   SymbolFileDWARF::Initialize();
   SymbolFilePDB::Initialize();
   SymbolFileSymtab::Initialize();
+  wasm::SymbolVendorWasm::Initialize();
   UnwindAssemblyInstEmulation::Initialize();
   UnwindAssembly_x86::Initialize();
 
   EmulateInstructionARM::Initialize();
   EmulateInstructionARM64::Initialize();
   EmulateInstructionMIPS::Initialize();
   EmulateInstructionMIPS64::Initialize();
   EmulateInstructionPPC64::Initialize();
#include "llvm/Config/Targets.def"
   ProcessElfCore::Terminate();
   ProcessMachCore::Terminate();
   minidump::ProcessMinidump::Terminate();
   MemoryHistoryASan::Terminate();
   AddressSanitizerRuntime::Terminate();
   ThreadSanitizerRuntime::Terminate();
   UndefinedBehaviorSanitizerRuntime::Terminate();
   MainThreadCheckerRuntime::Terminate();
+  wasm::SymbolVendorWasm::Terminate();
   SymbolVendorELF::Terminate();
   breakpad::SymbolFileBreakpad::Terminate();
   SymbolFileDWARF::Terminate();
   SymbolFilePDB::Terminate();
   SymbolFileSymtab::Terminate();
   UnwindAssembly_x86::Terminate();
   UnwindAssemblyInstEmulation::Terminate();
 

lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.h

static size_t GetModuleSpecifications(const FileSpec &file,
                                         ModuleSpecList &specs);
 
   /// PluginInterface protocol.
   /// \{
   ConstString GetPluginName() override { return GetPluginNameStatic(); }
   uint32_t GetPluginVersion() override { return 1; }
   /// \}
 
+  /// LLVM RTTI support
+  /// \{
+  static char ID;
+  bool isA(const void *ClassID) const override {
+    return ClassID == &ID || ObjectFile::isA(ClassID);
+  }
+  static bool classof(const ObjectFile *obj) { return obj->isA(&ID); }
+  /// \}
+
   /// ObjectFile Protocol.
   /// \{
   bool ParseHeader() override;
 
   lldb::ByteOrder GetByteOrder() const override {
     return m_arch.GetByteOrder();
   }
 
bool SetLoadAddress(lldb_private::Target &target, lldb::addr_t value,
                       bool value_is_offset) override;
 
   lldb_private::Address GetBaseAddress() override {
     return IsInMemory() ? Address(m_memory_addr + m_code_section_offset)
                         : Address(m_code_section_offset);
   }
   /// \}
 
+  /// A Wasm module that has external DWARF debug information should contain a
+  /// custom section named "external_debug_info", whose payload is an UTF-8
+  /// encoded string that points to a Wasm module that contains the debug
+  /// information for this module.
+  llvm::Optional<FileSpec> GetExternalDebugInfoFileSpec();
+
 private:
   ObjectFileWasm(const lldb::ModuleSP &module_sp, lldb::DataBufferSP &data_sp,
                  lldb::offset_t data_offset, const FileSpec *file,
                  lldb::offset_t offset, lldb::offset_t length);
   ObjectFileWasm(const lldb::ModuleSP &module_sp,
                  lldb::DataBufferSP &header_data_sp,
                  const lldb::ProcessSP &process_sp, lldb::addr_t header_addr);
 

lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp

if (!c) {
     consumeError(c.takeError());
     return llvm::None;
   }
 
   llvm::StringRef str = toStringRef(makeArrayRef(str_storage));
   return ConstString(str);
 }
 
+char ObjectFileWasm::ID;
+
 void ObjectFileWasm::Initialize() {
   PluginManager::RegisterPlugin(GetPluginNameStatic(),
                                 GetPluginDescriptionStatic(), CreateInstance,
                                 CreateMemoryInstance, GetModuleSpecifications);
 }
 
 void ObjectFileWasm::Terminate() {
   PluginManager::UnregisterPlugin(CreateInstance);
bool ObjectFileWasm::DecodeNextSection(lldb::offset_t *offset_ptr) {
   uint64_t payload_len = data.getULEB128(c);
   if (!c)
     return !llvm::errorToBool(c.takeError());
 
   if (payload_len >= (uint64_t(1) << 32))
     return false;
 
   if (section_id == llvm::wasm::WASM_SEC_CUSTOM) {
+    // Custom sections have the id 0. Their contents consist of a name
+    // identifying the custom section, followed by an uninterpreted sequence
+    // of bytes.
     lldb::offset_t prev_offset = c.tell();
     llvm::Optional<ConstString> sect_name = GetWasmString(data, c);
     if (!sect_name)
       return false;
 
     if (payload_len < c.tell() - prev_offset)
       return false;
 
if (process_sp) {
       }
     }
   }
 
   data.SetByteOrder(GetByteOrder());
   return data;
 }
 
+llvm::Optional<FileSpec> ObjectFileWasm::GetExternalDebugInfoFileSpec() {
+  static ConstString g_sect_name_external_debug_info("external_debug_info");
+
+  for (const section_info &sect_info : m_sect_infos) {
+    if (g_sect_name_external_debug_info == sect_info.name) {
+      const uint32_t kBufferSize = 1024;
+      DataExtractor section_header_data =
+          ReadImageData(sect_info.offset, kBufferSize);
+      llvm::DataExtractor data = section_header_data.GetAsLLVM();
+      llvm::DataExtractor::Cursor c(0);
+      llvm::Optional<ConstString> symbols_url = GetWasmString(data, c);
+      if (symbols_url)
+        return FileSpec(symbols_url->GetStringRef());
+    }
+  }
+  return llvm::None;
+}
+
 void ObjectFileWasm::Dump(Stream *s) {
   ModuleSP module_sp(GetModule());
   if (!module_sp)
     return;
 
   std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
 
   llvm::raw_ostream &ostream = s->AsRawOstream();

lldb/source/Plugins/SymbolVendor/CMakeLists.txt

 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_subdirectory(MacOSX)
 endif()
 
 add_subdirectory(ELF)
+add_subdirectory(wasm)

lldb/source/Plugins/SymbolVendor/wasm/CMakeLists.txt

+add_lldb_library(lldbPluginSymbolVendorWasm PLUGIN
+  SymbolVendorWasm.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbHost
+    lldbSymbol
+    lldbPluginObjectFileWasm
+  )

lldb/source/Plugins/SymbolVendor/wasm/SymbolVendorWasm.h

+//===-- SymbolVendorWasm.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef liblldb_SymbolVendorWasm_h_
+#define liblldb_SymbolVendorWasm_h_
+
+#include "lldb/Symbol/SymbolVendor.h"
+#include "lldb/lldb-private.h"
+
+namespace lldb_private {
+namespace wasm {
+
+class SymbolVendorWasm : public lldb_private::SymbolVendor {
+public:
+  SymbolVendorWasm(const lldb::ModuleSP &module_sp);
+
+  static void Initialize();
+  static void Terminate();
+  static lldb_private::ConstString GetPluginNameStatic();
+  static const char *GetPluginDescriptionStatic();
+
+  static lldb_private::SymbolVendor *
+  CreateInstance(const lldb::ModuleSP &module_sp,
+                 lldb_private::Stream *feedback_strm);
+
+  /// PluginInterface protocol.
+  /// \{
+  lldb_private::ConstString GetPluginName() override;
+  uint32_t GetPluginVersion() override;
+  /// \}
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(SymbolVendorWasm);
+};
+
+} // namespace wasm
+} // namespace lldb_private
+
+#endif // liblldb_SymbolVendorWasm_h_

lldb/source/Plugins/SymbolVendor/wasm/SymbolVendorWasm.cpp

+//===-- SymbolVendorWasm.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolVendorWasm.h"
+
+#include <string.h>
+
+#include "Plugins/ObjectFile/wasm/ObjectFileWasm.h"
+#include "lldb/Core/Module.h"
+#include "lldb/Core/ModuleSpec.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Section.h"
+#include "lldb/Host/Host.h"
+#include "lldb/Symbol/LocateSymbolFile.h"
+#include "lldb/Symbol/ObjectFile.h"
+#include "lldb/Target/Target.h"
+#include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/Timer.h"
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::wasm;
+
+// SymbolVendorWasm constructor
+SymbolVendorWasm::SymbolVendorWasm(const lldb::ModuleSP &module_sp)
+    : SymbolVendor(module_sp) {}
+
+void SymbolVendorWasm::Initialize() {
+  PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                GetPluginDescriptionStatic(), CreateInstance);
+}
+
+void SymbolVendorWasm::Terminate() {
+  PluginManager::UnregisterPlugin(CreateInstance);
+}
+
+lldb_private::ConstString SymbolVendorWasm::GetPluginNameStatic() {
+  static ConstString g_name("WASM");
+  return g_name;
+}
+
+const char *SymbolVendorWasm::GetPluginDescriptionStatic() {
+  return "Symbol vendor for WASM that looks for dwo files that match "
+         "executables.";
+}
+
+// CreateInstance
+//
+// Platforms can register a callback to use when creating symbol vendors to
+// allow for complex debug information file setups, and to also allow for
+// finding separate debug information files.
+SymbolVendor *
+SymbolVendorWasm::CreateInstance(const lldb::ModuleSP &module_sp,
+                                 lldb_private::Stream *feedback_strm) {
+  if (!module_sp)
+    return nullptr;
+
+  ObjectFileWasm *obj_file =
+      llvm::dyn_cast_or_null<ObjectFileWasm>(module_sp->GetObjectFile());
+  if (!obj_file)
+    return nullptr;
+
+  // If the main object file already contains debug info, then we are done.
+  if (obj_file->GetSectionList()->FindSectionByType(
+          lldb::eSectionTypeDWARFDebugInfo, true))
+    return nullptr;
+
+  static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
+  Timer scoped_timer(func_cat, "SymbolVendorWasm::CreateInstance (module = %s)",
+                     module_sp->GetFileSpec().GetPath().c_str());
+
+  ModuleSpec module_spec;
+  module_spec.GetFileSpec() = obj_file->GetFileSpec();
+  FileSystem::Instance().Resolve(module_spec.GetFileSpec());
+  module_spec.GetUUID() = obj_file->GetUUID();
+
+  // A Wasm module may have a custom section named "external_debug_info" whose
+  // content is the absolute or relative path of the Wasm module that contains
+  // debug symbols for this module.
+  llvm::Optional<FileSpec> symbol_file_spec =
+      obj_file->GetExternalDebugInfoFileSpec();
+  if (!symbol_file_spec)
+    return nullptr;
+  module_spec.GetSymbolFileSpec() = *symbol_file_spec;
+
+  FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths();
+  FileSpec sym_fspec =
+      Symbols::LocateExecutableSymbolFile(module_spec, search_paths);
+  if (!sym_fspec)
+    return nullptr;
+
+  DataBufferSP sym_file_data_sp;
+  lldb::offset_t sym_file_data_offset = 0;
+  ObjectFileSP sym_objfile_sp = ObjectFile::FindPlugin(
+      module_sp, &sym_fspec, 0, FileSystem::Instance().GetByteSize(sym_fspec),
+      sym_file_data_sp, sym_file_data_offset);
+  if (!sym_objfile_sp)
+    return nullptr;
+
+  // This objfile is for debugging purposes.
+  sym_objfile_sp->SetType(ObjectFile::eTypeDebugInfo);
+
+  SymbolVendorWasm *symbol_vendor = new SymbolVendorWasm(module_sp);
+
+  // Get the module unified section list and add our debug sections to
+  // that.
+  SectionList *module_section_list = module_sp->GetSectionList();
+  SectionList *objfile_section_list = sym_objfile_sp->GetSectionList();
+
+  static const SectionType g_sections[] = {
+      eSectionTypeDWARFDebugAbbrev,   eSectionTypeDWARFDebugAddr,
+      eSectionTypeDWARFDebugAranges,  eSectionTypeDWARFDebugCuIndex,
+      eSectionTypeDWARFDebugFrame,    eSectionTypeDWARFDebugInfo,
+      eSectionTypeDWARFDebugLine,     eSectionTypeDWARFDebugLineStr,
+      eSectionTypeDWARFDebugLoc,      eSectionTypeDWARFDebugLocLists,
+      eSectionTypeDWARFDebugMacInfo,  eSectionTypeDWARFDebugMacro,
+      eSectionTypeDWARFDebugPubNames, eSectionTypeDWARFDebugPubTypes,
+      eSectionTypeDWARFDebugRanges,   eSectionTypeDWARFDebugRngLists,
+      eSectionTypeDWARFDebugStr,      eSectionTypeDWARFDebugStrOffsets,
+      eSectionTypeDWARFDebugTypes};
+  for (SectionType section_type : g_sections) {
+    if (SectionSP section_sp =
+            objfile_section_list->FindSectionByType(section_type, true)) {
+      if (SectionSP module_section_sp =
+              module_section_list->FindSectionByType(section_type, true))
+        module_section_list->ReplaceSection(module_section_sp->GetID(),
+                                            section_sp);
+      else
+        module_section_list->AddSection(section_sp);
+    }
+  }
+
+  symbol_vendor->AddSymbolFileRepresentation(sym_objfile_sp);
+  return symbol_vendor;
+}
+
+// PluginInterface protocol
+ConstString SymbolVendorWasm::GetPluginName() { return GetPluginNameStatic(); }
+
+uint32_t SymbolVendorWasm::GetPluginVersion() { return 1; }

lldb/test/Shell/ObjectFile/wasm/unified-debug-sections.yaml

+# RUN: yaml2obj -docnum=1 %s > test.wasm
+# RUN: yaml2obj -docnum=2 %s > test_sym.wasm
+# RUN: lldb-test object-file test.wasm | FileCheck %s
+
+# This test checks that SymbolVendorWasm correctly loads DWARF debug sections
+# that have been stripped out into a separated Wasm module. The original Wasm
+# module contains a "external_debug_info" custom section with the absolute or
+# relative path of the debug module.
+
+# CHECK: Plugin name: wasm
+# CHECK: Architecture: wasm32-unknown-unknown-wasm
+# CHECK: UUID: 
+# CHECK: Executable: true
+# CHECK: Stripped: true
+# CHECK: Type: executable
+# CHECK: Strata: user
+# CHECK: Base VM address: 0xa
+
+# CHECK: Name: code
+# CHECK: Type: code
+# CHECK: VM address: 0x0
+# CHECK: VM size: 56
+# CHECK: File size: 56
+
+# CHECK: Name: .debug_info
+# CHECK: Type: dwarf-info
+# CHECK: VM address: 0x0
+# CHECK: VM size: 0
+# CHECK: File size: 2
+
+# CHECK: Name: .debug_abbrev
+# CHECK: Type: dwarf-abbrev
+# CHECK: VM address: 0x0
+# CHECK: VM size: 0
+# CHECK: File size: 2
+
+# CHECK: Name: .debug_line
+# CHECK: Type: dwarf-line
+# CHECK: VM address: 0x0
+# CHECK: VM size: 0
+# CHECK: File size: 2
+
+# CHECK: Name: .debug_str
+# CHECK: Type: dwarf-str
+# CHECK: VM address: 0x0
+# CHECK: VM size: 0
+# CHECK: File size: 3
+
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:
+          - Type:            I32
+            Count:           6
+        Body:            238080808000210141102102200120026B21032003200036020C200328020C2104200328020C2105200420056C210620060F0B
+  - Type:            CUSTOM
+    Name:            external_debug_info
+    Payload:         0D746573745F73796D2E7761736D  # test_sym.wasm
+
+...
+
+
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+
+  - Type:            CUSTOM
+    Name:            .debug_info
+    Payload:         4C00
+  - Type:            CUSTOM
+    Name:            .debug_abbrev
+    Payload:         0111
+  - Type:            CUSTOM
+    Name:            .debug_line
+    Payload:         5100
+  - Type:            CUSTOM
+    Name:            .debug_str
+    Payload:         636CFF
+
+...

lldb/tools/lldb-test/SystemInitializerTest.cpp

 #include "Plugins/Process/minidump/ProcessMinidump.h"
 #include "Plugins/ScriptInterpreter/None/ScriptInterpreterNone.h"
 #include "Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARF.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h"
 #include "Plugins/SymbolFile/PDB/SymbolFilePDB.h"
 #include "Plugins/SymbolFile/Symtab/SymbolFileSymtab.h"
 #include "Plugins/SymbolVendor/ELF/SymbolVendorELF.h"
+#include "Plugins/SymbolVendor/wasm/SymbolVendorWasm.h"
 #include "Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.h"
 #include "Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h"
 #include "Plugins/UnwindAssembly/x86/UnwindAssembly-x86.h"
 
 #if defined(__APPLE__)
 #include "Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.h"
 #include "Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h"
 #include "Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h"
#include "llvm/Config/Targets.def"
   UndefinedBehaviorSanitizerRuntime::Initialize();
   MainThreadCheckerRuntime::Initialize();
 
   SymbolVendorELF::Initialize();
   breakpad::SymbolFileBreakpad::Initialize();
   SymbolFileDWARF::Initialize();
   SymbolFilePDB::Initialize();
   SymbolFileSymtab::Initialize();
+  wasm::SymbolVendorWasm::Initialize();
   UnwindAssemblyInstEmulation::Initialize();
   UnwindAssembly_x86::Initialize();
   EmulateInstructionARM64::Initialize();
   EmulateInstructionPPC64::Initialize();
   SymbolFileDWARFDebugMap::Initialize();
   ItaniumABILanguageRuntime::Initialize();
   AppleObjCRuntimeV2::Initialize();
   AppleObjCRuntimeV1::Initialize();
#include "llvm/Config/Targets.def"
   ThreadSanitizerRuntime::Terminate();
   UndefinedBehaviorSanitizerRuntime::Terminate();
   MainThreadCheckerRuntime::Terminate();
   SymbolVendorELF::Terminate();
   breakpad::SymbolFileBreakpad::Terminate();
   SymbolFileDWARF::Terminate();
   SymbolFilePDB::Terminate();
   SymbolFileSymtab::Terminate();
+  wasm::SymbolVendorWasm::Terminate();
   UnwindAssembly_x86::Terminate();
   UnwindAssemblyInstEmulation::Terminate();
   EmulateInstructionARM64::Terminate();
   EmulateInstructionPPC64::Terminate();
   SymbolFileDWARFDebugMap::Terminate();
   ItaniumABILanguageRuntime::Terminate();
   AppleObjCRuntimeV2::Terminate();
   AppleObjCRuntimeV1::Terminate();