[lldb] DWZ 12/17: DIERef support

[jankratochvil]

Summary

DIERef needs to contain more context information as DWZ needs to track main unit and it also needs to locate some DIEs in a separate shared common file in /usr/lib/debug/.dwz/ .

[jankratochvil]

My question about upstreaming of this patchset.

[jankratochvil]

I will be no longer involved with this patchset.

lldb/source/Plugins/SymbolFile/DWARF/DIERef.h

 
 #include "lldb/Core/dwarf.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/FormatProviders.h"
 #include <cassert>
 #include <vector>
 
-/// Identifies a DWARF debug info entry within a given Module. It contains three
-/// "coordinates":
-/// - dwo_num: identifies the dwo file in the Module. If this field is not set,
-///   the DIERef references the main file.
+/// Identifies a DWARF debug info entry within a given Module. It contains
+/// multiple "coordinates":
+/// - dwo_num: -gsplit-dwarf only: identifies the dwo file in the Module.
+///   If this field is not set, the DIERef references the main file.
+/// - main_cu: DWZ only: index of main compilation unit which used
+///   DW_TAG_imported_unit for this DIERef or some its parent.
+///   For transitive imports this is the very first CU.
+///   It is always a CU in the main file (and not DWZ common symbol file).
+/// - DWZ common flag: DWZ only: the DIE is in a shared DWZ common symbol file.
 /// - section: identifies the section of the debug info entry in the given file:
 ///   debug_info or debug_types.
 /// - die_offset: The offset of the debug info entry as an absolute offset from
 ///   the beginning of the section specified in the section field.
 class DIERef {
 public:
   enum Section : uint8_t { DebugInfo, DebugTypes };
 
-  DIERef(llvm::Optional<uint32_t> dwo_num, Section section,
-         dw_offset_t die_offset)
-      : m_u(dwo_num, section), m_die_offset(die_offset) {
+  // DWZ only: Is the DIE located in DWZ common symbol file?
+  enum DwzCommon : uint8_t { MainDwz, CommonDwz };
+
+  enum Kind : uint8_t {
+    NoneKind,
+
+    // DWZ only: The DIE is in DW_TAG_partial_unit in main symbol file and it
+    // needs main_cu.
+    MainDwzKind,
+
+    // -gsplit-dwarf only: The DIE is in a separate dwo file specified by
+    // dwo_num.
+    DwoKind,
+
+    // DWZ only: The DIE is in DW_TAG_partial_unit in DWZ common symbol file and
+    // it needs main_cu.
+    DwzCommonKind
+  };
+
+  DIERef(llvm::Optional<uint32_t> dwo_num, llvm::Optional<uint32_t> main_cu,
+         DwzCommon dwz_common, Section section, dw_offset_t die_offset)
+      : m_u(dwo_num, main_cu, dwz_common, section), m_die_offset(die_offset) {
     assert(this->dwo_num() == dwo_num && "Dwo number out of range?");
+    assert(this->main_cu() == main_cu && "Main Cu number out of range?");
+    assert(dwz_common == MainDwz || main_cu);
   }
 
   llvm::Optional<uint32_t> dwo_num() const {
-    if (m_u.s.dwo_num_valid)
-      return m_u.s.dwo_num;
+    if (m_u.s.data_kind == DwoKind)
+      return m_u.s.data;
+    return llvm::None;
+  }
+
+  // It indexes DWARFCompileUnit's excl. DWARFTypeUnit's.
+  // It is the index used as parameter of SymbolFileDWARF::GetDWARFUnitIndex.
+  llvm::Optional<uint32_t> main_cu() const {
+    if (m_u.s.data_kind == MainDwzKind || m_u.s.data_kind == DwzCommonKind)
+      return m_u.s.data;
     return llvm::None;
   }
 
+  DwzCommon dwz_common() const {
+    assert(m_u.s.data_kind == MainDwzKind || m_u.s.data_kind == DwzCommonKind);
+    return m_u.s.data_kind == MainDwzKind ? MainDwz : CommonDwz;
+  }
+
+  Kind kind_get() const { return Kind(m_u.s.data_kind); }
+
   Section section() const { return static_cast<Section>(m_u.s.section); }
 
   dw_offset_t die_offset() const { return m_die_offset; }
 
   bool operator<(DIERef other) const {
-    if (m_u.s.dwo_num_valid != other.m_u.s.dwo_num_valid)
-      return m_u.s.dwo_num_valid < other.m_u.s.dwo_num_valid;
-    // Assuming if not m_u.s.dwo_num_valid then m_u.s.dwo_num is zero.
-    if (m_u.s.dwo_num != other.m_u.s.dwo_num)
-      return m_u.s.dwo_num < other.m_u.s.dwo_num;
+    if (m_u.s.data_kind != other.m_u.s.data_kind)
+      return m_u.s.data_kind < other.m_u.s.data_kind;
+    // Assuming if m_u.s.data_kind == NoneKind then m_u.s.data is zero.
+    if (m_u.s.data != other.m_u.s.data)
+      return m_u.s.data < other.m_u.s.data;
     if (m_u.s.section != other.m_u.s.section)
       return m_u.s.section < other.m_u.s.section;
     return m_die_offset < other.m_die_offset;
   }
 
   bool operator==(DIERef other) const {
-    if (m_u.s.dwo_num_valid != other.m_u.s.dwo_num_valid)
-      return m_u.s.dwo_num_valid == other.m_u.s.dwo_num_valid;
-    // Assuming if not m_u.s.dwo_num_valid then m_u.s.dwo_num is zero.
-    if (m_u.s.dwo_num != other.m_u.s.dwo_num)
-      return m_u.s.dwo_num == other.m_u.s.dwo_num;
+    if (m_u.s.data_kind != other.m_u.s.data_kind)
+      return m_u.s.data_kind == other.m_u.s.data_kind;
+    // Assuming if m_u.s.data_kind == NoneKind then m_u.s.data is zero.
+    if (m_u.s.data != other.m_u.s.data)
+      return m_u.s.data == other.m_u.s.data;
     if (m_u.s.section != other.m_u.s.section)
       return m_u.s.section == other.m_u.s.section;
     return m_die_offset == other.m_die_offset;
   }
 
 private:
   friend struct llvm::DenseMapInfo<DIERef>;
-  DIERef(unsigned unique) : m_u(llvm::None, DebugInfo), m_die_offset(0) {
-    m_u.s.dwo_num = unique;
+  DIERef(unsigned unique)
+      : m_u(llvm::None, llvm::None, MainDwz, DebugInfo), m_die_offset(0) {
+    m_u.s.data = unique;
   }
   uint32_t get_hash_value() const {
     return llvm::detail::combineHashValue(m_u.hash_bits, m_die_offset);
   }
 
   union U {
     struct S {
-      uint32_t dwo_num : 30;
-      uint32_t dwo_num_valid : 1;
-      uint32_t section : 1;
-      S(llvm::Optional<uint32_t> dwo_num_, Section section_)
-          : dwo_num(dwo_num_.getValueOr(0)), dwo_num_valid(bool(dwo_num_)),
+      uint32_t data : 29;
+      uint32_t data_kind : 2; // Kind type
+      uint32_t section : 1;   // Section type
+      S(llvm::Optional<uint32_t> dwo_num, llvm::Optional<uint32_t> main_cu,
+        DwzCommon dwz_common, Section section_)
+          : data(dwo_num.getValueOr(0) | main_cu.getValueOr(0)),
+            data_kind(dwo_num
+                          ? DwoKind
+                          : (main_cu ? (dwz_common == MainDwz ? MainDwzKind
+                                                              : DwzCommonKind)
+                                     : NoneKind)),
             section(section_) {}
     } s;
     uint32_t hash_bits;
-    U(llvm::Optional<uint32_t> dwo_num, Section section)
-        : s(dwo_num, section) {}
+    U(llvm::Optional<uint32_t> dwo_num, llvm::Optional<uint32_t> main_cu,
+      DwzCommon dwz_common, Section section)
+        : s(dwo_num, main_cu, dwz_common, section) {}
   } m_u;
   dw_offset_t m_die_offset;
 };
 static_assert(sizeof(DIERef) == 8, "");
 
 typedef std::vector<DIERef> DIEArray;
 
 namespace llvm {

lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp

 //===-- DIERef.cpp --------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "DIERef.h"
 #include "llvm/Support/Format.h"
 
 void llvm::format_provider<DIERef>::format(const DIERef &ref, raw_ostream &OS,
                                            StringRef Style) {
   if (ref.dwo_num())
     OS << format_hex_no_prefix(*ref.dwo_num(), 8) << "/";
-  OS << (ref.section() == DIERef::DebugInfo ? "INFO" : "TYPE");
+  if (ref.main_cu())
+    OS << format_hex_no_prefix(*ref.main_cu(), 8) << "/";
+  OS << (ref.section() == DIERef::Section::DebugInfo ? "INFO" : "TYPE");
+  switch (ref.kind_get()) {
+  case DIERef::Kind::NoneKind:
+  case DIERef::Kind::DwoKind:
+    break;
+  case DIERef::Kind::MainDwzKind:
+    OS << "/DWZ";
+    break;
+  case DIERef::Kind::DwzCommonKind:
+    OS << "/DWZCOMMON";
+    break;
+  }
   OS << "/" << format_hex_no_prefix(ref.die_offset(), 8);
 }

lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp

 #include "lldb/Utility/Log.h"
 
 using namespace lldb_private;
 
 llvm::Optional<DIERef> DWARFBaseDIE::GetDIERef() const {
   if (!IsValid())
     return llvm::None;
 
-  return DIERef(m_cu->GetSymbolFileDWARF().GetDwoNum(), m_cu->GetDebugSection(),
-                m_die->GetOffset());
+  DWARFUnit *main_unit = GetMainCU();
+
+  if (GetCU()->GetSymbolFileDWARF().GetDwoNum().hasValue())
+    main_unit = nullptr;
+  if (GetCU().GetCU() == main_unit)
+    main_unit = nullptr;
+
+  return DIERef(
+      m_cu->GetSymbolFileDWARF().GetDwoNum(),
+      (!main_unit ? llvm::None : llvm::Optional<uint32_t>(main_unit->GetID())),
+      DIERef::MainDwz, m_cu->GetDebugSection(), m_die->GetOffset());
 }
 
 dw_tag_t DWARFBaseDIE::Tag() const {
   if (m_die)
     return m_die->Tag();
   else
     return llvm::dwarf::DW_TAG_null;
 }

lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp

DebugNamesDWARFIndex::ToDIERef(const DebugNames::Entry &entry) {
 
   DWARFUnit *cu = m_debug_info.GetUnitAtOffset(DIERef::Section::DebugInfo, *cu_offset);
   if (!cu)
     return llvm::None;
 
   cu = &cu->GetNonSkeletonUnit();
   if (llvm::Optional<uint64_t> die_offset = entry.getDIEUnitOffset())
     // FIXME: .debug_names have no DWZ support yet.
-    return DIERef(cu->GetSymbolFileDWARF().GetDwoNum(),
-                  DIERef::Section::DebugInfo, cu->GetOffset() + *die_offset);
+    return DIERef(cu->GetSymbolFileDWARF().GetDwoNum(), llvm::None,
+                  DIERef::MainDwz, DIERef::Section::DebugInfo,
+                  cu->GetOffset() + *die_offset);
 
   return llvm::None;
 }
 
 bool DebugNamesDWARFIndex::ProcessEntry(
     const DebugNames::Entry &entry,
     llvm::function_ref<bool(DWARFDIE die)> callback, llvm::StringRef name) {
   llvm::Optional<DIERef> ref = ToDIERef(entry);

lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h

struct DIEInfo {
 
     /// A 32 bit hash of the fully qualified name.
     uint32_t qualified_name_hash = 0;
 
     DIEInfo() = default;
     DIEInfo(dw_offset_t o, dw_tag_t t, uint32_t f, uint32_t h);
 
     explicit operator DIERef() const {
-      return DIERef(llvm::None, DIERef::Section::DebugInfo, die_offset);
+      return DIERef(llvm::None, llvm::None, DIERef::MainDwz,
+                    DIERef::Section::DebugInfo, die_offset);
     }
   };
 
   struct Atom {
     AtomType type;
     dw_form_t form;
   };
 

lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp

 //===-- NameToDIE.cpp -----------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "NameToDIE.h"
+#include "DWARFDebugInfo.h"
 #include "DWARFUnit.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
 
 using namespace lldb;
 using namespace lldb_private;
for (const auto &entry : m_map)
     }
   return true;
 }
 
 void NameToDIE::FindAllEntriesForUnit(
     DWARFUnit &s_unit, llvm::function_ref<bool(DIERef ref)> callback) const {
   lldbassert(!s_unit.GetSymbolFileDWARF().GetDwoNum());
   const DWARFUnit &ns_unit = s_unit.GetNonSkeletonUnit();
+  SymbolFileDWARF *ns_symfile = &ns_unit.GetSymbolFileDWARF();
   const uint32_t size = m_map.GetSize();
   for (uint32_t i = 0; i < size; ++i) {
     const DIERef &die_ref = m_map.GetValueAtIndexUnchecked(i);
-    if (ns_unit.GetSymbolFileDWARF().GetDwoNum() == die_ref.dwo_num() &&
-        ns_unit.GetDebugSection() == die_ref.section() &&
+    if (ns_unit.GetDebugSection() != die_ref.section())
+      continue;
+    if (die_ref.main_cu()) {
+      DWARFDebugInfo &ns_info = ns_symfile->DebugInfo();
+      DWARFUnit *die_main_unit = ns_info.GetUnitAtIndex(
+          *ns_symfile->GetDWARFUnitIndex(*die_ref.main_cu()));
+      if (&ns_unit != die_main_unit)
+        continue;
+    } else if (!(ns_symfile->GetDwoNum() == die_ref.dwo_num() &&
                  ns_unit.GetOffset() <= die_ref.die_offset() &&
-        die_ref.die_offset() < ns_unit.GetNextUnitOffset()) {
-      if (!callback(die_ref))
-        return;
-    }
-  }
-}
+                 die_ref.die_offset() < ns_unit.GetNextUnitOffset()))
+      continue;
 
 void NameToDIE::Dump(Stream *s) {
   const uint32_t size = m_map.GetSize();
   for (uint32_t i = 0; i < size; ++i) {
     s->Format("{0} \"{1}\"\n", m_map.GetValueAtIndexUnchecked(i),
               m_map.GetCStringAtIndexUnchecked(i));
   }
 }

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h

public:
   static DWARFDeclContext GetDWARFDeclContext(const DWARFDIE &die);
 
   static lldb::LanguageType LanguageTypeFromDWARF(uint64_t val);
 
   static lldb::LanguageType GetLanguage(DWARFUnit &unit);
   /// Same as GetLanguage() but reports all C++ versions as C++ (no version).
   static lldb::LanguageType GetLanguageFamily(DWARFUnit &unit);
 
+  llvm::Optional<uint32_t> GetDWARFUnitIndex(uint32_t cu_idx);
+
 protected:
   typedef llvm::DenseMap<DIERef, lldb_private::Type *> DIERefToTypePtr;
   typedef llvm::DenseMap<DIERef, lldb::VariableSP> DIERefToVariableSP;
   typedef llvm::DenseMap<DIERef, lldb::opaque_compiler_type_t>
       DIERefToClangType;
   typedef llvm::DenseMap<lldb::opaque_compiler_type_t, DIERef> ClangTypeToDIE;
 
   SymbolFileDWARF(const SymbolFileDWARF &) = delete;
virtual DIERefToClangType &GetForwardDeclDIERefToClangType() {
     return m_forward_decl_dieref_to_clang_type;
   }
 
   virtual ClangTypeToDIE &GetForwardDeclClangTypeToDie() {
     return m_forward_decl_clang_type_to_die;
   }
 
   void BuildCuTranslationTable();
-  llvm::Optional<uint32_t> GetDWARFUnitIndex(uint32_t cu_idx);
 
   struct DecodedUID {
     SymbolFileDWARF &dwarf;
     DIERef ref;
   };
   llvm::Optional<DecodedUID> DecodeUID(lldb::user_id_t uid);
 
   void FindDwpSymbolFile();

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

if (type_system != nullptr)
     type_system->GetDWARFParser()->EnsureAllDIEsInDeclContextHaveBeenParsed(
         decl_ctx);
 }
 
 user_id_t SymbolFileDWARF::GetUID(DIERef ref) {
   if (GetDebugMapSymfile())
     return GetID() | ref.die_offset();
 
-  lldbassert(GetDwoNum().getValueOr(0) <= 0x3fffffff);
-  return user_id_t(GetDwoNum().getValueOr(0)) << 32 | ref.die_offset() |
-         lldb::user_id_t(GetDwoNum().hasValue()) << 62 |
-         lldb::user_id_t(ref.section() == DIERef::Section::DebugTypes) << 63;
+#ifndef NDEBUG
+  DWARFDIE dwarfdie_check = GetDIE(ref);
+  lldbassert(dwarfdie_check.IsValid());
+  lldbassert(*dwarfdie_check.GetDIERef() == ref);
+#endif
+
+  // WARNING: Use ref.dwo_num() as GetDwoNum() may not be valid in 'this'.
+  static_assert(sizeof(ref.die_offset()) * 8 == 32, "");
+  lldbassert(!ref.dwo_num().hasValue() || *ref.dwo_num() <= 0x1fffffff);
+  lldbassert(!ref.main_cu().hasValue() || *ref.main_cu() <= 0x1fffffff);
+  lldbassert(0 <= ref.kind_get());
+  lldbassert(ref.kind_get() <= 3);
+  user_id_t retval =
+      user_id_t(ref.dwo_num() ? *ref.dwo_num()
+                              : (ref.main_cu() ? *ref.main_cu() : 0))
+          << 32 |
+      ref.die_offset() | user_id_t(ref.kind_get()) << 61 |
+
+#ifndef NDEBUG
+  DWARFDIE dwarfdie_check2 = GetDIE(retval);
+  lldbassert(dwarfdie_check2 == dwarfdie_check);
+#endif
+
+  return retval;
 }
 
 llvm::Optional<SymbolFileDWARF::DecodedUID>
 SymbolFileDWARF::DecodeUID(lldb::user_id_t uid) {
   // This method can be called without going through the symbol vendor so we
   // need to lock the module.
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   // Anytime we get a "lldb::user_id_t" from an lldb_private::SymbolFile API we
   // must make sure we use the correct DWARF file when resolving things. On
   // MacOSX, when using SymbolFileDWARFDebugMap, we will use multiple
   // SymbolFileDWARF classes, one for each .o file. We can often end up with
   // references to other DWARF objects and we must be ready to receive a
   // "lldb::user_id_t" that specifies a DIE from another SymbolFileDWARF
   // instance.
   if (SymbolFileDWARFDebugMap *debug_map = GetDebugMapSymfile()) {
     SymbolFileDWARF *dwarf = debug_map->GetSymbolFileByOSOIndex(
         debug_map->GetOSOIndexFromUserID(uid));
-    return DecodedUID{
-        *dwarf, {llvm::None, DIERef::Section::DebugInfo, dw_offset_t(uid)}};
+    return DecodedUID{*dwarf,
+                      {llvm::None, llvm::None, DIERef::MainDwz,
+                       DIERef::Section::DebugInfo, dw_offset_t(uid)}};
   }
   dw_offset_t die_offset = uid;
   if (die_offset == DW_INVALID_OFFSET)
     return llvm::None;
 
   DIERef::Section section =
       uid >> 63 ? DIERef::Section::DebugTypes : DIERef::Section::DebugInfo;
 
+  DIERef::Kind kind = DIERef::Kind(uid >> 61 & 3);
+
   llvm::Optional<uint32_t> dwo_num;
-  bool dwo_valid = uid >> 62 & 1;
-  if (dwo_valid)
-    dwo_num = uid >> 32 & 0x3fffffff;
+  if (kind == DIERef::Kind::DwoKind)
+    dwo_num = uid >> 32 & 0x1fffffff;
 
-  return DecodedUID{*this, {dwo_num, section, die_offset}};
+  return DecodedUID{
+      *this, {dwo_num, llvm::None, DIERef::MainDwz, section, die_offset}};
 }
 
 DWARFDIE
 SymbolFileDWARF::GetDIE(lldb::user_id_t uid) {
   // This method can be called without going through the symbol vendor so we
   // need to lock the module.
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
 
if (pos != m_external_type_modules.end())
     return pos->second;
   else
     return lldb::ModuleSP();
 }
 
 DWARFDIE
 SymbolFileDWARF::GetDIE(const DIERef &die_ref) {
   if (die_ref.dwo_num()) {
-    SymbolFileDWARF *dwarf = *die_ref.dwo_num() == 0x3fffffff
+    SymbolFileDWARF *dwarf = *die_ref.dwo_num() == 0x1fffffff
                                  ? m_dwp_symfile.get()
                                  : this->DebugInfo()
                                        .GetUnitAtIndex(*die_ref.dwo_num())
                                        ->GetDwoSymbolFile();
     return dwarf->DebugInfo().GetDIE(die_ref);
   }
 
   return DebugInfo().GetDIE(die_ref);
SymbolFileDWARF::GetDwoSymbolFileForCompileUnit(
   lldb::offset_t dwo_file_data_offset = 0;
   ObjectFileSP dwo_obj_file = ObjectFile::FindPlugin(
       GetObjectFile()->GetModule(), &dwo_file, file_offset,
       FileSystem::Instance().GetByteSize(dwo_file), dwo_file_data_sp,
       dwo_file_data_offset);
   if (dwo_obj_file == nullptr)
     return nullptr;
 
+  lldbassert(dwarf_cu->GetID() < 0x1fffffff);
   return std::make_shared<SymbolFileDWARFDwo>(*this, dwo_obj_file,
                                               dwarf_cu->GetID());
 }
 
 void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() {
   if (m_fetched_external_modules)
     return;
   m_fetched_external_modules = true;
if (FileSystem::Instance().Exists(dwp_filespec)) {
       lldb::offset_t dwp_file_data_offset = 0;
       ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
           GetObjectFile()->GetModule(), &dwp_filespec, 0,
           FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp,
           dwp_file_data_offset);
       if (!dwp_obj_file)
         return;
       m_dwp_symfile =
-          std::make_shared<SymbolFileDWARFDwo>(*this, dwp_obj_file, 0x3fffffff);
+          std::make_shared<SymbolFileDWARFDwo>(*this, dwp_obj_file, 0x1fffffff);
     }
   });
   return m_dwp_symfile;
 }
 
 llvm::Expected<TypeSystem &> SymbolFileDWARF::GetTypeSystem(DWARFUnit &unit) {
   return unit.GetSymbolFileDWARF().GetTypeSystemForLanguage(GetLanguage(unit));
 }