DWZ 08/12: DWARFCompileUnit::ExtractDIEsIfNeeded can now expand AllDiesButCuDieOnlyForPartialUnits

[jankratochvil]

Summary

It does not make sense to extract whole DW_TAG_partial_unit only to find out it is a partial unit and so it cannot be indexed on its own.

All DWZ patches are also applied in: git clone -b dwz git://git.jankratochvil.net/lldb

[clayborg]

See inline comments.

source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

2050–2052

All of these changes can be avoided by just appropriately checking for partial units and skipping them in this index function. Do you plan on using this functionality in any other place? I would rather you just fix this function to filter out the partial units.

source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h

 };
 
 class DWARFCompileUnit : public DWARFCompileUnitDecls {
 public:
   static DWARFCompileUnitSP Extract(SymbolFileDWARF *dwarf2Data,
       lldb::offset_t *offset_ptr);
   ~DWARFCompileUnit();
 
-  size_t ExtractDIEsIfNeeded(bool cu_die_only);
+  enum class ExtractDIEsIfNeededKind {
+    CuDieOnly,
+    AllDies,
+    AllDiesButCuDieOnlyForPartialUnits,
+  };
+  size_t ExtractDIEsIfNeeded(ExtractDIEsIfNeededKind kind);
+
   DWARFDIE LookupAddress(const dw_addr_t address);
   size_t AppendDIEsWithTag(const dw_tag_t tag,
                            DWARFDIECollection &matching_dies,
                            uint32_t depth = UINT32_MAX) const;
   bool Verify(lldb_private::Stream *s) const;
   void Dump(lldb_private::Stream *s) const;
   // Offset of the initial length field.
   dw_offset_t GetOffset() const { return m_offset; }
protected:
 
   // Offset of the initial length field.
   dw_offset_t m_offset;
 
 private:
   DWARFCompileUnit(SymbolFileDWARF *dwarf2Data);
 
   const DWARFDebugInfoEntry *GetCompileUnitDIEPtrOnly() {
-    ExtractDIEsIfNeeded(true);
+    ExtractDIEsIfNeeded(ExtractDIEsIfNeededKind::CuDieOnly);
     if (m_data->m_die_array.empty())
       return NULL;
     return &m_data->m_die_array[0];
   }
 
   const DWARFDebugInfoEntry *DIEPtr() {
-    ExtractDIEsIfNeeded(false);
+    ExtractDIEsIfNeeded(ExtractDIEsIfNeededKind::AllDies);
     if (m_data->m_die_array.empty())
       return NULL;
     return &m_data->m_die_array[0];
   }
 
   DISALLOW_COPY_AND_ASSIGN(DWARFCompileUnit);
 };
 
 #endif // SymbolFileDWARF_DWARFCompileUnit_h_

source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp

 }
 
 //----------------------------------------------------------------------
 // ParseCompileUnitDIEsIfNeeded
 //
 // Parses a compile unit and indexes its DIEs if it hasn't already been
 // done.
 //----------------------------------------------------------------------
-size_t DWARFCompileUnit::ExtractDIEsIfNeeded(bool cu_die_only) {
+size_t DWARFCompileUnit::ExtractDIEsIfNeeded(ExtractDIEsIfNeededKind kind) {
   size_t initial_die_array_size;
-  auto already_parsed = [cu_die_only, &initial_die_array_size]() -> bool {
-    return (cu_die_only && initial_die_array_size > 0)
-        || initial_die_array_size > 1;
+  auto already_parsed =
+      [kind, &initial_die_array_size, this]() -> bool {
+    initial_die_array_size = m_data->m_die_array.size();    
+    if (initial_die_array_size > 1)
+      return true;
+    switch (kind) {
+    case ExtractDIEsIfNeededKind::CuDieOnly:
+      if (initial_die_array_size > 0)
+        return true;
+      break;
+    case ExtractDIEsIfNeededKind::AllDies:
+      break;
+    case ExtractDIEsIfNeededKind::AllDiesButCuDieOnlyForPartialUnits: {
+      if (initial_die_array_size > 0
+          && GetCompileUnitDIEOnly().TagOrig() == DW_TAG_partial_unit)
+        return true;
+      } break;
+    }
+    return false;
   };
   if (already_parsed() && m_data->m_die_array_finished)
     return 0;
   std::lock_guard<std::mutex> guard(m_data->m_extractdies_mutex);
   if (already_parsed()) {
     lldbassert(m_data->m_die_array_finished);
     return 0;
   }
   m_data->m_die_array_finished = false;
   std::shared_ptr<void> m_die_array_finished_set(nullptr,
       [&](void*){ m_data->m_die_array_finished = true; });
 
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(
       func_cat,
-      "%8.8x: DWARFCompileUnit::ExtractDIEsIfNeeded( cu_die_only = %i )",
-      m_offset, cu_die_only);
+      "%8.8x: DWARFCompileUnit::ExtractDIEsIfNeeded( kind = %i )",
+      m_offset, int(kind));
 
   // Set the offset to that of the first DIE and calculate the start of the
   // next compilation unit header.
   lldb::offset_t file_offset = ThisCUUniqToFileOffset(GetFirstDIEOffset());
   lldb::offset_t next_cu_offset = GetNextCompileUnitOffset();
 
   DWARFDebugInfoEntry die;
   // Keep a flat array of the DIE for binary lookup by DIE offset
-  if (!cu_die_only) {
+  if (kind != ExtractDIEsIfNeededKind::CuDieOnly) {
     Log *log(
         LogChannelDWARF::GetLogIfAny(DWARF_LOG_DEBUG_INFO | DWARF_LOG_LOOKUPS));
     if (log) {
       m_data->m_dwarf2Data->GetObjectFile()->GetModule()
           ->LogMessageVerboseBacktrace(
               log, "DWARFCompileUnit::ExtractDIEsIfNeeded () "
                    "for compile unit at .debug_info[0x%8.8x]",
               GetOffset());
if (depth == 0) {
       if (initial_die_array_size == 0)
         AddCompileUnitDIE(die);
       uint64_t base_addr = die.GetAttributeValueAsAddress(
           m_data->m_dwarf2Data, this, DW_AT_low_pc, LLDB_INVALID_ADDRESS);
       if (base_addr == LLDB_INVALID_ADDRESS)
         base_addr = die.GetAttributeValueAsAddress(m_data->m_dwarf2Data, this,
                                                    DW_AT_entry_pc, 0);
       SetBaseAddress(base_addr);
-      if (cu_die_only)
+      if (kind == ExtractDIEsIfNeededKind::CuDieOnly)
+        return 1;
+      if (kind == ExtractDIEsIfNeededKind::AllDiesButCuDieOnlyForPartialUnits
+          && die.TagOrig() == DW_TAG_partial_unit)
         return 1;
     } else {
       if (null_die) {
         if (prev_die_had_children) {
           // This will only happen if a DIE says is has children
           // but all it contains is a NULL tag. Since we are removing
           // the NULL DIEs from the list (saves up to 25% in C++ code),
           // we need a way to let the DIE know that it actually doesn't
else
       m_data->m_die_array[0].Dump(m_data->m_dwarf2Data, this, strm, UINT32_MAX);
     log->PutString(strm.GetString());
   }
 
   if (!m_data->m_dwo_symbol_file)
     return m_data->m_die_array.size();
 
   DWARFCompileUnit *dwo_cu = m_data->m_dwo_symbol_file->GetCompileUnit();
-  size_t dwo_die_count = dwo_cu->ExtractDIEsIfNeeded(cu_die_only);
+  size_t dwo_die_count = dwo_cu->ExtractDIEsIfNeeded(kind);
   return m_data->m_die_array.size() + dwo_die_count -
          1; // We have 2 CU die, but we want to count it only as one
 }
 
 void DWARFCompileUnit::AddCompileUnitDIE(DWARFDebugInfoEntry &die) {
   assert(m_data->m_die_array.empty() && "Compile unit DIE already added");
   AddDIE(die);
 
if (die) {
     }
   }
   // We don't have a DW_AT_ranges attribute, so we need to parse the DWARF
 
   // If the DIEs weren't parsed, then we don't want all dies for all compile
   // units
   // to stay loaded when they weren't needed. So we can end up parsing the DWARF
   // and then throwing them all away to keep memory usage down.
-  const bool clear_dies = ExtractDIEsIfNeeded(false) > 1;
+  const bool clear_dies = ExtractDIEsIfNeeded(
+      ExtractDIEsIfNeededKind::AllDiesButCuDieOnlyForPartialUnits) > 1;
 
-  die = DIEPtr();
+  die = GetCompileUnitDIEPtrOnly();
   if (die)
     die->BuildAddressRangeTable(dwarf2Data, this, debug_aranges);
 
   if (debug_aranges->GetNumRanges() == num_debug_aranges) {
     // We got nothing from the functions, maybe we have a line tables only
     // situation. Check the line tables and build the arange table from this.
     SymbolContext sc;
     sc.comp_unit = dwarf2Data->GetCompUnitForDWARFCompUnit(this);

 //----------------------------------------------------------------------
 DWARFDIE
 DWARFCompileUnit::GetDIE(dw_offset_t die_offset) {
   if (die_offset != DW_INVALID_OFFSET) {
     if (m_data->m_dwo_symbol_file)
       return m_data->m_dwo_symbol_file->GetCompileUnit()->GetDIE(die_offset);
 
     if (ContainsDIEOffset(die_offset)) {
-      ExtractDIEsIfNeeded(false);
+      ExtractDIEsIfNeeded(ExtractDIEsIfNeededKind::AllDies);
       dw_offset_t die_file_offset = ThisCUUniqToFileOffset(die_offset);
       DWARFDebugInfoEntry::iterator end = m_data->m_die_array.end();
       DWARFDebugInfoEntry::iterator pos = lower_bound(
           m_data->m_die_array.begin(), end, die_file_offset,
           CompareDIEFileOffset);
       if (pos != end) {
         if (die_file_offset == (*pos).GetFileOffset())
           return DWARFDIE(this, &(*pos));

source/Plugins/SymbolFile/DWARF/DWARFDebugPubnames.cpp

if (debug_info) {
     for (cu_idx = 0; cu_idx < num_compile_units; ++cu_idx) {
 
       DWARFCompileUnit *cu = debug_info->GetCompileUnitAtIndex(cu_idx);
 
       DWARFFormValue::FixedFormSizes fixed_form_sizes =
           DWARFFormValue::GetFixedFormSizesForAddressSize(
               cu->GetAddressByteSize(), cu->IsDWARF64());
 
-      bool clear_dies = cu->ExtractDIEsIfNeeded(false) > 1;
+      bool clear_dies = cu->ExtractDIEsIfNeeded(
+          DWARFCompileUnit::ExtractDIEsIfNeededKind::AllDies) > 1;
+      // In DW_TAG_partial_unit by DWZ cannot be DW_TAG_subprogram definition
+      // (see DWZ checksum_die).  But there can be DW_TAG_variable.
 
       DWARFDIECollection dies;
       const size_t die_count = cu->AppendDIEsWithTag(DW_TAG_subprogram, dies) +
                                cu->AppendDIEsWithTag(DW_TAG_variable, dies);
 
       dw_offset_t cu_offset = cu->GetOffset();
       DWARFDebugPubnamesSet pubnames_set(DW_INVALID_OFFSET, cu_offset,
                                          cu->GetNextCompileUnitOffset() -

source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

auto parser_fn = [debug_info, &function_basename_index,
             objc_class_selectors_index[cu_idx], global_index[cu_idx],
             type_index[cu_idx], namespace_index[cu_idx]);
       }
     };
 
     auto extract_fn = [debug_info, &clear_cu_dies](size_t cu_idx) {
       DWARFCompileUnit *dwarf_cu = debug_info->GetCompileUnitAtIndex(cu_idx);
       if (dwarf_cu) {
-        // dwarf_cu->ExtractDIEsIfNeeded(false) will return zero if the
+        // dwarf_cu->ExtractDIEsIfNeeded will return zero if the
         // DIEs for a compile unit have already been parsed.
-        if (dwarf_cu->ExtractDIEsIfNeeded(false) > 1)
+        if (dwarf_cu->ExtractDIEsIfNeeded(DWARFCompileUnit::
+            ExtractDIEsIfNeededKind::AllDiesButCuDieOnlyForPartialUnits) > 1)
           clear_cu_dies[cu_idx] = true;
       }
     };
 
     // Create a task runner that extracts dies for each DWARF compile unit in a
     // separate thread
     //----------------------------------------------------------------------
     // First figure out which compile units didn't have their DIEs already
     // parsed and remember this.  If no DIEs were parsed prior to this index