[lldb] Support compressed CTF

[JDevlieghere]

Summary

Add support for compressed CTF data. The flags in the header can indicate whether the CTF body is compressed with zlib deflate. This patch supports uncompressing the data before parsing.

[jasonmolenda]

LGTM.

lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp

151

Should this be m_data.GetByteSize() - sizeof(ctf_header_t) ? m_data is a compressed ctf_header_t plus a compressed data payload?

183

Do you need to copy the ctf_header_t from m_data into the decompressed_data buffer, then decompressing the compressed data, to get header + data in m_data? Oh wait, I see, after you've copied the header into your ctf_header local, you don't need it any longer in m_data and m_body_offset either points to the end of the header (uncompressed CTF) or it points to the start of m_data which doesn't include the header. ok.

lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h

enum TypeKind : uint32_t {
     eTypedef = 10,
     eVolatile = 11,
     eConst = 12,
     eRestrict = 13,
     eSlice = 14,
   };
 
 private:
+  enum Flags : uint32_t {
+    eFlagCompress = (1u << 0),
+    eFlagNewFuncInfo = (1u << 1),
+    eFlagIdxSorted = (1u << 2),
+    eFlagDynStr = (1u << 3),
+  };
+
   enum IntEncoding : uint32_t {
     eSigned = 0x1,
     eChar = 0x2,
     eBool = 0x4,
     eVarArgs = 0x8,
   };
 
   struct ctf_preamble_t {
llvm::Expected<lldb::TypeSP> ParseRecord(lldb::offset_t &offset,
                                            uint32_t fields, uint32_t size);
 
   llvm::StringRef ReadString(lldb::offset_t offset) const;
 
   std::vector<uint16_t> GetFieldSizes(lldb::offset_t field_offset,
                                       uint32_t fields, uint32_t struct_size);
 
   DataExtractor m_data;
+
+  /// The start offset of the CTF body into m_data. If the body is uncompressed,
+  /// m_data contains the header and the body and the body starts after the
+  /// header. If the body is compressed, m_data only contains the body and the
+  /// offset is zero.
+  lldb::offset_t m_body_offset = 0;
+
   TypeSystemClang *m_ast;
   lldb::CompUnitSP m_comp_unit_sp;
 
   std::optional<ctf_header_t> m_header;
   std::vector<lldb::TypeSP> m_types;
   std::vector<lldb::FunctionSP> m_functions;
   std::vector<lldb::VariableSP> m_variables;
 
   static constexpr uint16_t g_ctf_magic = 0xcff1;
   static constexpr uint8_t g_ctf_version = 4;
 };
 } // namespace lldb_private
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_CTF_SYMBOLFILECTF_H

lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp

 //===-- SymbolFileCTF.cpp ----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "SymbolFileCTF.h"
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/StreamBuffer.h"
+#include "lldb/Host/Config.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/SymbolContext.h"
 #include "lldb/Symbol/Symtab.h"
 #include "lldb/Symbol/TypeList.h"
 #include "lldb/Symbol/TypeMap.h"
 #include "lldb/Symbol/Variable.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 #include "Plugins/ExpressionParser/Clang/ClangASTMetadata.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 
 #include <memory>
 #include <optional>
 
+#if LLVM_ENABLE_ZLIB
+#include <zlib.h>
+#endif
+
 using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
 
 LLDB_PLUGIN_DEFINE(SymbolFileCTF)
 
 char SymbolFileCTF::ID;
 
if (ctf_header.preamble.version != g_ctf_version) {
     LLDB_LOG(log, "CTF parsing failed: unsupported version: {0}",
              ctf_header.preamble.version);
     return false;
   }
 
   LLDB_LOG(log, "Parsed valid CTF preamble: version {0}, flags {1:x}",
            ctf_header.preamble.version, ctf_header.preamble.flags);
 
-  const lldb::offset_t header_offset = offset;
+  m_body_offset = offset;
+
+  if (ctf_header.preamble.flags & eFlagCompress) {
+    // The body has been compressed with zlib deflate. Header offsets point into
+    // the decompressed data.
+#if LLVM_ENABLE_ZLIB
+    const std::size_t decompressed_size = ctf_header.stroff + ctf_header.strlen;
+    DataBufferSP decompressed_data =
+        std::make_shared<DataBufferHeap>(decompressed_size, 0x0);
+
+    z_stream zstr;
+    memset(&zstr, 0, sizeof(zstr));
+    zstr.next_in = (Bytef *)const_cast<uint8_t *>(m_data.GetDataStart() +
+                                                  sizeof(ctf_header_t));
+    zstr.avail_in = m_data.BytesLeft(offset);
+    zstr.next_out =
+        (Bytef *)const_cast<uint8_t *>(decompressed_data->GetBytes());
+    zstr.avail_out = decompressed_size;
+
+    int rc = inflateInit(&zstr);
+    if (rc != Z_OK) {
+      LLDB_LOG(log, "CTF parsing failed: inflate initialization error: {0}",
+               zError(rc));
+      return false;
+    }
+
+    rc = inflate(&zstr, Z_FINISH);
+    if (rc != Z_STREAM_END) {
+      LLDB_LOG(log, "CTF parsing failed: inflate error: {0}", zError(rc));
+      return false;
+    }
+
+    rc = inflateEnd(&zstr);
+    if (rc != Z_OK) {
+      LLDB_LOG(log, "CTF parsing failed: inflate end error: {0}", zError(rc));
+      return false;
+    }
+
+    if (zstr.total_out != decompressed_size) {
+      LLDB_LOG(log,
+               "CTF parsing failed: decompressed size ({0}) doesn't match "
+               "expected size ([1})",
+               zstr.total_out, decompressed_size);
+      return false;
+    }
+
+    m_data = DataExtractor(decompressed_data, m_data.GetByteOrder(),
+                           m_data.GetAddressByteSize());
+    m_body_offset = 0;
+#else
+    LLDB_LOG(
+        log,
+        "CTF parsing failed: data is compressed but no zlib inflate support");
+    return false;
+#endif
+  }
 
   // Validate the header.
-  if (!m_data.ValidOffset(header_offset + ctf_header.lbloff)) {
+  if (!m_data.ValidOffset(m_body_offset + ctf_header.lbloff)) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid label section offset in header: {0}",
              ctf_header.lbloff);
     return false;
   }
 
-  if (!m_data.ValidOffset(header_offset + ctf_header.objtoff)) {
+  if (!m_data.ValidOffset(m_body_offset + ctf_header.objtoff)) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid object section offset in header: {0}",
              ctf_header.objtoff);
     return false;
   }
 
-  if (!m_data.ValidOffset(header_offset + ctf_header.funcoff)) {
+  if (!m_data.ValidOffset(m_body_offset + ctf_header.funcoff)) {
     LLDB_LOG(
         log,
         "CTF parsing failed: invalid function section offset in header: {0}",
         ctf_header.funcoff);
     return false;
   }
 
-  if (!m_data.ValidOffset(header_offset + ctf_header.typeoff)) {
+  if (!m_data.ValidOffset(m_body_offset + ctf_header.typeoff)) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid type section offset in header: {0}",
              ctf_header.typeoff);
     return false;
   }
 
-  if (!m_data.ValidOffset(header_offset + ctf_header.stroff)) {
+  if (!m_data.ValidOffset(m_body_offset + ctf_header.stroff)) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid string section offset in header: {0}",
              ctf_header.stroff);
     return false;
   }
 
   const lldb::offset_t str_end_offset =
-      header_offset + ctf_header.stroff + ctf_header.strlen;
+      m_body_offset + ctf_header.stroff + ctf_header.strlen;
   if (!m_data.ValidOffset(str_end_offset - 1)) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid string section length in header: {0}",
              ctf_header.strlen);
     return false;
   }
 
-  if (header_offset + ctf_header.stroff + ctf_header.parlabel >
+  if (m_body_offset + ctf_header.stroff + ctf_header.parlabel >
       str_end_offset) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid parent label offset: {0} exceeds end "
              "of string section ({1})",
              ctf_header.parlabel, str_end_offset);
     return false;
   }
 
-  if (header_offset + ctf_header.stroff + ctf_header.parname > str_end_offset) {
+  if (m_body_offset + ctf_header.stroff + ctf_header.parname > str_end_offset) {
     LLDB_LOG(log,
              "CTF parsing failed: invalid parent name offset: {0} exceeds end "
              "of string section ({1})",
              ctf_header.parname, str_end_offset);
     return false;
   }
 
   LLDB_LOG(log,
void SymbolFileCTF::InitializeObject() {
   LazyBool optimized = eLazyBoolNo;
   m_comp_unit_sp = std::make_shared<CompileUnit>(
       m_objfile_sp->GetModule(), nullptr, "", 0, eLanguageTypeC, optimized);
 
   ParseTypes(*m_comp_unit_sp);
 }
 
 llvm::StringRef SymbolFileCTF::ReadString(lldb::offset_t str_offset) const {
-  lldb::offset_t offset = sizeof(ctf_header_t) + m_header->stroff + str_offset;
+  lldb::offset_t offset = m_body_offset + m_header->stroff + str_offset;
   if (!m_data.ValidOffset(offset))
     return "(invalid)";
   const char *str = m_data.GetCStr(&offset);
   if (str && !*str)
     return "(anon)";
   return llvm::StringRef(str);
 }
 
if (!m_types.empty())
     return 0;
 
   if (!m_ast)
     return 0;
 
   Log *log = GetLog(LLDBLog::Symbols);
   LLDB_LOG(log, "Parsing CTF types");
 
-  lldb::offset_t type_offset = sizeof(ctf_header_t) + m_header->typeoff;
-  const lldb::offset_t type_offset_end =
-      sizeof(ctf_header_t) + m_header->stroff;
+  lldb::offset_t type_offset = m_body_offset + m_header->typeoff;
+  const lldb::offset_t type_offset_end = m_body_offset + m_header->stroff;
 
   lldb::user_id_t type_uid = 1;
   while (type_offset < type_offset_end) {
     ctf_stype_t ctf_stype;
     ctf_stype.name = m_data.GetU32(&type_offset);
     ctf_stype.info = m_data.GetU32(&type_offset);
     ctf_stype.size = m_data.GetU32(&type_offset);
 
size_t SymbolFileCTF::ParseFunctions(CompileUnit &cu) {
 
   Symtab *symtab = GetObjectFile()->GetModule()->GetSymtab();
   if (!symtab)
     return 0;
 
   Log *log = GetLog(LLDBLog::Symbols);
   LLDB_LOG(log, "Parsing CTF functions");
 
-  lldb::offset_t function_offset = sizeof(ctf_header_t) + m_header->funcoff;
-  const lldb::offset_t function_offset_end =
-      sizeof(ctf_header_t) + m_header->typeoff;
+  lldb::offset_t function_offset = m_body_offset + m_header->funcoff;
+  const lldb::offset_t function_offset_end = m_body_offset + m_header->typeoff;
 
   uint32_t symbol_idx = 0;
   Declaration decl;
   while (function_offset < function_offset_end) {
     const uint32_t info = m_data.GetU32(&function_offset);
     const uint16_t kind = GetKind(info);
     const uint16_t variable_length = GetVLen(info);
 
size_t SymbolFileCTF::ParseObjects(CompileUnit &comp_unit) {
   ModuleSP module_sp = GetObjectFile()->GetModule();
   Symtab *symtab = module_sp->GetSymtab();
   if (!symtab)
     return 0;
 
   Log *log = GetLog(LLDBLog::Symbols);
   LLDB_LOG(log, "Parsing CTF objects");
 
-  lldb::offset_t object_offset = sizeof(ctf_header_t) + m_header->objtoff;
-  const lldb::offset_t object_offset_end =
-      sizeof(ctf_header_t) + m_header->funcoff;
+  lldb::offset_t object_offset = m_body_offset + m_header->objtoff;
+  const lldb::offset_t object_offset_end = m_body_offset + m_header->funcoff;
 
   uint32_t symbol_idx = 0;
   Declaration decl;
   while (object_offset < object_offset_end) {
     const uint32_t type_uid = m_data.GetU32(&object_offset);
 
     if (Symbol *symbol =
             symtab->FindSymbolWithType(eSymbolTypeData, Symtab::eDebugYes,

lldb/test/API/macosx/ctf/Makefile

 C_SOURCES := test.c
 MAKE_DSYM := YES
 
+ifeq "$(COMPRESS_CTF)" "YES"
+  COMPRESS := -c
+else
+   COMPRESS :=
+endif
+
 all: a.out a.ctf
 
 include Makefile.rules
 
 a.ctf: a.out.dSYM
-        ctfconvert -l a -o a.ctf a.out.dSYM/Contents/Resources/DWARF/a.out
-        $(OBJCOPY) \
-          -R __DWARF,__debug_line \
-          -R __DWARF,__debug_aranges \
-          -R __DWARF,__debug_info \
-          -R __DWARF,__debug_abbrev \
-          -R __DWARF,__debug_str \
-          -R __DWARF,__apple_names \
-          -R __DWARF,__apple_namespac \
-          -R __DWARF,__apple_types \
-          -R __DWARF,__apple_objc \
-          a.ctf a.ctf
-        rm -rf a.out.dSYM
+  ctfconvert $(COMPRESS) \
+    -l a \
+    -o a.ctf \
+    a.out.dSYM/Contents/Resources/DWARF/a.out
+    -R __DWARF,__debug_line \
+    -R __DWARF,__debug_aranges \
+    -R __DWARF,__debug_info \
+    -R __DWARF,__debug_abbrev \
+    -R __DWARF,__debug_str \
+    -R __DWARF,__apple_names \
+    -R __DWARF,__apple_namespac \
+    -R __DWARF,__apple_types \
+    -R __DWARF,__apple_objc \
+    a.ctf a.ctf

lldb/test/API/macosx/ctf/TestCTF.py

def no_ctf_convert(self):
             return "ctfconvert not found in path"
         return None
 
     def no_objcopy(self):
         if not "OBJCOPY" in os.environ:
             return "llvm-objcopy not found in environment"
         return None
 
-    @skipTestIfFn(no_ctf_convert)
-    @skipTestIfFn(no_objcopy)
-    @skipUnlessDarwin
     def test(self):
         self.build()
+        self.do_test()
+
+    def test_compressed(self):
+        self.build(dictionary={"COMPRESS_CTF": "YES"})
+        self.do_test()
 
+    @skipTestIfFn(no_ctf_convert)
+    @skipTestIfFn(no_objcopy)
+    @skipUnlessDarwin
+    def do_test(self):
         lldbutil.run_to_name_breakpoint(self, "printf")
 
         symbol_file = self.getBuildArtifact("a.ctf")
 
         self.runCmd("target symbols add {}".format(symbol_file))
         self.expect(
             "target variable foo",
             substrs=[