Add a clang-ast subcommand to lldb-test

[zturner]

Summary

This is the bare minimum needed to dump ClangASTContexts via lldb-test.

Within the first 10 seconds of using this, I already found a bug. A FIXME note and repro is included in the comments in this patch.

With this, it should be possible to do deep testing of otherwise difficult to test scenarios involving ClangASTContext.

[zturner]

If I remove that assert, I get this output:

D:\src\llvmbuild\lldb>bin\lldb-test.exe clang-ast foo.o
error: foo.o {0x0000003b}: unhandled type tag 0x0005 (DW_TAG_formal_parameter), please file a bug and attach the file at the start of this error message
error: foo.o {0x00000049}: unhandled type tag 0x0005 (DW_TAG_formal_parameter), please file a bug and attach the file at the start of this error message
Module foo.o
0BA227E8:   ObjectFileELF, file = 'foo.o', arch = i386
ELF Header
e_ident[EI_MAG0   ] = 0x7f
e_ident[EI_MAG1   ] = 0x45 'E'
e_ident[EI_MAG2   ] = 0x4c 'L'
e_ident[EI_MAG3   ] = 0x46 'F'
e_ident[EI_CLASS  ] = 0x01
e_ident[EI_DATA   ] = 0x01 ELFDATA2LSB - Little Endian
e_ident[EI_VERSION] = 0x01
e_ident[EI_PAD    ] = 0x00
e_type      = 0x0001 ET_REL
e_machine   = 0x0003
e_version   = 0x00000001
e_entry     = 0x00000000
e_phoff     = 0x00000000
e_shoff     = 0x0000047c
e_flags     = 0x00000000
e_ehsize    = 0x0034
e_phentsize = 0x0000
e_phnum     = 0x00000000
e_shentsize = 0x0028
e_shnum     = 0x00000014
e_shstrndx  = 0x00000001


Section Headers
IDX  name     type         flags                            addr     offset   size     link     info     addralgn entsize  Name
==== -------- ------------ -------------------------------- -------- -------- -------- -------- -------- -------- -------- ====================
[ 0] 00000000 SHT_NULL     00000000 (                     ) 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 1] 000000bc SHT_STRTAB   00000000 (                     ) 00000000 000003b0 000000cc 00000000 00000000 00000001 00000000 .strtab
[ 2] 0000000f SHT_PROGBITS 00000006 (      ALLOC+EXECINSTR) 00000000 00000040 00000022 00000000 00000000 00000010 00000000 .text
[ 3] 00000054 SHT_PROGBITS 00000030 (                     ) 00000000 00000062 0000004b 00000000 00000000 00000001 00000001 .debug_str
[ 4] 00000001 SHT_PROGBITS 00000000 (                     ) 00000000 000000ad 0000004d 00000000 00000000 00000001 00000000 .debug_abbrev
[ 5] 0000007a SHT_PROGBITS 00000000 (                     ) 00000000 000000fa 00000071 00000000 00000000 00000001 00000000 .debug_info
[ 6] 00000076 SHT_REL      00000000 (                     ) 00000000 00000328 00000060 00000013 00000005 00000004 00000008 .rel.debug_info
[ 7] 00000046 SHT_PROGBITS 00000000 (                     ) 00000000 0000016b 00000000 00000000 00000000 00000001 00000000 .debug_ranges
[ 8] 00000067 SHT_PROGBITS 00000000 (                     ) 00000000 0000016b 00000001 00000000 00000000 00000001 00000000 .debug_macinfo
[ 9] 00000036 SHT_PROGBITS 00000000 (                     ) 00000000 0000016c 0000001b 00000000 00000000 00000001 00000000 .debug_pubnames
[10] 00000032 SHT_REL      00000000 (                     ) 00000000 00000388 00000008 00000013 00000009 00000004 00000008 .rel.debug_pubnames
[11] 00000022 SHT_PROGBITS 00000000 (                     ) 00000000 00000187 00000023 00000000 00000000 00000001 00000000 .debug_pubtypes
[12] 0000001e SHT_REL      00000000 (                     ) 00000000 00000390 00000008 00000013 0000000b 00000004 00000008 .rel.debug_pubtypes
[13] 00000015 SHT_PROGBITS 00000030 (                     ) 00000000 000001aa 00000016 00000000 00000000 00000001 00000001 .comment
[14] 0000008b SHT_PROGBITS 00000000 (                     ) 00000000 000001c0 00000000 00000000 00000000 00000001 00000000 .note.GNU-stack
[15] 000000af SHT_PROGBITS 00000000 (                     ) 00000000 000001c0 0000002c 00000000 00000000 00000004 00000000 .debug_frame
[16] 000000ab SHT_REL      00000000 (                     ) 00000000 00000398 00000010 00000013 0000000f 00000004 00000008 .rel.debug_frame
[17] 0000009f SHT_PROGBITS 00000000 (                     ) 00000000 000001ec 0000003a 00000000 00000000 00000001 00000000 .debug_line
[18] 0000009b SHT_REL      00000000 (                     ) 00000000 000003a8 00000008 00000013 00000011 00000004 00000008 .rel.debug_line
[19] 000000c4 SHT_SYMTAB   00000000 (                     ) 00000000 00000228 00000100 00000001 0000000f 00000004 00000010 .symtab

  SectID     Type             File Address                             Perm File Off.  File Size  Flags      Section Name
  ---------- ---------------- ---------------------------------------  ---- ---------- ---------- ---------- ----------------------------
  0x00000001 regular                                                   ---  0x00000000 0x00000000 0x00000000 foo.o.
  0x00000002 regular                                                   ---  0x000003b0 0x000000cc 0x00000000 foo.o..strtab
  0x00000003 code             [0x0000000000000000-0x0000000000000022)  r-x  0x00000040 0x00000022 0x00000006 foo.o..text
  0x00000004 dwarf-str                                                 ---  0x00000062 0x0000004b 0x00000030 foo.o..debug_str
  0x00000005 dwarf-abbrev                                              ---  0x000000ad 0x0000004d 0x00000000 foo.o..debug_abbrev
  0x00000006 dwarf-info                                                ---  0x000000fa 0x00000071 0x00000000 foo.o..debug_info
  0x00000007 elf-relocation-entries                                          ---  0x00000328 0x00000060 0x00000000 foo.o..rel.debug_info
  0x00000008 dwarf-ranges                                              ---  0x0000016b 0x00000000 0x00000000 foo.o..debug_ranges
  0x00000009 dwarf-macinfo                                             ---  0x0000016b 0x00000001 0x00000000 foo.o..debug_macinfo
  0x0000000a dwarf-pubnames                                            ---  0x0000016c 0x0000001b 0x00000000 foo.o..debug_pubnames
  0x0000000b elf-relocation-entries                                          ---  0x00000388 0x00000008 0x00000000 foo.o..rel.debug_pubnames
  0x0000000c dwarf-pubtypes                                            ---  0x00000187 0x00000023 0x00000000 foo.o..debug_pubtypes
  0x0000000d elf-relocation-entries                                          ---  0x00000390 0x00000008 0x00000000 foo.o..rel.debug_pubtypes
  0x0000000e regular                                                   ---  0x000001aa 0x00000016 0x00000030 foo.o..comment
  0x0000000f regular                                                   ---  0x000001c0 0x00000000 0x00000000 foo.o..note.GNU-stack
  0x00000010 dwarf-frame                                               ---  0x000001c0 0x0000002c 0x00000000 foo.o..debug_frame
  0x00000011 elf-relocation-entries                                          ---  0x00000398 0x00000010 0x00000000 foo.o..rel.debug_frame
  0x00000012 dwarf-line                                                ---  0x000001ec 0x0000003a 0x00000000 foo.o..debug_line
  0x00000013 elf-relocation-entries                                          ---  0x000003a8 0x00000008 0x00000000 foo.o..rel.debug_line
  0x00000014 elf-symbol-table                                          ---  0x00000228 0x00000100 0x00000000 foo.o..symtab
Symtab, file = foo.o, num_symbols = 7:
               Debug symbol
               |Synthetic symbol
               ||Externally Visible
               |||
Index   UserID DSX Type            File Address/Value Load Address       Size               Flags      Name
------- ------ --- --------------- ------------------ ------------------ ------------------ ---------- ----------------------------------
[    0]      1     SourceFile      0x0000000000000000                    0x0000000000000000 0x00000004 foo.cpp
[    1]     10     Invalid         0x0000000000000000                    0x0000000000000022 0x00000003
[    2]     11     Invalid         0x0000000000000000                    0x0000000000000022 0x00000003
[    3]     12     Invalid         0x0000000000000000                    0x0000000000000022 0x00000003
[    4]     13     Invalid         0x0000000000000000                    0x0000000000000022 0x00000003
[    5]     14     Invalid         0x0000000000000000                    0x0000000000000022 0x00000003
[    6]     15   X Code            0x0000000000000000                    0x0000000000000022 0x00000012 main


0B9964E0: SymbolVendor (foo.o)
0BA4AE48:   Type{0x00000058} , name = "clang version 6.0.0 ", size = 4, compiler_type = 0x0ba34a90 int
0BA4AE48:   Type{0x00000058} , name = "clang version 6.0.0 ", size = 4, compiler_type = 0x0ba34a90 int
0BA597D8:   Type{0x0000005f} , compiler_type = 0x0ba34d70 signed char **
0BA597D8:   Type{0x0000005f} , compiler_type = 0x0ba34d70 signed char **
0BA59860:   Type{0x00000064} , compiler_type = 0x0ba34d50 signed char *
0BA59860:   Type{0x00000064} , compiler_type = 0x0ba34d50 signed char *
0BA4EEE8:   Type{0x00000069} , name = "clang version 6.0.0 ", size = 1, compiler_type = 0x0ba34a70 signed char
0BA4EEE8:   Type{0x00000069} , name = "clang version 6.0.0 ", size = 1, compiler_type = 0x0ba34a70 signed char
0BA4EF70:   Type{0x00000026} , name = "clang version 6.0.0 ", decl = foo.cpp:1, compiler_type = 0x0ba34dc0 int (int, signed char **)
0BA4EF70:   Type{0x00000026} , name = "clang version 6.0.0 ", decl = foo.cpp:1, compiler_type = 0x0ba34dc0 int (int, signed char **)
0BA4AE48:   Type{0x00000058} , name = "clang version 6.0.0 ", size = 4, compiler_type = 0x0ba34a90 int
0BA597D8:   Type{0x0000005f} , compiler_type = 0x0ba34d70 signed char **
0BA59860:   Type{0x00000064} , compiler_type = 0x0ba34d50 signed char *
0BA4EEE8:   Type{0x00000069} , name = "clang version 6.0.0 ", size = 1, compiler_type = 0x0ba34a70 signed char
0BA1FCA8:   CompileUnit{0x00000000}, language = "c++", file = 'clang version 6.0.0 \clang version 6.0.0 '

It should be pretty easy to write FileCheck tests against this sort of output, and indicidentally this already exposes an additional bug aside from the one mentioned in the original message (see the first two lines of output).

[davide]

I really like this approach. I think it's going to expose a large amount of bugs, and probably facilitate the transition in case we want to move to the LLVM readers for this.
@aprantl / @vsk , what do you think?

[davide]

lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp

2764–2765

It's unclear to me why PC-rel and 32-bit abs rel are not handled, but the error is really misleading.
Even if we don't expect them, we should just throw an error gracefully.

[vsk]

The general approach sgtm, and the patch itself looks reasonable.

[jingham]

I'm sure this is just a "quick and dirty implementation" thing, but depending on the output of Dump functions doesn't seem like a great idea for long term stable testing.

Those functions are meant to be useful for debugging lldb, and gathering data when you can't get repro cases. They probably don't want to have all the detail you might need, and are laid out for readability, so they are no more than ad hoc structured.

For instance, your dumpClangASTContext finds it's way to CompileUnit::Dump which doesn't actually call m_types->Dump. That's probably because it was so noisy as to render the CU dump useless, though it was commented out before the sources were imported to llvm.org so I don't know for sure.

Because of that, this command is not currently useful for the purpose for which it was intended. I think at present it doesn't actually dump anything that actually gets put into the AST, it just dumps Symbol file information which is not relevant, CU structures, Function blocks.

[zturner]

In D40745#942913, @jingham wrote:

I'm sure this is just a "quick and dirty implementation" thing, but depending on the output of Dump functions doesn't seem like a great idea for long term stable testing.

Those functions are meant to be useful for debugging lldb, and gathering data when you can't get repro cases. They probably don't want to have all the detail you might need, and are laid out for readability, so they are no more than ad hoc structured.

For instance, your dumpClangASTContext finds it's way to CompileUnit::Dump which doesn't actually call m_types->Dump. That's probably because it was so noisy as to render the CU dump useless, though it was commented out before the sources were imported to llvm.org so I don't know for sure.

Because of that, this command is not currently useful for the purpose for which it was intended. I think at present it doesn't actually dump anything that actually gets put into the AST, it just dumps Symbol file information which is not relevant, CU structures, Function blocks.

I definitely agree we would want to re-write the dumping logic to be in the tool itself, so that it can be designed with the purpose in mind. I could probably rename this command to be 'symbols' (even then, we'd probably want to re-write the logic for it), but at least this gets the skeleton in place.

[jingham]

Cool. We do need to make sure people don't start writing tests against it yet, however. That would be wasted effort.

[clayborg]

One really nice way we can get a lot of testing of the DWARF to clang::ASTContext conversion is to:
1 - compile a source file with clang and dumps the AST for a specific type as the compiler knows it
2 - using the .o file with debug info from step 1, load it into LLDB and have the DWARF to clang::ASTContext conversion happen and dump the AST info for the type
3 - compare the two and look for differences

lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp

unsigned ObjectFileELF::ApplyRelocations(
   for (unsigned i = 0; i < num_relocations; ++i) {
     if (rel.Parse(rel_data, &offset) == false)
       break;
 
     Symbol *symbol = NULL;
 
     if (hdr->Is32Bit()) {
       switch (reloc_type(rel)) {
       case R_386_32:
       case R_386_PC32:
       default:
+        // FIXME: This asserts with this input:
+        //
+        // foo.cpp
+        // int main(int argc, char **argv) { return 0; }
+        //
+        // clang++.exe --target=i686-unknown-linux-gnu -g -c foo.cpp -o foo.o
+        //
+        // and running this on the foo.o module.
         assert(false && "unexpected relocation type");
       }
     } else {
       switch (reloc_type(rel)) {
       case R_X86_64_64: {
         symbol = symtab->FindSymbolByID(reloc_symbol(rel));
         if (symbol) {
           addr_t value = symbol->GetAddressRef().GetFileAddress();

lldb/tools/lldb-test/lldb-test.cpp

 //===- lldb-test.cpp ------------------------------------------ *- C++ --*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "FormatUtil.h"
 #include "SystemInitializerTest.h"
 
+#include "Plugins/SymbolFile/DWARF/SymbolFileDWARF.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Initialization/SystemLifetimeManager.h"
+#include "lldb/Symbol/ClangASTContext.h"
+#include "lldb/Symbol/ClangASTImporter.h"
 #include "lldb/Utility/DataExtractor.h"
+#include "lldb/Utility/StreamString.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
 #include <thread>
 
 using namespace lldb;
 using namespace lldb_private;
 using namespace llvm;
 
 namespace opts {
 cl::SubCommand ModuleSubcommand("module-sections",
                                 "Display LLDB Module Information");
+cl::SubCommand ClangASTSubcommand("clang-ast", "Dump Clang AST for input file");
 
 namespace module {
 cl::opt<bool> SectionContents("contents",
                               cl::desc("Dump each section's contents"),
                               cl::sub(ModuleSubcommand));
 cl::list<std::string> InputFilenames(cl::Positional, cl::desc("<input files>"),
                                      cl::OneOrMore, cl::sub(ModuleSubcommand));
 } // namespace module
+
+namespace clang_ast {
+cl::list<std::string> InputFilenames(cl::Positional, cl::desc("<input files>"),
+                                     cl::OneOrMore,
+                                     cl::sub(ClangASTSubcommand));
+}
 } // namespace opts
 
 static llvm::ManagedStatic<SystemLifetimeManager> DebuggerLifetime;
 
+static void dumpClangASTContext(Debugger &Dbg) {
+  for (const auto &File : opts::clang_ast::InputFilenames) {
+    ModuleSpec Spec{FileSpec(File, false)};
+    Spec.GetSymbolFileSpec().SetFile(File, false);
+
+    auto ModulePtr = std::make_shared<lldb_private::Module>(Spec);
+
+    StreamString Stream;
+    ModulePtr->ParseAllDebugSymbols();
+    ModulePtr->Dump(&Stream);
+    llvm::outs() << Stream.GetData() << "\n";
+    llvm::outs().flush();
+  }
+}
+
 static void dumpModules(Debugger &Dbg) {
   LinePrinter Printer(4, llvm::outs());
 
   for (const auto &File : opts::module::InputFilenames) {
     ModuleSpec Spec{FileSpec(File, false)};
     Spec.GetSymbolFileSpec().SetFile(File, false);
 
-    auto ModulePtr = std::make_shared<Module>(Spec);
+    auto ModulePtr = std::make_shared<lldb_private::Module>(Spec);
     SectionList *Sections = ModulePtr->GetSectionList();
     if (!Sections) {
       llvm::errs() << "Could not load sections for module " << File << "\n";
       continue;
     }
 
     size_t Count = Sections->GetNumSections(0);
     Printer.formatLine("Showing {0} sections", Count);
int main(int argc, const char *argv[]) {
 
   DebuggerLifetime->Initialize(llvm::make_unique<SystemInitializerTest>(),
                                nullptr);
 
   auto Dbg = lldb_private::Debugger::CreateInstance();
 
   if (opts::ModuleSubcommand)
     dumpModules(*Dbg);
+  else if (opts::ClangASTSubcommand)
+    dumpClangASTContext(*Dbg);
 
   DebuggerLifetime->Terminate();
   return 0;
 }