[DWARFYAML] Make the unit_length and header_length fields optional.

[Higuoxing]

Summary

This patch makes the unit_length and header_length fields of line tables
optional. yaml2obj is able to infer them for us.

[Higuoxing]

The added test case is verified using llvm-dwarfdump.

[Higuoxing]

Fix failed test.

[jhenderson]

llvm/lib/ObjectYAML/DWARFEmitter.cpp

508

Could you add a TODO somewhere in this function saying that this doesn't support DWARF v5 yet, please?

511

524

I'd get rid of the auto here and below in the for loops. I think it's quite important we know what type they are because of how we write things.

llvm/test/tools/yaml2obj/ELF/DWARF/debug-line.yaml

483

It may be worth having a DWARF64 version of this case too (i.e. where the format is explicitly specified as such). You could add it as a separate subsequent table to this same case, I think (which also then shows that the lengths are calculated independently of one another).

489

The unit_length field looks incorrect. It is currently apparently 0x39, yet the end of the things you check is at offset 0x28, so I'd expect a value of 0x24. The header_length looks right though.

489

Never mind you fixed it whilst I was looking.

518

Nit: delete the extra DWARF:

528

I would suggest a non-standard number of lengths, to show that the header length isn't hard-coded.

529

Similarly, I'd add an include directory here, to show that the length properly includes this field.

[Higuoxing]

Address review comments.

[jhenderson]

LGTM, with nit.

llvm/lib/ObjectYAML/DWARFEmitter.cpp

536

No need for the cast, since OpcodeLength is uint8_t. Does that apply to any of the other writeInteger calls elsewhere in this function too?

[Higuoxing]

Address comments.

Thanks for reviewing!

llvm/include/llvm/ObjectYAML/DWARFYAML.h

struct LineTableOpcode {
   int64_t SData;
   File FileEntry;
   std::vector<llvm::yaml::Hex8> UnknownOpcodeData;
   std::vector<llvm::yaml::Hex64> StandardOpcodeData;
 };
 
 struct LineTable {
   dwarf::DwarfFormat Format;
-  uint64_t Length;
+  Optional<uint64_t> Length;
   uint16_t Version;
-  uint64_t PrologueLength;
+  Optional<uint64_t> PrologueLength;
   uint8_t MinInstLength;
   uint8_t MaxOpsPerInst;
   uint8_t DefaultIsStmt;
   uint8_t LineBase;
   uint8_t LineRange;
   uint8_t OpcodeBase;
   std::vector<uint8_t> StandardOpcodeLengths;
   std::vector<StringRef> IncludeDirs;

llvm/lib/ObjectYAML/DWARFEmitter.cpp

 static void emitFileEntry(raw_ostream &OS, const DWARFYAML::File &File) {
   OS.write(File.Name.data(), File.Name.size());
   OS.write('\0');
   encodeULEB128(File.DirIdx, OS);
   encodeULEB128(File.ModTime, OS);
   encodeULEB128(File.Length, OS);
 }
 
-Error DWARFYAML::emitDebugLine(raw_ostream &OS, const DWARFYAML::Data &DI) {
-  for (const auto &LineTable : DI.DebugLines) {
-    writeInitialLength(LineTable.Format, LineTable.Length, OS,
-                       DI.IsLittleEndian);
-    writeInteger((uint16_t)LineTable.Version, OS, DI.IsLittleEndian);
-    writeDWARFOffset(LineTable.PrologueLength, LineTable.Format, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)LineTable.MinInstLength, OS, DI.IsLittleEndian);
-    if (LineTable.Version >= 4)
-      writeInteger((uint8_t)LineTable.MaxOpsPerInst, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)LineTable.DefaultIsStmt, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)LineTable.LineBase, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)LineTable.LineRange, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)LineTable.OpcodeBase, OS, DI.IsLittleEndian);
-
-    for (auto OpcodeLength : LineTable.StandardOpcodeLengths)
-      writeInteger((uint8_t)OpcodeLength, OS, DI.IsLittleEndian);
-
-    for (auto IncludeDir : LineTable.IncludeDirs) {
-      OS.write(IncludeDir.data(), IncludeDir.size());
-      OS.write('\0');
-    }
-    OS.write('\0');
-
-    for (auto File : LineTable.Files)
-      emitFileEntry(OS, File);
-    OS.write('\0');
-
-    uint8_t AddrSize = DI.Is64BitAddrSize ? 8 : 4;
-
-    for (auto Op : LineTable.Opcodes) {
-      writeInteger((uint8_t)Op.Opcode, OS, DI.IsLittleEndian);
-      if (Op.Opcode == 0) {
-        encodeULEB128(Op.ExtLen, OS);
-        writeInteger((uint8_t)Op.SubOpcode, OS, DI.IsLittleEndian);
-        switch (Op.SubOpcode) {
-        case dwarf::DW_LNE_set_address:
-          cantFail(writeVariableSizedInteger(Op.Data, AddrSize, OS,
-                                             DI.IsLittleEndian));
-          break;
-        case dwarf::DW_LNE_define_file:
-          emitFileEntry(OS, Op.FileEntry);
-          break;
-        case dwarf::DW_LNE_set_discriminator:
-          encodeULEB128(Op.Data, OS);
-          break;
-        case dwarf::DW_LNE_end_sequence:
-          break;
-        default:
-          for (auto OpByte : Op.UnknownOpcodeData)
-            writeInteger((uint8_t)OpByte, OS, DI.IsLittleEndian);
-        }
-      } else if (Op.Opcode < LineTable.OpcodeBase) {
-        switch (Op.Opcode) {
-        case dwarf::DW_LNS_copy:
-        case dwarf::DW_LNS_negate_stmt:
-        case dwarf::DW_LNS_set_basic_block:
-        case dwarf::DW_LNS_const_add_pc:
-        case dwarf::DW_LNS_set_prologue_end:
-        case dwarf::DW_LNS_set_epilogue_begin:
-          break;
+static void writeLineTableOpcode(const DWARFYAML::LineTableOpcode &Op,
+                                 uint8_t OpcodeBase, uint8_t AddrSize,
+                                 raw_ostream &OS, bool IsLittleEndian) {
+  writeInteger((uint8_t)Op.Opcode, OS, IsLittleEndian);
+    writeInteger((uint8_t)Op.SubOpcode, OS, IsLittleEndian);
+      cantFail(
+          writeVariableSizedInteger(Op.Data, AddrSize, OS, IsLittleEndian));
+        writeInteger((uint8_t)OpByte, OS, IsLittleEndian);
+  } else if (Op.Opcode < OpcodeBase) {
 
-        case dwarf::DW_LNS_advance_pc:
-        case dwarf::DW_LNS_set_file:
-        case dwarf::DW_LNS_set_column:
-        case dwarf::DW_LNS_set_isa:
-          encodeULEB128(Op.Data, OS);
-          break;
 
-        case dwarf::DW_LNS_advance_line:
-          encodeSLEB128(Op.SData, OS);
-          break;
 
-        case dwarf::DW_LNS_fixed_advance_pc:
-          writeInteger((uint16_t)Op.Data, OS, DI.IsLittleEndian);
-          break;
+      writeInteger((uint16_t)Op.Data, OS, IsLittleEndian);
 
-        default:
-          for (auto OpData : Op.StandardOpcodeData) {
-            encodeULEB128(OpData, OS);
-          }
-        }
-      }
-    }
+
+Error DWARFYAML::emitDebugLine(raw_ostream &OS, const DWARFYAML::Data &DI) {
+  for (const auto &LineTable : DI.DebugLines) {
+    // Buffer holds the bytes following the header_length (or prologue_length in
+    // DWARFv2) field to the end of line number program itself.
+    std::string Buffer;
+    raw_string_ostream BufferOS(Buffer);
+
+    writeInteger((uint8_t)LineTable.MinInstLength, BufferOS, DI.IsLittleEndian);
+    if (LineTable.Version >= 4)
+      writeInteger((uint8_t)LineTable.MaxOpsPerInst, BufferOS,
+                   DI.IsLittleEndian);
+    writeInteger((uint8_t)LineTable.DefaultIsStmt, BufferOS, DI.IsLittleEndian);
+    writeInteger((uint8_t)LineTable.LineBase, BufferOS, DI.IsLittleEndian);
+    writeInteger((uint8_t)LineTable.LineRange, BufferOS, DI.IsLittleEndian);
+    writeInteger((uint8_t)LineTable.OpcodeBase, BufferOS, DI.IsLittleEndian);
+
+    for (auto OpcodeLength : LineTable.StandardOpcodeLengths)
+      writeInteger((uint8_t)OpcodeLength, BufferOS, DI.IsLittleEndian);
+
+    for (auto IncludeDir : LineTable.IncludeDirs) {
+      BufferOS.write(IncludeDir.data(), IncludeDir.size());
+      BufferOS.write('\0');
+    }
+    BufferOS.write('\0');
+
+    for (auto File : LineTable.Files)
+      emitFileEntry(BufferOS, File);
+    BufferOS.write('\0');
+
+    uint64_t HeaderLength =
+        LineTable.PrologueLength ? *LineTable.PrologueLength : Buffer.size();
+
+    for (const DWARFYAML::LineTableOpcode &Op : LineTable.Opcodes)
+      writeLineTableOpcode(Op, LineTable.OpcodeBase, DI.Is64BitAddrSize ? 8 : 4,
+                           BufferOS, DI.IsLittleEndian);
+
+    uint64_t Length;
+    if (LineTable.Length) {
+      Length = *LineTable.Length;
+    } else {
+      Length = 2; // sizeof(version)
+      Length +=
+          (LineTable.Format == dwarf::DWARF64 ? 8 : 4); // sizeof(header_length)
+      Length += Buffer.size();
+    }
+
+    writeInitialLength(LineTable.Format, Length, OS, DI.IsLittleEndian);
+    writeInteger((uint16_t)LineTable.Version, OS, DI.IsLittleEndian);
+    writeDWARFOffset(HeaderLength, LineTable.Format, OS, DI.IsLittleEndian);
+    OS.write(Buffer.data(), Buffer.size());
   }
 
   return Error::success();
 }
 
 Error DWARFYAML::emitDebugAddr(raw_ostream &OS, const Data &DI) {
   for (const AddrTableEntry &TableEntry : DI.DebugAddr) {
     uint8_t AddrSize;

llvm/lib/ObjectYAML/DWARFYAML.cpp

void MappingTraits<DWARFYAML::LineTableOpcode>::mapping(
   if (LineTableOpcode.Opcode == dwarf::DW_LNS_advance_line || !IO.outputting())
     IO.mapOptional("SData", LineTableOpcode.SData);
   IO.mapOptional("Data", LineTableOpcode.Data);
 }
 
 void MappingTraits<DWARFYAML::LineTable>::mapping(
     IO &IO, DWARFYAML::LineTable &LineTable) {
   IO.mapOptional("Format", LineTable.Format, dwarf::DWARF32);
-  IO.mapRequired("Length", LineTable.Length);
+  IO.mapOptional("Length", LineTable.Length);
   IO.mapRequired("Version", LineTable.Version);
-  IO.mapRequired("PrologueLength", LineTable.PrologueLength);
+  IO.mapOptional("PrologueLength", LineTable.PrologueLength);
   IO.mapRequired("MinInstLength", LineTable.MinInstLength);
   if(LineTable.Version >= 4)
     IO.mapRequired("MaxOpsPerInst", LineTable.MaxOpsPerInst);
   IO.mapRequired("DefaultIsStmt", LineTable.DefaultIsStmt);
   IO.mapRequired("LineBase", LineTable.LineBase);
   IO.mapRequired("LineRange", LineTable.LineRange);
   IO.mapRequired("OpcodeBase", LineTable.OpcodeBase);
   IO.mapRequired("StandardOpcodeLengths", LineTable.StandardOpcodeLengths);

llvm/test/tools/yaml2obj/ELF/DWARF/debug-line.yaml

- Length:                0x1234
             Name:    abcd
             DirIdx:  0x1234
             ModTime: 0x1234
             Length:  0x1234
         - Opcode:    DW_LNS_extended_op
           ExtLen:    0x09
           SubOpcode: DW_LNE_set_discriminator
           Data:      0x1234
+
+## j) Test that yaml2obj is able to infer the length and header_length fields.
+
+# RUN: yaml2obj --docnum=10 %s -o %t10.o
+# RUN: llvm-readelf --hex-dump=.debug_line %t10.o | FileCheck %s --check-prefix=INFER-LENGTH
+
+#      INFER-LENGTH: Hex dump of section '.debug_line':
+# INFER-LENGTH-NEXT: 0x00000000 39000000 04001b00 00000101 01fb0e0d 9...............
+##                              ^-------                            unit_length (4-byte) 0x39
+##                                       ^---                       version (2-byte)
+##                                           ^--------              header_length (4-byte) 0x1b
+##                                                    ^-            minimum_instruction_length (1-byte)
+##                                                      ^-          maximum_operations_per_instruction (1-byte)
+##                                                         ^-       default_is_stmt (1-byte)
+##                                                           ^-     line_base (1-byte) -5
+##                                                             ^-   line_range (1-byte)
+##                                                               ^- opcode_base (1-byte)
+# INFER-LENGTH-NEXT: 0x00000010 00010101 01000000 01000001 00612e63 .............a.c
+##                              ^-------------------------          standard_opcode_lengths (12-byte)
+##                                                         ^-       include_directories null byte
+##                                                           ^----- file_names[0] file_name "a.c\0"
+# INFER-LENGTH-NEXT: 0x00000020 00000000 00000101                   ........
+##                              --
+##                                ^-                                file_names[0] directory index 0x00
+##                                  ^-                              file_names[0] modification time 0x00
+##                                    ^-                            file_names[0] file length 0x00
+##                                       ^-                         file_names null byte
+##                                         ^-                       DW_LNS_extended_op
+##                                           ^-                     extended op length (ULEB128) 0x01
+##                                             ^-                   DW_LNE_end_sequence
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+DWARF:
+DWARF:
+  debug_line:
+    - Version:               4
+      MinInstLength:         1
+      MaxOpsPerInst:         1
+      DefaultIsStmt:         1
+      LineBase:              251
+      LineRange:             14
+      OpcodeBase:            13
+      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
+      IncludeDirs:           []
+      Files:
+        - Name:    a.c
+          DirIdx:  0
+          ModTime: 0
+          Length:  0
+      Opcodes:
+        - Opcode:    DW_LNS_extended_op
+          ExtLen:    1
+          SubOpcode: DW_LNE_end_sequence
+          Data:      0

llvm/tools/obj2yaml/dwarf2yaml.cpp

if (auto StmtOffset =
       uint64_t LengthOrDWARF64Prefix = LineData.getU32(&Offset);
       if (LengthOrDWARF64Prefix == dwarf::DW_LENGTH_DWARF64) {
         DebugLines.Format = dwarf::DWARF64;
         DebugLines.Length = LineData.getU64(&Offset);
       } else {
         DebugLines.Format = dwarf::DWARF32;
         DebugLines.Length = LengthOrDWARF64Prefix;
       }
-      uint64_t LineTableLength = DebugLines.Length;
+      assert(DebugLines.Length);
+      uint64_t LineTableLength = *DebugLines.Length;
       uint64_t SizeOfPrologueLength =
           DebugLines.Format == dwarf::DWARF64 ? 8 : 4;
       DebugLines.Version = LineData.getU16(&Offset);
       DebugLines.PrologueLength =
           LineData.getUnsigned(&Offset, SizeOfPrologueLength);
-      const uint64_t EndPrologue = DebugLines.PrologueLength + Offset;
+      assert(DebugLines.PrologueLength);
+      const uint64_t EndPrologue = *DebugLines.PrologueLength + Offset;
 
       DebugLines.MinInstLength = LineData.getU8(&Offset);
       if (DebugLines.Version >= 4)
         DebugLines.MaxOpsPerInst = LineData.getU8(&Offset);
       DebugLines.DefaultIsStmt = LineData.getU8(&Offset);
       DebugLines.LineBase = LineData.getU8(&Offset);
       DebugLines.LineRange = LineData.getU8(&Offset);
       DebugLines.OpcodeBase = LineData.getU8(&Offset);

llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp

debug_info:
         AbbrOffset:      0
         AddrSize:        8
         Entries:
           - AbbrCode:        0x00000001
             Values:
               - Value:           0x0000000000000001
               - Value:           0x0000000000000000
     debug_line:
-      - Length:          68
-        Version:         2
-        PrologueLength:  34
+      - Version:         2
         MinInstLength:   1
         DefaultIsStmt:   1
         LineBase:        251
         LineRange:       14
         OpcodeBase:      13
         StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
         IncludeDirs:
           - /tmp
debug_info:
         AbbrOffset:      0
         AddrSize:        8
         Entries:
           - AbbrCode:        0x00000001
             Values:
               - Value:           0x0000000000000001
               - Value:           0x0000000000000000
     debug_line:
-      - Length:          61
-        Version:         2
-        PrologueLength:  34
+      - Version:         2
         MinInstLength:   1
         DefaultIsStmt:   1
         LineBase:        251
         LineRange:       14
         OpcodeBase:      13
         StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
         IncludeDirs:
           - /tmp
debug_info:
         AbbrOffset:      0
         AddrSize:        8
         Entries:
           - AbbrCode:        0x00000001
             Values:
               - Value:           0x0000000000000001
               - Value:           0x0000000000000000
     debug_line:
-      - Length:          61
-        Version:         2
-        PrologueLength:  34
+      - Version:         2
         MinInstLength:   1
         DefaultIsStmt:   1
         LineBase:        251
         LineRange:       14
         OpcodeBase:      13
         StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
         IncludeDirs:
           - /tmp
debug_info:
         AbbrOffset:      0
         AddrSize:        8
         Entries:
           - AbbrCode:        0x00000001
             Values:
               - Value:           0x0000000000000001
               - Value:           0x0000000000000000
     debug_line:
-      - Length:          71
-        Version:         2
-        PrologueLength:  44
+      - Version:         2
         MinInstLength:   1
         DefaultIsStmt:   1
         LineBase:        251
         LineRange:       14
         OpcodeBase:      13
         StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
         IncludeDirs:
           - /tmp
debug_info:
         AbbrOffset:      0
         AddrSize:        8
         Entries:
           - AbbrCode:        0x00000001
             Values:
               - Value:           0x000000000000000D
               - Value:           0x0000000000000000
     debug_line:
-      - Length:          60
-        Version:         2
-        PrologueLength:  34
+      - Version:         2
         MinInstLength:   1
         DefaultIsStmt:   1
         LineBase:        251
         LineRange:       14
         OpcodeBase:      13
         StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
         IncludeDirs:
           - /tmp