[WebAssembly] Expose the offset of each data segment

[sbc100]

Summary

This allows tools like lld that process relocations
to apply data relocation correctly. This information
is required because relocation are stored as section
offset.

[sbc100]

• remove debugging

[sbc100]

@dschuff PTAL? (or sunfish if you have time?)

[dschuff]

So just to be sure I understand, we need this because we only have segments for initialized data (and there is implicitly BSS in between), so the offset can't just be calculated from the sizes of the segments?

include/llvm/Object/Wasm.h

82

Maybe this should be called WasmObjectSegment or something like that WasmSegment made me think it should be some sort of generalization of data and elem segments, which are pretty similar, although we model them differently in LLVM. Or thinking of it another way, will we eventually want a similar model for elem segments (with an offset in the table instead of in the section?)

Actually I guess this is in the llvm:object namespace, so maybe ObjectSegment is redundant and this is unambiguous enough.

[sbc100]

No the motivation is to allow the linker to know which relocations apply to a given segment. Since the relocations all defined as offsets in to the section. Does that make sense? We could go back to the drawing board and define relocations for each segment separately, but that also complicated the definition of relocations. Alternatively the linker could parse the data section itself rather then relying on the libObject abstraction.

[dschuff]

OK, I got it; this is for the linker's use of the lib/object abstraction, that's what I was missing.

include/llvm/BinaryFormat/Wasm.h

 };
 
 struct WasmFunction {
   std::vector<WasmLocalDecl> Locals;
   ArrayRef<uint8_t> Body;
 };
 
 struct WasmDataSegment {
-  uint32_t Index;
+  uint32_t MemoryIndex;
   WasmInitExpr Offset;
   ArrayRef<uint8_t> Content;
 };
 
 struct WasmElemSegment {
   uint32_t TableIndex;
   WasmInitExpr Offset;
   std::vector<uint32_t> Functions;

include/llvm/Object/Wasm.h

Out << "Name=" << Name << ", Type=" << static_cast<int>(Type)
         << ", Flags=" << Flags << " ElemIndex=" << ElementIndex;
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
 #endif
 };
 
-class WasmSection {
-public:
+struct WasmSection {
   WasmSection() = default;
 
   uint32_t Type = 0; // Section type (See below)
   uint32_t Offset = 0; // Offset with in the file
   StringRef Name; // Section name (User-defined sections only)
   ArrayRef<uint8_t> Content; // Section content
   std::vector<wasm::WasmRelocation> Relocations; // Relocations for this section
 };
 
+struct WasmSegment {
+  uint32_t SectionOffset;
+  wasm::WasmDataSegment Data;
+};
+
 class WasmObjectFile : public ObjectFile {
 
 public:
   WasmObjectFile(MemoryBufferRef Object, Error &Err);
 
   const wasm::WasmObjectHeader &getHeader() const;
   const WasmSymbol &getWasmSymbol(const DataRefImpl &Symb) const;
   const WasmSymbol &getWasmSymbol(const SymbolRef &Symbol) const;
public:
   uint32_t getNumberOfSymbols() const {
     return Symbols.size();
   }
 
   const std::vector<wasm::WasmElemSegment>& elements() const {
     return ElemSegments;
   }
 
-  const std::vector<wasm::WasmDataSegment>& dataSegments() const {
+  const std::vector<WasmSegment>& dataSegments() const {
     return DataSegments;
   }
 
   const std::vector<wasm::WasmFunction>& functions() const { return Functions; }
   const ArrayRef<uint8_t>& code() const { return CodeSection; }
   uint32_t startFunction() const { return StartFunction; }
 
   void moveSymbolNext(DataRefImpl &Symb) const override;
private:
   std::vector<wasm::WasmSignature> Signatures;
   std::vector<uint32_t> FunctionTypes;
   std::vector<wasm::WasmTable> Tables;
   std::vector<wasm::WasmLimits> Memories;
   std::vector<wasm::WasmGlobal> Globals;
   std::vector<wasm::WasmImport> Imports;
   std::vector<wasm::WasmExport> Exports;
   std::vector<wasm::WasmElemSegment> ElemSegments;
-  std::vector<wasm::WasmDataSegment> DataSegments;
+  std::vector<WasmSegment> DataSegments;
   std::vector<wasm::WasmFunction> Functions;
   std::vector<WasmSymbol> Symbols;
   ArrayRef<uint8_t> CodeSection;
   uint32_t StartFunction = -1;
   bool HasLinkingSection = false;
   wasm::WasmLinkingData LinkingData;
 
   StringMap<uint32_t> SymbolMap;

include/llvm/ObjectYAML/WasmYAML.h

 struct Relocation {
   RelocType Type;
   uint32_t Index;
   yaml::Hex32 Offset;
   int32_t Addend;
 };
 
 struct DataSegment {
-  uint32_t Index;
+  uint32_t MemoryIndex;
+  uint32_t SectionOffset;
   wasm::WasmInitExpr Offset;
   yaml::BinaryRef Content;
 };
 
 struct NameEntry {
   uint32_t Index;
   StringRef Name;
 };

lib/MC/WasmObjectWriter.cpp

if (WS.isFunction()) {
       if (uint64_t(Size) != Layout.getSectionFileSize(&DataSection))
         report_fatal_error("data sections must contain at most one variable");
 
       DataBytes.resize(alignTo(DataBytes.size(), DataSection.getAlignment()));
       DataAlignment = std::max(DataAlignment, DataSection.getAlignment());
 
       DataSection.setSectionOffset(DataBytes.size());
 
+      uint64_t OldSize = DataBytes.size();
       for (const MCFragment &Frag : DataSection) {
         if (Frag.hasInstructions())
           report_fatal_error("only data supported in data sections");
 
         if (auto *Align = dyn_cast<MCAlignFragment>(&Frag)) {
           if (Align->getValueSize() != 1)
             report_fatal_error("only byte values supported for alignment");
           // If nops are requested, use zeros, as this is the data section.
           uint8_t Value = Align->hasEmitNops() ? 0 : Align->getValue();
           uint64_t Size = std::min<uint64_t>(alignTo(DataBytes.size(),
                                                      Align->getAlignment()),
                                              DataBytes.size() +
                                                  Align->getMaxBytesToEmit());
           DataBytes.resize(Size, Value);
+          DEBUG(dbgs() << "Data: " << DataBytes.size() << " added alignment\n");
         } else if (auto *Fill = dyn_cast<MCFillFragment>(&Frag)) {
+          if (Fill->getSize()) {
             DataBytes.insert(DataBytes.end(), Fill->getSize(), Fill->getValue());
+            DEBUG(dbgs() << "Data: " << DataBytes.size() << " added fill: " << Fill->getSize() << "\n");
+          }
         } else {
           const auto &DataFrag = cast<MCDataFragment>(Frag);
           const SmallVectorImpl<char> &Contents = DataFrag.getContents();
 
           DataBytes.insert(DataBytes.end(), Contents.begin(), Contents.end());
+          DEBUG(dbgs() << "Data: " << DataBytes.size() << " added bytes: " << Contents.size() << "\n");
         }
       }
+      DEBUG(dbgs() << "Added Data: " << WS.getName() << ": " << DataBytes.size() - OldSize << "\n");
 
       // For each global, prepare a corresponding wasm global holding its
       // address.  For externals these will also be named exports.
       Index = NumGlobalImports + Globals.size();
 
       WasmGlobal Global;
       Global.Type = PtrType;
       Global.IsMutable = false;

lib/Object/WasmObjectFile.cpp

Error WasmObjectFile::parseElemSection(const uint8_t *Ptr, const uint8_t *End) {
   }
   if (Ptr != End)
     return make_error<GenericBinaryError>("Elem section ended prematurely",
                                           object_error::parse_failed);
   return Error::success();
 }
 
 Error WasmObjectFile::parseDataSection(const uint8_t *Ptr, const uint8_t *End) {
+  const uint8_t *Start = Ptr;
   uint32_t Count = readVaruint32(Ptr);
   DataSegments.reserve(Count);
   while (Count--) {
-    wasm::WasmDataSegment Segment;
-    Segment.Index = readVaruint32(Ptr);
-    if (Error Err = readInitExpr(Segment.Offset, Ptr))
+    WasmSegment Segment;
+    Segment.Data.MemoryIndex = readVaruint32(Ptr);
+    if (Error Err = readInitExpr(Segment.Data.Offset, Ptr))
       return Err;
     uint32_t Size = readVaruint32(Ptr);
-    Segment.Content = ArrayRef<uint8_t>(Ptr, Size);
+    Segment.Data.Content = ArrayRef<uint8_t>(Ptr, Size);
+    Segment.SectionOffset = Ptr - Start;
     Ptr += Size;
     DataSegments.push_back(Segment);
   }
   if (Ptr != End)
     return make_error<GenericBinaryError>("Data section ended prematurely",
                                           object_error::parse_failed);
   return Error::success();
 }

lib/ObjectYAML/WasmYAML.cpp

void MappingTraits<wasm::WasmInitExpr>::mapping(IO &IO,
   case wasm::WASM_OPCODE_GET_GLOBAL:
     IO.mapRequired("Index", Expr.Value.Global);
     break;
   }
 }
 
 void MappingTraits<WasmYAML::DataSegment>::mapping(
     IO &IO, WasmYAML::DataSegment &Segment) {
-  IO.mapRequired("Index", Segment.Index);
+  IO.mapOptional("SectionOffset", Segment.SectionOffset);
+  IO.mapRequired("MemoryIndex", Segment.MemoryIndex);
   IO.mapRequired("Offset", Segment.Offset);
   IO.mapRequired("Content", Segment.Content);
 }
 
 void MappingTraits<WasmYAML::SymbolInfo>::mapping(IO &IO,
                                                   WasmYAML::SymbolInfo &Info) {
   IO.mapRequired("Name", Info.Name);
   IO.mapRequired("Flags", Info.Flags);

test/MC/WebAssembly/external-data.ll

 ; RUN: llc -mtriple wasm32-unknown-unknown-wasm -filetype=obj %s -o - | obj2yaml | FileCheck %s
 ; Verify relocations are correctly generated for addresses of externals
 ; in the data section.
 
 @myimport = external global i32, align 4
 
 @foo = global i64 7, align 4
 @bar = hidden global i32* @myimport, align 4
 
 ; CHECK:   - Type:            DATA
 ; CHECK:     Relocations:
 ; CHECK:       - Type:            R_WEBASSEMBLY_GLOBAL_ADDR_I32
 ; CHECK:         Index:           0
 ; CHECK:         Offset:          0x0000000E
 ; CHECK:     Segments:
-; CHECK:       - Index:           0
+; CHECK:       - SectionOffset:   6
+; CHECK:         MemoryIndex:     0
 ; CHECK:         Offset:
 ; CHECK:           Opcode:          I32_CONST
 ; CHECK:           Value:           0
 ; CHECK:         Content:         0700000000000000FFFFFFFF
 

test/MC/WebAssembly/unnamed-data.ll

 ; CHECK-NEXT:     Relocations:     
 ; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_GLOBAL_ADDR_I32
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x00000016
 ; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_GLOBAL_ADDR_I32
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x0000001E
 ; CHECK-NEXT:     Segments:        
-; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:       - SectionOffset:   6
+; CHECK-NEXT:         MemoryIndex:     0
 ; CHECK-NEXT:         Offset:          
 ; CHECK-NEXT:           Opcode:          I32_CONST
 ; CHECK-NEXT:           Value:           0
 ; CHECK-NEXT:         Content:         68656C6C6F00776F726C640000000000000000000000000006000000
 ; CHECK-NEXT:   - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
 ; CHECK-NEXT:     DataSize:        28
 ; CHECK-NEXT:     DataAlignment:   8
 ; CHECK-NEXT:     SymbolInfo:      
 ; CHECK-NEXT: ...

test/ObjectYAML/wasm/data_section.yaml

 # RUN: yaml2obj %s | obj2yaml | FileCheck %s
 --- !WASM
 FileHeader:
   Version:         0x00000001
 Sections:
   - Type:            MEMORY
     Memories:
       - Initial:         0x00000003
   - Type:            DATA
     Segments:
-      - Index:           0
+      - MemoryIndex:      0
         Offset:
           Opcode:          I32_CONST
           Value:           4
         Content:         '10001000'
     Relocations:
       - Type:            R_WEBASSEMBLY_GLOBAL_ADDR_I32
         Index:           0
         Offset:          0x00000006

 # CHECK-NEXT:        Index:           0
 # CHECK-NEXT:        Offset:          0x00000006
 # CHECK-NEXT:        Addend:          8
 # CHECK-NEXT:      - Type:            R_WEBASSEMBLY_GLOBAL_ADDR_I32
 # CHECK-NEXT:        Index:           0
 # CHECK-NEXT:        Offset:          0x00000006
 # CHECK-NEXT:        Addend:          -6
 # CHECK-NEXT:     Segments:
-# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:       - SectionOffset:   6
+# CHECK-NEXT:         MemoryIndex:     0
 # CHECK-NEXT:         Offset:
 # CHECK-NEXT:           Opcode:          I32_CONST
 # CHECK-NEXT:           Value:           4
 # CHECK-NEXT:         Content:         '10001000'
 # CHECK-NEXT: ...

tools/obj2yaml/wasm2yaml.cpp

case wasm::WASM_SEC_CODE: {
       }
       S = std::move(CodeSec);
       break;
     }
     case wasm::WASM_SEC_DATA: {
       auto DataSec = make_unique<WasmYAML::DataSection>();
       for (auto &Segment : Obj.dataSegments()) {
         WasmYAML::DataSegment Seg;
-        Seg.Index = Segment.Index;
-        Seg.Offset = Segment.Offset;
-        Seg.Content = yaml::BinaryRef(Segment.Content);
+        Seg.SectionOffset = Segment.SectionOffset;
+        Seg.MemoryIndex = Segment.Data.MemoryIndex;
+        Seg.Offset = Segment.Data.Offset;
+        Seg.Content = yaml::BinaryRef(Segment.Data.Content);
         DataSec->Segments.push_back(Seg);
       }
       S = std::move(DataSec);
       break;
     }
     default:
       llvm_unreachable("Unknown section type");
       break;

tools/yaml2obj/yaml2wasm.cpp

int WasmWriter::writeSectionContent(raw_ostream &OS,
   }
   return 0;
 }
 
 int WasmWriter::writeSectionContent(raw_ostream &OS,
                                     WasmYAML::DataSection &Section) {
   encodeULEB128(Section.Segments.size(), OS);
   for (auto &Segment : Section.Segments) {
-    encodeULEB128(Segment.Index, OS);
+    encodeULEB128(Segment.MemoryIndex, OS);
     writeInitExpr(Segment.Offset, OS);
     encodeULEB128(Segment.Content.binary_size(), OS);
     Segment.Content.writeAsBinary(OS);
   }
   return 0;
 }
 
 int WasmWriter::writeRelocSection(raw_ostream &OS,