[WebAssembly] Improve libObject support for reading/writing wasm imports and exports

[sbc100]

Summary

Previously we had only supported reading/writing certain
types of imports and exports (functions and global).

Also add usefull overload of getWasmSymbol() and
getNumberOfSymbols() in support of lld port.

[dschuff]

include/llvm/Object/Wasm.h

85

I'm surprised there's not some method for this purpose on ObjectFile... but I don't see one...

lib/Object/WasmObjectFile.cpp

512–513

I think table is the only one left, right?

[sbc100]

• handle all import and export types

[sbc100]

lib/Object/WasmObjectFile.cpp

512–513

OK, I made this a more complete change so now we can support all external types.

[dschuff]

Cool, looks good. Maybe just clarify in the commit description that this really just applies to reading and writing the sections rather than e.g. codegen.

include/llvm/Object/Wasm.h

public:
   std::vector<wasm::WasmRelocation> Relocations; // Relocations for this section
 };
 
 class WasmObjectFile : public ObjectFile {
 public:
   WasmObjectFile(MemoryBufferRef Object, Error &Err);
 
   const wasm::WasmObjectHeader &getHeader() const;
-  const WasmSymbol &getWasmSymbol(DataRefImpl Symb) const;
+  const WasmSymbol &getWasmSymbol(const DataRefImpl &Symb) const;
+  const WasmSymbol &getWasmSymbol(const SymbolRef &Symbol) const;
   const WasmSection &getWasmSection(const SectionRef &Section) const;
   const wasm::WasmRelocation &getWasmRelocation(const RelocationRef& Ref) const;
 
   static bool classof(const Binary *v) { return v->isWasm(); }
 
   const std::vector<wasm::WasmSignature>& types() const { return Signatures; }
   const std::vector<uint32_t>& functionTypes() const { return FunctionTypes; }
   const std::vector<wasm::WasmImport>& imports() const { return Imports; }
   const std::vector<wasm::WasmTable>& tables() const { return Tables; }
   const std::vector<wasm::WasmLimits>& memories() const { return Memories; }
   const std::vector<wasm::WasmGlobal>& globals() const { return Globals; }
   const std::vector<wasm::WasmExport>& exports() const { return Exports; }
 
+  uint32_t getNumberOfSymbols() const {
+    return Symbols.size();
+  }
+
   const std::vector<wasm::WasmElemSegment>& elements() const {
     return ElemSegments;
   }
 
   const std::vector<wasm::WasmDataSegment>& dataSegments() const {
     return DataSegments;
   }
 

include/llvm/ObjectYAML/WasmYAML.h

 LLVM_YAML_STRONG_TYPEDEF(uint32_t, ExportKind)
 LLVM_YAML_STRONG_TYPEDEF(uint32_t, Opcode)
 LLVM_YAML_STRONG_TYPEDEF(uint32_t, RelocType)
 
 struct FileHeader {
   yaml::Hex32 Version;
 };
 
-struct Import {
-  StringRef Module;
-  StringRef Field;
-  ExportKind Kind;
-  union {
-    uint32_t SigIndex;
-    ValueType GlobalType;
-  };
-  bool GlobalMutable;
-};
-
 struct Limits {
   yaml::Hex32 Flags;
   yaml::Hex32 Initial;
   yaml::Hex32 Maximum;
 };
 
 struct Table {
   TableType ElemType;

 };
 
 struct Global {
   ValueType Type;
   bool Mutable;
   wasm::WasmInitExpr InitExpr;
 };
 
+struct Import {
+  StringRef Module;
+  StringRef Field;
+  ExportKind Kind;
+  union {
+    uint32_t SigIndex;
+    Global Global;
+    Table Table;
+    Limits Memory;
+  };
+};
+
 struct LocalDecl {
   ValueType Type;
   uint32_t Count;
 };
 
 struct Function {
   std::vector<LocalDecl> Locals;
   yaml::BinaryRef Body;

include/llvm/Support/Wasm.h

struct WasmObjectHeader {
   uint32_t Version;
 };
 
 struct WasmSignature {
   std::vector<int32_t> ParamTypes;
   int32_t ReturnType;
 };
 
-struct WasmImport {
-  StringRef Module;
-  StringRef Field;
-  uint32_t Kind;
-  union {
-    uint32_t SigIndex;
-    int32_t GlobalType;
-  };
-  bool GlobalMutable;
-};
-
 struct WasmExport {
   StringRef Name;
   uint32_t Kind;
   uint32_t Index;
 };
 
 struct WasmLimits {
   uint32_t Flags;

 };
 
 struct WasmGlobal {
   int32_t Type;
   bool Mutable;
   WasmInitExpr InitExpr;
 };
 
+struct WasmImport {
+  StringRef Module;
+  StringRef Field;
+  uint32_t Kind;
+  union {
+    uint32_t SigIndex;
+    WasmGlobal Global;
+    WasmTable Table;
+    WasmLimits Memory;
+  };
+};
+
 struct WasmLocalDecl {
   int32_t Type;
   uint32_t Count;
 };
 
 struct WasmFunction {
   std::vector<WasmLocalDecl> Locals;
   ArrayRef<uint8_t> Body;

lib/Object/WasmObjectFile.cpp

static wasm::WasmLimits readLimits(const uint8_t *&Ptr) {
   wasm::WasmLimits Result;
   Result.Flags = readVaruint1(Ptr);
   Result.Initial = readVaruint32(Ptr);
   if (Result.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
     Result.Maximum = readVaruint32(Ptr);
   return Result;
 }
 
+static wasm::WasmTable readTable(const uint8_t *&Ptr) {
+  wasm::WasmTable Table;
+  Table.ElemType = readVarint7(Ptr);
+  Table.Limits = readLimits(Ptr);
+  return Table;
+}
+
 static Error readSection(WasmSection &Section, const uint8_t *&Ptr,
                          const uint8_t *Start) {
   // TODO(sbc): Avoid reading past EOF in the case of malformed files.
   Section.Offset = Ptr - Start;
   Section.Type = readVaruint7(Ptr);
   uint32_t Size = readVaruint32(Ptr);
   if (Size == 0)
     return make_error<StringError>("Zero length section",
for (uint32_t i = 0; i < Count; i++) {
     Im.Kind = readUint8(Ptr);
     switch (Im.Kind) {
     case wasm::WASM_EXTERNAL_FUNCTION:
       Im.SigIndex = readVaruint32(Ptr);
       Symbols.emplace_back(Im.Field, WasmSymbol::SymbolType::FUNCTION_IMPORT,
                            Sections.size(), i);
       break;
     case wasm::WASM_EXTERNAL_GLOBAL:
-      Im.GlobalType = readVarint7(Ptr);
-      Im.GlobalMutable = readVaruint1(Ptr);
+      Im.Global.Type = readVarint7(Ptr);
+      Im.Global.Mutable = readVaruint1(Ptr);
       Symbols.emplace_back(Im.Field, WasmSymbol::SymbolType::GLOBAL_IMPORT,
                            Sections.size(), i);
       break;
+    case wasm::WASM_EXTERNAL_MEMORY:
+      Im.Memory = readLimits(Ptr);
+      break;
+    case wasm::WASM_EXTERNAL_TABLE:
+      Im.Table = readTable(Ptr);
+      if (Im.Table.ElemType != wasm::WASM_TYPE_ANYFUNC) {
+        return make_error<GenericBinaryError>("Invalid table element type",
+                                              object_error::parse_failed);
+      }
+      break;
     default:
-      // TODO(sbc): Handle other kinds of imports
       return make_error<GenericBinaryError>(
           "Unexpected import kind", object_error::parse_failed);
     }
     Imports.push_back(Im);
   }
   if (Ptr != End)
     return make_error<GenericBinaryError>("Import section ended prematurely",
                                           object_error::parse_failed);
return make_error<GenericBinaryError>("Function section ended prematurely",
                                           object_error::parse_failed);
   return Error::success();
 }
 
 Error WasmObjectFile::parseTableSection(const uint8_t *Ptr, const uint8_t *End) {
   uint32_t Count = readVaruint32(Ptr);
   Tables.reserve(Count);
   while (Count--) {
-    wasm::WasmTable Table;
-    Table.ElemType = readVarint7(Ptr);
-    if (Table.ElemType != wasm::WASM_TYPE_ANYFUNC) {
+    Tables.push_back(readTable(Ptr));
+    if (Tables.back().ElemType != wasm::WASM_TYPE_ANYFUNC) {
       return make_error<GenericBinaryError>("Invalid table element type",
                                             object_error::parse_failed);
     }
-    Table.Limits = readLimits(Ptr);
-    Tables.push_back(Table);
   }
   if (Ptr != End)
     return make_error<GenericBinaryError>("Table section ended prematurely",
                                           object_error::parse_failed);
   return Error::success();
 }
 
 Error WasmObjectFile::parseMemorySection(const uint8_t *Ptr, const uint8_t *End) {
for (uint32_t i = 0; i < Count; i++) {
     case wasm::WASM_EXTERNAL_FUNCTION:
       Symbols.emplace_back(Ex.Name, WasmSymbol::SymbolType::FUNCTION_EXPORT,
                            Sections.size(), i);
       break;
     case wasm::WASM_EXTERNAL_GLOBAL:
       Symbols.emplace_back(Ex.Name, WasmSymbol::SymbolType::GLOBAL_EXPORT,
                            Sections.size(), i);
       break;
+    case wasm::WASM_EXTERNAL_MEMORY:
+    case wasm::WASM_EXTERNAL_TABLE:
+      break;
     default:
-      // TODO(sbc): Handle other kinds of exports
       return make_error<GenericBinaryError>(
           "Unexpected export kind", object_error::parse_failed);
     }
   }
   if (Ptr != End)
     return make_error<GenericBinaryError>("Export section ended prematurely",
                                           object_error::parse_failed);
   return Error::success();
 }

 }
 
 basic_symbol_iterator WasmObjectFile::symbol_end() const {
   DataRefImpl Ref;
   Ref.d.a = Symbols.size();
   return BasicSymbolRef(Ref, this);
 }
 
-const WasmSymbol &WasmObjectFile::getWasmSymbol(DataRefImpl Symb) const {
+const WasmSymbol &WasmObjectFile::getWasmSymbol(const DataRefImpl &Symb) const {
   return Symbols[Symb.d.a];
 }
 
+const WasmSymbol &WasmObjectFile::getWasmSymbol(const SymbolRef &Symb) const {
+  return getWasmSymbol(Symb.getRawDataRefImpl());
+}
+
 Expected<StringRef> WasmObjectFile::getSymbolName(DataRefImpl Symb) const {
   const WasmSymbol &Sym = getWasmSymbol(Symb);
   return Sym.Name;
 }
 
 Expected<uint64_t> WasmObjectFile::getSymbolAddress(DataRefImpl Symb) const {
   return getSymbolValue(Symb);
 }

lib/ObjectYAML/WasmYAML.cpp

 void MappingTraits<WasmYAML::Import>::mapping(IO &IO,
                                               WasmYAML::Import &Import) {
   IO.mapRequired("Module", Import.Module);
   IO.mapRequired("Field", Import.Field);
   IO.mapRequired("Kind", Import.Kind);
   if (Import.Kind == wasm::WASM_EXTERNAL_FUNCTION) {
     IO.mapRequired("SigIndex", Import.SigIndex);
   } else if (Import.Kind == wasm::WASM_EXTERNAL_GLOBAL) {
-    IO.mapRequired("GlobalType", Import.GlobalType);
-    IO.mapRequired("GlobalMutable", Import.GlobalMutable);
+    IO.mapRequired("GlobalType", Import.Global.Type);
+    IO.mapRequired("GlobalMutable", Import.Global.Mutable);
+  } else if (Import.Kind == wasm::WASM_EXTERNAL_TABLE) {
+    IO.mapRequired("Table", Import.Table);
+  } else if (Import.Kind == wasm::WASM_EXTERNAL_MEMORY ) {
+    IO.mapRequired("Memory", Import.Memory);
   } else {
     llvm_unreachable("unhandled import type");
   }
 }
 
 void MappingTraits<WasmYAML::Export>::mapping(IO &IO,
                                               WasmYAML::Export &Export) {
   IO.mapRequired("Name", Export.Name);

test/ObjectYAML/wasm/export_section.yaml

 # RUN: yaml2obj %s | obj2yaml | FileCheck %s
 --- !WASM
 FileHeader:
   Version:         0x00000001
 Sections:
   - Type:            EXPORT
     Exports:         
-      - Name:            foo
-        Kind:            FUNCTION
-        Index:           0
-      - Name:            bar
+      - Name:            function_export
         Kind:            FUNCTION
         Index:           1
+      - Name:            global_export
+        Kind:            GLOBAL
+        Index:           1
+      - Name:            memory_export
+        Kind:            MEMORY
+        Index:           0
+      - Name:            table_export
+        Kind:            TABLE
+        Index:           0
 ...
 # CHECK: --- !WASM
 # CHECK: FileHeader:
 # CHECK:   Version:           0x00000001
 # CHECK: Sections:
 # CHECK:   - Type:            EXPORT
 # CHECK:     Exports:         
-# CHECK:       - Name:            foo
-# CHECK:         Kind:            FUNCTION
-# CHECK:         Index:           0
-# CHECK:       - Name:            bar
+# CHECK:       - Name:            function_export
 # CHECK:         Kind:            FUNCTION
 # CHECK:         Index:           1
+# CHECK:       - Name:            global_export
+# CHECK:         Kind:            GLOBAL
+# CHECK:         Index:           1
+# CHECK:       - Name:            memory_export
+# CHECK:         Kind:            MEMORY
+# CHECK:         Index:           0
+# CHECK:       - Name:            table_export
+# CHECK:         Kind:            TABLE
+# CHECK:         Index:           0
 # CHECK: ...

test/ObjectYAML/wasm/import_section.yaml

 # RUN: yaml2obj %s | obj2yaml | FileCheck %s
 --- !WASM
 FileHeader:
   Version:         0x00000001
 Sections:
   - Type:            TYPE
     Signatures:
       - ReturnType:      I32
         ParamTypes:
           - I32
   - Type:            IMPORT
-    Imports:         
+    Imports:
       - Module:          foo
-        Field:           bar
+        Field:           imported_function
         Kind:            FUNCTION
         SigIndex:        0
       - Module:          fiz
-        Field:           baz
+        Field:           imported_global
         Kind:            GLOBAL
         GlobalType:      I32
         GlobalMutable:   false
-  - Type:            FUNCTION
-    FunctionTypes:
-      - 0
+      - Module:          foo
+        Field:           imported_memory
+        Kind:            MEMORY
+        Memory:
+          Flags:           0x00000001
+          Initial:         0x00000010
+          Maximum:         0x00000011
+      - Module:          foo
+        Field:           imported_table
+        Kind:            TABLE
+        Table:
+          ElemType:      ANYFUNC
+          Limits:
+            Flags:           0x00000001
+            Initial:         0x00000020
+            Maximum:         0x00000022
 ...
 # CHECK: --- !WASM
 # CHECK: FileHeader:
 # CHECK:   Version:           0x00000001
 # CHECK: Sections:
 # CHECK:   - Type:            IMPORT
 # CHECK:     Imports:         
 # CHECK:       - Module:          foo
-# CHECK:         Field:           bar
+# CHECK:         Field:           imported_function
 # CHECK:         Kind:            FUNCTION
 # CHECK:         SigIndex:        0
 # CHECK:       - Module:          fiz
-# CHECK:         Field:           baz
+# CHECK:         Field:           imported_global
 # CHECK:         Kind:            GLOBAL
 # CHECK:         GlobalType:      I32
 # CHECK:         GlobalMutable:   false
+# CHECK:       - Module:          foo
+# CHECK:         Field:           imported_memory
+# CHECK:         Kind:            MEMORY
+# CHECK:         Memory:
+# CHECK:           Flags:           0x00000001
+# CHECK:           Initial:         0x00000010
+# CHECK:           Maximum:         0x00000011
+# CHECK:       - Module:          foo
+# CHECK:         Field:           imported_table
+# CHECK:         Kind:            TABLE
+# CHECK:         Table:
+# CHECK:           ElemType:      ANYFUNC
+# CHECK:           Limits:
+# CHECK:             Flags:           0x00000001
+# CHECK:             Initial:         0x00000020
+# CHECK:             Maximum:         0x00000022
 # CHECK: ...

tools/obj2yaml/wasm2yaml.cpp

 class WasmDumper {
   const object::WasmObjectFile &Obj;
 
 public:
   WasmDumper(const object::WasmObjectFile &O) : Obj(O) {}
   ErrorOr<WasmYAML::Object *> dump();
 };
 
+WasmYAML::Table make_table(const wasm::WasmTable &Table) {
+  WasmYAML::Table T;
+  T.ElemType = Table.ElemType;
+  T.TableLimits.Flags = Table.Limits.Flags;
+  T.TableLimits.Initial = Table.Limits.Initial;
+  T.TableLimits.Maximum = Table.Limits.Maximum;
+  return T;
+}
+
+WasmYAML::Limits make_limits(const wasm::WasmLimits &Limits) {
+  WasmYAML::Limits L;
+  L.Flags = Limits.Flags;
+  L.Initial = Limits.Initial;
+  L.Maximum = Limits.Maximum;
+  return L;
+}
+
 ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
   auto Y = make_unique<WasmYAML::Object>();
 
   // Dump header
   Y->Header.Version = Obj.getHeader().Version;
 
   // Dump sections
   for (const auto &Sec : Obj.sections()) {
case wasm::WASM_SEC_TYPE: {
         TypeSec->Signatures.push_back(Sig);
       }
       S = std::move(TypeSec);
       break;
     }
     case wasm::WASM_SEC_IMPORT: {
       auto ImportSec = make_unique<WasmYAML::ImportSection>();
       for (auto &Import : Obj.imports()) {
-        WasmYAML::Import Ex;
-        Ex.Module = Import.Module;
-        Ex.Field = Import.Field;
-        Ex.Kind = Import.Kind;
-        if (Ex.Kind == wasm::WASM_EXTERNAL_FUNCTION) {
-          Ex.SigIndex = Import.SigIndex;
-        } else if (Ex.Kind == wasm::WASM_EXTERNAL_GLOBAL) {
-          Ex.GlobalType = Import.GlobalType;
-          Ex.GlobalMutable = Import.GlobalMutable;
+        WasmYAML::Import Im;
+        Im.Module = Import.Module;
+        Im.Field = Import.Field;
+        Im.Kind = Import.Kind;
+        switch (Im.Kind) {
+        case wasm::WASM_EXTERNAL_FUNCTION:
+          Im.SigIndex = Import.SigIndex;
+          break;
+        case wasm::WASM_EXTERNAL_GLOBAL:
+          Im.Global.Type = Import.Global.Type;
+          Im.Global.Mutable = Import.Global.Mutable;
+          break;
+        case wasm::WASM_EXTERNAL_TABLE:
+          Im.Table = make_table(Import.Table);
+          break;
+        case wasm::WASM_EXTERNAL_MEMORY:
+          Im.Memory = make_limits(Import.Memory);
+          break;
         }
-        ImportSec->Imports.push_back(Ex);
+        ImportSec->Imports.push_back(Im);
       }
       S = std::move(ImportSec);
       break;
     }
     case wasm::WASM_SEC_FUNCTION: {
       auto FuncSec = make_unique<WasmYAML::FunctionSection>();
       for (const auto &Func : Obj.functionTypes()) {
         FuncSec->FunctionTypes.push_back(Func);
       }
       S = std::move(FuncSec);
       break;
     }
     case wasm::WASM_SEC_TABLE: {
       auto TableSec = make_unique<WasmYAML::TableSection>();
-      for (auto &Table : Obj.tables()) {
-        WasmYAML::Table T;
-        T.ElemType = Table.ElemType;
-        T.TableLimits.Flags = Table.Limits.Flags;
-        T.TableLimits.Initial = Table.Limits.Initial;
-        T.TableLimits.Maximum = Table.Limits.Maximum;
-        TableSec->Tables.push_back(T);
+      for (const wasm::WasmTable &Table : Obj.tables()) {
+        TableSec->Tables.push_back(make_table(Table));
       }
       S = std::move(TableSec);
       break;
     }
     case wasm::WASM_SEC_MEMORY: {
       auto MemorySec = make_unique<WasmYAML::MemorySection>();
-      for (auto &Memory : Obj.memories()) {
-        WasmYAML::Limits L;
-        L.Flags = Memory.Flags;
-        L.Initial = Memory.Initial;
-        L.Maximum = Memory.Maximum;
-        MemorySec->Memories.push_back(L);
+      for (const wasm::WasmLimits &Memory : Obj.memories()) {
+        MemorySec->Memories.push_back(make_limits(Memory));
       }
       S = std::move(MemorySec);
       break;
     }
     case wasm::WASM_SEC_GLOBAL: {
       auto GlobalSec = make_unique<WasmYAML::GlobalSection>();
       for (auto &Global : Obj.globals()) {
         WasmYAML::Global G;

tools/yaml2obj/yaml2wasm.cpp

for (const WasmYAML::Import &Import : Section.Imports) {
     writeStringRef(Import.Module, OS);
     writeStringRef(Import.Field, OS);
     encodeULEB128(Import.Kind, OS);
     switch (Import.Kind) {
     case wasm::WASM_EXTERNAL_FUNCTION:
       encodeULEB128(Import.SigIndex, OS);
       break;
     case wasm::WASM_EXTERNAL_GLOBAL:
-      encodeSLEB128(Import.GlobalType, OS);
-      writeUint8(OS, Import.GlobalMutable);
+      encodeSLEB128(Import.Global.Type, OS);
+      writeUint8(OS, Import.Global.Mutable);
+      break;
+    case wasm::WASM_EXTERNAL_MEMORY:
+      writeLimits(Import.Memory, OS);
+      break;
+    case wasm::WASM_EXTERNAL_TABLE:
+      encodeSLEB128(Import.Table.ElemType, OS);
+      writeLimits(Import.Table.TableLimits, OS);
       break;
     default:
       errs() << "Unknown import type: " << Import.Kind;
       return 1;
     }
   }
   return 0;
 }