diff --git a/lld/test/wasm/data-segment-merging.ll b/lld/test/wasm/data-segment-merging.ll --- a/lld/test/wasm/data-segment-merging.ll +++ b/lld/test/wasm/data-segment-merging.ll @@ -1,3 +1,23 @@ +; RUN: llc -filetype=obj %s -o %t.o +; RUN: llc -filetype=obj %s -o %t.atomics.o -mattr=+atomics +; RUN: llc -filetype=obj %s -o %t.bulk-mem.o -mattr=+bulk-memory +; RUN: llc -filetype=obj %s -o %t.atomics.bulk-mem.o -mattr=+atomics,+bulk-memory + +; RUN: wasm-ld -no-gc-sections --no-entry -o %t.merged.wasm %t.o +; RUN: obj2yaml %t.merged.wasm | FileCheck %s --check-prefixes=MERGE,NO-BULK-MEM + +; RUN: wasm-ld -no-gc-sections --no-entry --no-merge-data-segments -o %t.separate.wasm %t.o +; RUN: obj2yaml %t.separate.wasm | FileCheck %s --check-prefixes=SEPARATE,NO-BULK-MEM + +; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 -o %t.merged.atomics.wasm %t.atomics.o +; RUN: obj2yaml %t.merged.atomics.wasm | FileCheck %s --check-prefixes=MERGE-SHARED,NO-BULK-MEM + +; RUN: wasm-ld -no-gc-sections --no-entry -o %t.merged.bulk-mem.wasm %t.bulk-mem.o +; RUN: obj2yaml %t.merged.bulk-mem.wasm | FileCheck %s --check-prefixes=MERGE,BULK-MEM + +; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 -o %t.merged.atomics.bulk-mem.wasm %t.atomics.bulk-mem.o +; RUN: obj2yaml %t.merged.atomics.bulk-mem.wasm | FileCheck %s --check-prefixes=MERGE-SHARED,BULK-MEM + target triple = "wasm32-unknown-unknown" @a = hidden global [6 x i8] c"hello\00", align 1 @@ -8,40 +28,56 @@ @e = private constant [9 x i8] c"constant\00", align 1 @f = private constant i8 43, align 4 -; RUN: llc -filetype=obj %s -o %t.data-segment-merging.o +; NO-BULK-MEM-NOT: DATACOUNT +; BULK-MEM: - Type: DATACOUNT +; BULK-MEM-NEXT: Count: 2 -; RUN: wasm-ld -no-gc-sections --no-entry -o %t.merged.wasm %t.data-segment-merging.o -; RUN: obj2yaml %t.merged.wasm | FileCheck %s --check-prefix=MERGE - -; MERGE-NOT: DATACOUNT +; MERGE: - Type: CODE +; MERGE: Body: 0B ; MERGE: - Type: DATA ; MERGE: Segments: +; MERGE: InitFlags: 0 ; MERGE: Content: 68656C6C6F00676F6F6462796500776861746576657200002A000000 +; MERGE: InitFlags: 0 ; MERGE: Content: 636F6E7374616E74000000002B ; MERGE-NOT: Content: -; RUN: wasm-ld -no-gc-sections --no-entry --no-merge-data-segments -o %t.separate.wasm %t.data-segment-merging.o -; RUN: obj2yaml %t.separate.wasm | FileCheck %s --check-prefix=SEPARATE - -; SEPARATE-NOT: DATACOUNT +; SEPARATE: - Type: CODE +; SEPARATE: Body: 0B ; SEPARATE: - Type: DATA ; SEPARATE: Segments: +; SEPARATE: InitFlags: 0 ; SEPARATE: Content: 68656C6C6F00 +; SEPARATE: InitFlags: 0 ; SEPARATE: Content: 676F6F6462796500 +; SEPARATE: InitFlags: 0 ; SEPARATE: Content: '776861746576657200' +; SEPARATE: InitFlags: 0 ; SEPARATE: Content: 2A000000 +; SEPARATE: InitFlags: 0 ; SEPARATE: Content: 636F6E7374616E7400 ; SEPARATE: Content: 2B ; SEPARATE-NOT: Content: -; RUN: llc -filetype=obj %s -mattr=+bulk-memory -o %t.data-segment-merging.bulk-memory.o -; RUN: wasm-ld -no-gc-sections --no-entry -o %t.merged.bulk-memory.wasm %t.data-segment-merging.bulk-memory.o -; RUN: obj2yaml %t.merged.bulk-memory.wasm | FileCheck %s --check-prefix=BULK-MEMORY - -; BULK-MEMORY: - Type: DATACOUNT -; BULK-MEMORY: Count: 2 -; BULK-MEMORY: - Type: DATA -; BULK-MEMORY: Segments: -; BULK-MEMORY: Content: 68656C6C6F00676F6F6462796500776861746576657200002A000000 -; BULK-MEMORY: Content: 636F6E7374616E74000000002B -; BULK-MEMORY-NOT: Content: +; MERGE-SHARED: - Type: CODE +; MERGE-SHARED-NEXT: Functions: +; MERGE-SHARED-NEXT: - Index: 0 +; MERGE-SHARED-NEXT: Locals: [] +; MERGE-SHARED-NEXT: Body: 10010B +; MERGE-SHARED-NEXT: - Index: 1 +; MERGE-SHARED-NEXT: Locals: [] +; MERGE-SHARED-NEXT: Body: 4180084100411CFC080000FC0900419C084100410DFC080100FC09010B +; MERGE-SHARED: - Type: DATA +; MERGE-SHARED: Segments: +; MERGE-SHARED: InitFlags: 1 +; MERGE-SHARED: Content: 68656C6C6F00676F6F6462796500776861746576657200002A000000 +; MERGE-SHARED: InitFlags: 1 +; MERGE-SHARED: Content: 636F6E7374616E74000000002B +; MERGE-SHARED-NOT: Content: +; MERGE-SHARED: - Type: CUSTOM +; MERGE-SHARED-NEXT: Name: name +; MERGE-SHARED-NEXT: FunctionNames: +; MERGE-SHARED-NEXT: - Index: 0 +; MERGE-SHARED-NEXT: Name: __wasm_call_ctors +; MERGE-SHARED-NEXT: - Index: 1 +; MERGE-SHARED-NEXT: Name: __wasm_init_memory diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp --- a/lld/wasm/Driver.cpp +++ b/lld/wasm/Driver.cpp @@ -432,10 +432,20 @@ "__wasm_call_ctors", WASM_SYMBOL_VISIBILITY_HIDDEN, make(NullSignature, "__wasm_call_ctors")); + if (Config->SharedMemory) { + // When memory is shared, we use passive segments to avoid memory being + // reinitialized on each thread's instantiation. These passive segments + // are initialized and dropped in __wasm_init_memory, which is the first + // function called from __wasm_call_ctors. + WasmSym::InitMemory = Symtab->addSyntheticFunction( + "__wasm_init_memory", WASM_SYMBOL_VISIBILITY_HIDDEN, + make(NullSignature, "__wasm_init_memory")); + } + if (Config->Pic) { // For PIC code we create a synthetic function call __wasm_apply_relocs - // and add this as the first call in __wasm_call_ctors. - // We also unconditionally export + // and add this as the second call in __wasm_call_ctors. + // We also unconditionally export WasmSym::ApplyRelocs = Symtab->addSyntheticFunction( "__wasm_apply_relocs", WASM_SYMBOL_VISIBILITY_HIDDEN, make(NullSignature, "__wasm_apply_relocs")); diff --git a/lld/wasm/MarkLive.cpp b/lld/wasm/MarkLive.cpp --- a/lld/wasm/MarkLive.cpp +++ b/lld/wasm/MarkLive.cpp @@ -76,11 +76,18 @@ } } - if (Config->Pic) { + if (Config->Pic || Config->SharedMemory) { Enqueue(WasmSym::CallCtors); + } + + if (Config->Pic) { Enqueue(WasmSym::ApplyRelocs); } + if (Config->SharedMemory) { + Enqueue(WasmSym::InitMemory); + } + // Follow relocations to mark all reachable chunks. while (!Q.empty()) { InputChunk *C = Q.pop_back_val(); diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -159,20 +159,26 @@ OS.flush(); BodySize = DataSectionHeader.size(); + assert(!Config->Pic || + Segments.size() <= 1 && + "Currenly only a single data segment is supported in PIC mode"); + for (OutputSegment *Segment : Segments) { raw_string_ostream OS(Segment->Header); - writeUleb128(OS, 0, "memory index"); - WasmInitExpr InitExpr; - if (Config->Pic) { - assert(Segments.size() <= 1 && - "Currenly only a single data segment is supported in PIC mode"); - InitExpr.Opcode = WASM_OPCODE_GLOBAL_GET; - InitExpr.Value.Global = WasmSym::MemoryBase->getGlobalIndex(); - } else { - InitExpr.Opcode = WASM_OPCODE_I32_CONST; - InitExpr.Value.Int32 = Segment->StartVA; + writeUleb128(OS, Segment->InitFlags, "init flags"); + if (Segment->InitFlags & WASM_SEGMENT_HAS_MEMINDEX) + writeUleb128(OS, 0, "memory index"); + if ((Segment->InitFlags & WASM_SEGMENT_IS_PASSIVE) == 0) { + WasmInitExpr InitExpr; + if (Config->Pic) { + InitExpr.Opcode = WASM_OPCODE_GLOBAL_GET; + InitExpr.Value.Global = WasmSym::MemoryBase->getGlobalIndex(); + } else { + InitExpr.Opcode = WASM_OPCODE_I32_CONST; + InitExpr.Value.Int32 = Segment->StartVA; + } + writeInitExpr(OS, InitExpr); } - writeInitExpr(OS, InitExpr); writeUleb128(OS, Segment->Size, "segment size"); OS.flush(); diff --git a/lld/wasm/OutputSegment.h b/lld/wasm/OutputSegment.h --- a/lld/wasm/OutputSegment.h +++ b/lld/wasm/OutputSegment.h @@ -33,6 +33,7 @@ StringRef Name; const uint32_t Index; + uint32_t InitFlags = 0; uint32_t SectionOffset = 0; uint32_t Alignment = 0; uint32_t StartVA = 0; diff --git a/lld/wasm/Symbols.h b/lld/wasm/Symbols.h --- a/lld/wasm/Symbols.h +++ b/lld/wasm/Symbols.h @@ -406,6 +406,10 @@ // Function that directly calls all ctors in priority order. static DefinedFunction *CallCtors; + // __wasm_init_memory + // Function that initializes passive data segments post-instantiation. + static DefinedFunction *InitMemory; + // __wasm_apply_relocs // Function that applies relocations to data segment post-instantiation. static DefinedFunction *ApplyRelocs; diff --git a/lld/wasm/Symbols.cpp b/lld/wasm/Symbols.cpp --- a/lld/wasm/Symbols.cpp +++ b/lld/wasm/Symbols.cpp @@ -24,6 +24,7 @@ using namespace lld::wasm; DefinedFunction *WasmSym::CallCtors; +DefinedFunction *WasmSym::InitMemory; DefinedFunction *WasmSym::ApplyRelocs; DefinedData *WasmSym::DsoHandle; DefinedData *WasmSym::DataEnd; diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -65,6 +65,7 @@ uint32_t lookupType(const WasmSignature &Sig); uint32_t registerType(const WasmSignature &Sig); + void createInitMemoryFunction(); void createApplyRelocationsFunction(); void createCallCtorsFunction(); @@ -1295,6 +1296,8 @@ if (S == nullptr) { LLVM_DEBUG(dbgs() << "new segment: " << Name << "\n"); S = make(Name, Segments.size()); + if (Config->SharedMemory) + S->InitFlags = WASM_SEGMENT_IS_PASSIVE; Segments.push_back(S); } S->addInputSegment(Segment); @@ -1303,10 +1306,58 @@ } } +static void CreateFunction(DefinedFunction *Func, + const std::string &BodyContent) { + std::string FunctionBody; + { + raw_string_ostream OS(FunctionBody); + writeUleb128(OS, BodyContent.size(), "function size"); + OS << BodyContent; + } + ArrayRef Body = arrayRefFromStringRef(Saver.save(FunctionBody)); + cast(Func->Function)->setBody(Body); +} + +void Writer::createInitMemoryFunction() { + LLVM_DEBUG(dbgs() << "createMemoryInitFunction\n"); + std::string BodyContent; + { + raw_string_ostream OS(BodyContent); + writeUleb128(OS, 0, "num locals"); + + // initialize passive data segments + for (const OutputSegment *S : Segments) { + if (S->InitFlags & WASM_SEGMENT_IS_PASSIVE) { + // destination address + writeU8(OS, WASM_OPCODE_I32_CONST, "i32.const"); + writeUleb128(OS, S->StartVA, "destination address"); + // source segment offset + writeU8(OS, WASM_OPCODE_I32_CONST, "i32.const"); + writeUleb128(OS, 0, "segment offset"); + // memory region size + writeU8(OS, WASM_OPCODE_I32_CONST, "i32.const"); + writeUleb128(OS, S->Size, "memory region size"); + // memory.init instruction + writeU8(OS, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); + writeUleb128(OS, WASM_OPCODE_MEMORY_INIT, "MEMORY.INIT"); + writeUleb128(OS, S->Index, "segment index immediate"); + writeU8(OS, 0, "memory index immediate"); + // data.drop instruction + writeU8(OS, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); + writeUleb128(OS, WASM_OPCODE_DATA_DROP, "DATA.DROP"); + writeUleb128(OS, S->Index, "segment index immediate"); + } + } + writeU8(OS, WASM_OPCODE_END, "END"); + } + + CreateFunction(WasmSym::InitMemory, BodyContent); +} + // For -shared (PIC) output, we create create a synthetic function which will // apply any relocations to the data segments on startup. This function is -// called __wasm_apply_relocs and is added at the very beginning of -// __wasm_call_ctors before any of the constructors run. +// called __wasm_apply_relocs and is added at the beginning of __wasm_call_ctors +// before any of the constructors run. void Writer::createApplyRelocationsFunction() { LLVM_DEBUG(dbgs() << "createApplyRelocationsFunction\n"); // First write the body's contents to a string. @@ -1320,16 +1371,7 @@ writeU8(OS, WASM_OPCODE_END, "END"); } - // Once we know the size of the body we can create the final function body - std::string FunctionBody; - { - raw_string_ostream OS(FunctionBody); - writeUleb128(OS, BodyContent.size(), "function size"); - OS << BodyContent; - } - - ArrayRef Body = arrayRefFromStringRef(Saver.save(FunctionBody)); - cast(WasmSym::ApplyRelocs->Function)->setBody(Body); + CreateFunction(WasmSym::ApplyRelocs, BodyContent); } // Create synthetic "__wasm_call_ctors" function based on ctor functions @@ -1343,11 +1385,20 @@ { raw_string_ostream OS(BodyContent); writeUleb128(OS, 0, "num locals"); + + if (Config->SharedMemory) { + writeU8(OS, WASM_OPCODE_CALL, "CALL"); + writeUleb128(OS, WasmSym::InitMemory->getFunctionIndex(), + "function index"); + } + if (Config->Pic) { writeU8(OS, WASM_OPCODE_CALL, "CALL"); writeUleb128(OS, WasmSym::ApplyRelocs->getFunctionIndex(), "function index"); } + + // call constructors for (const WasmInitEntry &F : InitFunctions) { writeU8(OS, WASM_OPCODE_CALL, "CALL"); writeUleb128(OS, F.Sym->getFunctionIndex(), "function index"); @@ -1355,16 +1406,7 @@ writeU8(OS, WASM_OPCODE_END, "END"); } - // Once we know the size of the body we can create the final function body - std::string FunctionBody; - { - raw_string_ostream OS(FunctionBody); - writeUleb128(OS, BodyContent.size(), "function size"); - OS << BodyContent; - } - - ArrayRef Body = arrayRefFromStringRef(Saver.save(FunctionBody)); - cast(WasmSym::CallCtors->Function)->setBody(Body); + CreateFunction(WasmSym::CallCtors, BodyContent); } // Populate InitFunctions vector with init functions from all input objects. @@ -1408,13 +1450,15 @@ calculateImports(); log("-- assignIndexes"); assignIndexes(); - log("-- calculateInitFunctions"); - calculateInitFunctions(); log("-- calculateTypes"); calculateTypes(); log("-- layoutMemory"); layoutMemory(); + log("-- calculateInitFunctions"); + calculateInitFunctions(); if (!Config->Relocatable) { + if (Config->SharedMemory) + createInitMemoryFunction(); if (Config->Pic) createApplyRelocationsFunction(); createCallCtorsFunction(); diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h --- a/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/llvm/include/llvm/BinaryFormat/Wasm.h @@ -249,6 +249,9 @@ WASM_OPCODE_F32_CONST = 0x43, WASM_OPCODE_F64_CONST = 0x44, WASM_OPCODE_I32_ADD = 0x6a, + WASM_OPCODE_MISC_PREFIX = 0xfc, + WASM_OPCODE_MEMORY_INIT = 0x08, + WASM_OPCODE_DATA_DROP = 0x09, }; enum : unsigned {