diff --git a/lld/test/wasm/data-segment-merging.ll b/lld/test/wasm/data-segment-merging.ll --- a/lld/test/wasm/data-segment-merging.ll +++ b/lld/test/wasm/data-segment-merging.ll @@ -98,9 +98,9 @@ ; PASSIVE-MERGE-NEXT: - Index: 0 ; PASSIVE-MERGE-NEXT: Name: __wasm_call_ctors ; PASSIVE-MERGE-NEXT: - Index: 1 -; PASSIVE-MERGE-NEXT: Name: __wasm_init_memory -; PASSIVE-MERGE-NEXT: - Index: 2 ; PASSIVE-MERGE-NEXT: Name: __wasm_init_tls +; PASSIVE-MERGE-NEXT: - Index: 2 +; PASSIVE-MERGE-NEXT: Name: __wasm_init_memory ; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 -no-merge-data-segments -o %t.separate.passive.wasm %t.passive.o ; RUN: obj2yaml %t.separate.passive.wasm | FileCheck %s --check-prefix=PASSIVE-SEPARATE @@ -135,6 +135,6 @@ ; PASSIVE-SEPARATE-NEXT: - Index: 0 ; PASSIVE-SEPARATE-NEXT: Name: __wasm_call_ctors ; PASSIVE-SEPARATE-NEXT: - Index: 1 -; PASSIVE-SEPARATE-NEXT: Name: __wasm_init_memory -; PASSIVE-SEPARATE-NEXT: - Index: 2 ; PASSIVE-SEPARATE-NEXT: Name: __wasm_init_tls +; PASSIVE-SEPARATE-NEXT: - Index: 2 +; PASSIVE-SEPARATE-NEXT: Name: __wasm_init_memory diff --git a/lld/test/wasm/data-segments.ll b/lld/test/wasm/data-segments.ll --- a/lld/test/wasm/data-segments.ll +++ b/lld/test/wasm/data-segments.ll @@ -64,7 +64,7 @@ ; ACTIVE-NEXT: Name: __wasm_call_ctors ; PASSIVE-LABEL: - Type: START -; PASSIVE-NEXT: StartFunction: 1 +; PASSIVE-NEXT: StartFunction: 2 ; PASSIVE-LABEL: - Type: DATACOUNT ; PASSIVE-NEXT: Count: 2 ; PASSIVE-LABEL: - Type: CODE @@ -74,12 +74,11 @@ ; PASSIVE-NEXT: Body: 0B ; PASSIVE-NEXT: - Index: 1 ; PASSIVE-NEXT: Locals: [] +; PASSIVE-NEXT: Body: 0B +; PASSIVE-NEXT: - Index: 2 +; PASSIVE-NEXT: Locals: [] ; PASSIVE32-NEXT: Body: 41B4D60041004101FE480200044041B4D6004101427FFE0102001A054180084100410DFC08000041900841004114FC08010041B4D6004102FE17020041B4D600417FFE0002001A0BFC0900FC09010B ; PASSIVE64-NEXT: Body: 42B4D60041004101FE480200044042B4D6004101427FFE0102001A054280084100410DFC08000042900841004114FC08010042B4D6004102FE17020042B4D600417FFE0002001A0BFC0900FC09010B - -; PASSIVE-NEXT: - Index: 2 -; PASSIVE-NEXT: Locals: [] -; PASSIVE-NEXT: Body: 0B ; PASSIVE-NEXT: - Type: DATA ; PASSIVE-NEXT: Segments: ; PASSIVE-NEXT: - SectionOffset: 3 @@ -94,12 +93,12 @@ ; PASSIVE-NEXT: - Index: 0 ; PASSIVE-NEXT: Name: __wasm_call_ctors ; PASSIVE-NEXT: - Index: 1 -; PASSIVE-NEXT: Name: __wasm_init_memory -; PASSIVE-NEXT: - Index: 2 ; PASSIVE-NEXT: Name: __wasm_init_tls +; PASSIVE-NEXT: - Index: 2 +; PASSIVE-NEXT: Name: __wasm_init_memory ; PASSIVE-PIC: - Type: START -; PASSIVE-PIC-NEXT: StartFunction: 2 +; PASSIVE-PIC-NEXT: StartFunction: 3 ; PASSIVE-PIC-NEXT: - Type: DATACOUNT ; PASSIVE-PIC-NEXT: Count: 1 ; PASSIVE-PIC-NEXT: - Type: CODE @@ -111,15 +110,15 @@ ; PASSIVE-PIC-NEXT: Locals: [] ; PASSIVE-PIC-NEXT: Body: 0B ; PASSIVE-PIC-NEXT: - Index: 2 +; PASSIVE-PIC-NEXT: Locals: [] +; PASSIVE-PIC-NEXT: Body: 0B +; PASSIVE-PIC-NEXT: - Index: 3 ; PASSIVE-PIC-NEXT: Locals: ; PASSIVE32-PIC-NEXT: - Type: I32 ; PASSIVE64-PIC-NEXT: - Type: I64 ; PASSIVE-PIC-NEXT: Count: 1 ; PASSIVE32-PIC-NEXT: Body: 230141B4CE006A2100200041004101FE480200044020004101427FFE0102001A05410023016A410041B1CE00FC08000020004102FE1702002000417FFE0002001A0BFC09000B ; PASSIVE64-PIC-NEXT: Body: 230142B4CE006A2100200041004101FE480200044020004101427FFE0102001A05420023016A410041B1CE00FC08000020004102FE1702002000417FFE0002001A0BFC09000B -; PASSIVE-PIC-NEXT: - Index: 3 -; PASSIVE-PIC-NEXT: Locals: [] -; PASSIVE-PIC-NEXT: Body: 0B ; PASSIVE-PIC-NEXT: - Type: DATA ; PASSIVE-PIC-NEXT: Segments: ; PASSIVE-PIC-NEXT: - SectionOffset: 4 @@ -133,6 +132,6 @@ ; PASSIVE-PIC-NEXT: - Index: 1 ; PASSIVE-PIC-NEXT: Name: __wasm_apply_relocs ; PASSIVE-PIC-NEXT: - Index: 2 -; PASSIVE-PIC-NEXT: Name: __wasm_init_memory -; PASSIVE-PIC-NEXT: - Index: 3 ; PASSIVE-PIC-NEXT: Name: __wasm_init_tls +; PASSIVE-PIC-NEXT: - Index: 3 +; PASSIVE-PIC-NEXT: Name: __wasm_init_memory diff --git a/lld/test/wasm/no-tls.s b/lld/test/wasm/no-tls.s --- a/lld/test/wasm/no-tls.s +++ b/lld/test/wasm/no-tls.s @@ -28,7 +28,7 @@ # CHECK-NEXT: Mutable: true # CHECK-NEXT: InitExpr: # CHECK-NEXT: Opcode: I32_CONST -# CHECK-NEXT: Value: 66576 +# CHECK-NEXT: Value: 66560 # __tls_base # CHECK-NEXT: - Index: 1 diff --git a/lld/test/wasm/tls.s b/lld/test/wasm/tls.s --- a/lld/test/wasm/tls.s +++ b/lld/test/wasm/tls.s @@ -83,7 +83,7 @@ # CHECK-NEXT: Mutable: true # CHECK-NEXT: InitExpr: # CHECK-NEXT: Opcode: I32_CONST -# CHECK-NEXT: Value: 66592 +# CHECK-NEXT: Value: 66576 # __tls_base # CHECK-NEXT: - Index: 1 @@ -112,8 +112,8 @@ # CHECK: - Type: CODE # CHECK-NEXT: Functions: -# Skip __wasm_call_ctors and __wasm_init_memory -# CHECK: - Index: 2 +# Skip __wasm_call_ctors +# CHECK: - Index: 1 # CHECK-NEXT: Locals: [] # CHECK-NEXT: Body: 2000240120004100410CFC0800000B @@ -126,7 +126,7 @@ # memory.init 1, 0 # end -# CHECK-NEXT: - Index: 3 +# CHECK-NEXT: - Index: 2 # CHECK-NEXT: Locals: [] # CHECK-NEXT: Body: 2381808080004180808080006A0B @@ -136,7 +136,7 @@ # i32.add # end -# CHECK-NEXT: - Index: 4 +# CHECK-NEXT: - Index: 3 # CHECK-NEXT: Locals: [] # CHECK-NEXT: Body: 2381808080004184808080006A0B @@ -146,7 +146,7 @@ # i32.add # end -# CHECK-NEXT: - Index: 5 +# CHECK-NEXT: - Index: 4 # CHECK-NEXT: Locals: [] # CHECK-NEXT: Body: 2381808080004188808080006A0B @@ -156,7 +156,7 @@ # i32.add # end -# CHECK-NEXT: - Index: 6 +# CHECK-NEXT: - Index: 5 # CHECK-NEXT: Locals: [] # CHECK-NEXT: Body: 2383808080000B diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp --- a/lld/wasm/Driver.cpp +++ b/lld/wasm/Driver.cpp @@ -646,15 +646,6 @@ } if (config->sharedMemory) { - // Passive segments are used to avoid memory being reinitialized on each - // thread's instantiation. These passive segments are initialized and - // dropped in __wasm_init_memory, which is registered as the start function - WasmSym::initMemory = symtab->addSyntheticFunction( - "__wasm_init_memory", WASM_SYMBOL_VISIBILITY_HIDDEN, - make(nullSignature, "__wasm_init_memory")); - WasmSym::initMemoryFlag = symtab->addSyntheticDataSymbol( - "__wasm_init_memory_flag", WASM_SYMBOL_VISIBILITY_HIDDEN); - assert(WasmSym::initMemoryFlag); WasmSym::tlsBase = createGlobalVariable("__tls_base", true); WasmSym::tlsSize = createGlobalVariable("__tls_size", false); WasmSym::tlsAlign = createGlobalVariable("__tls_align", false); diff --git a/lld/wasm/MarkLive.cpp b/lld/wasm/MarkLive.cpp --- a/lld/wasm/MarkLive.cpp +++ b/lld/wasm/MarkLive.cpp @@ -101,9 +101,6 @@ if (WasmSym::applyRelocs) enqueue(WasmSym::applyRelocs); - if (WasmSym::initMemory) - enqueue(WasmSym::initMemory); - // Enqueue constructors in objects explicitly live from the command-line. for (const ObjFile *obj : symtab->objectFiles) if (obj->isLive()) diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h --- a/lld/wasm/SyntheticSections.h +++ b/lld/wasm/SyntheticSections.h @@ -239,14 +239,9 @@ class StartSection : public SyntheticSection { public: - StartSection(bool hasInitializedSegments) - : SyntheticSection(llvm::wasm::WASM_SEC_START), - hasInitializedSegments(hasInitializedSegments) {} + StartSection() : SyntheticSection(llvm::wasm::WASM_SEC_START) {} bool isNeeded() const override; void writeBody() override; - -protected: - bool hasInitializedSegments; }; class ElemSection : public SyntheticSection { diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -58,6 +58,7 @@ bool needsPassiveInitialization(const OutputSegment *segment); bool hasPassiveInitializedSegments(); + void createSyntheticInitFunctions(); void createInitMemoryFunction(); void createApplyRelocationsFunction(); void createCallCtorsFunction(); @@ -865,14 +866,81 @@ }) != segments.end(); } +void Writer::createSyntheticInitFunctions() { + // Passive segments are used to avoid memory being reinitialized on each + // thread's instantiation. These passive segments are initialized and + // dropped in __wasm_init_memory, which is registered as the start function + + if (config->sharedMemory && hasPassiveInitializedSegments()) { + static WasmSignature nullSignature = {{}, {}}; + WasmSym::initMemory = symtab->addSyntheticFunction( + "__wasm_init_memory", WASM_SYMBOL_VISIBILITY_HIDDEN, + make(nullSignature, "__wasm_init_memory")); + WasmSym::initMemory->markLive(); + WasmSym::initMemoryFlag = symtab->addSyntheticDataSymbol( + "__wasm_init_memory_flag", WASM_SYMBOL_VISIBILITY_HIDDEN); + WasmSym::initMemoryFlag->markLive(); + } +} + void Writer::createInitMemoryFunction() { LLVM_DEBUG(dbgs() << "createInitMemoryFunction\n"); + assert(WasmSym::initMemory); assert(WasmSym::initMemoryFlag); + assert(hasPassiveInitializedSegments()); uint64_t flagAddress = WasmSym::initMemoryFlag->getVirtualAddress(); bool is64 = config->is64.getValueOr(false); std::string bodyContent; { raw_string_ostream os(bodyContent); + // Initialize memory in a thread-safe manner. The thread that successfully + // increments the flag from 0 to 1 is is responsible for performing the + // memory initialization. Other threads go sleep on the flag until the + // first thread finishing initializing memory, increments the flag to 2, + // and wakes all the other threads. Once the flag has been set to 2, + // subsequently started threads will skip the sleep. All threads + // unconditionally drop their passive data segments once memory has been + // initialized. The generated code is as follows: + // + // (func $__wasm_init_memory + // (if + // (i32.atomic.rmw.cmpxchg align=2 offset=0 + // (i32.const $__init_memory_flag) + // (i32.const 0) + // (i32.const 1) + // ) + // (then + // (drop + // (i32.atomic.wait align=2 offset=0 + // (i32.const $__init_memory_flag) + // (i32.const 1) + // (i32.const -1) + // ) + // ) + // ) + // (else + // ( ... initialize data segments ... ) + // (i32.atomic.store align=2 offset=0 + // (i32.const $__init_memory_flag) + // (i32.const 2) + // ) + // (drop + // (i32.atomic.notify align=2 offset=0 + // (i32.const $__init_memory_flag) + // (i32.const -1u) + // ) + // ) + // ) + // ) + // ( ... drop data segments ... ) + // ) + // + // When we are building with PIC, calculate the flag location using: + // + // (global.get $__memory_base) + // (i32.const $__init_memory_flag) + // (i32.const 1) + // With PIC code we cache the flag address in local 0 if (config->isPic) { writeUleb128(os, 1, "num local decls"); @@ -888,134 +956,84 @@ writeUleb128(os, 0, "num locals"); } - if (hasPassiveInitializedSegments()) { - // Initialize memory in a thread-safe manner. The thread that successfully - // increments the flag from 0 to 1 is is responsible for performing the - // memory initialization. Other threads go sleep on the flag until the - // first thread finishing initializing memory, increments the flag to 2, - // and wakes all the other threads. Once the flag has been set to 2, - // subsequently started threads will skip the sleep. All threads - // unconditionally drop their passive data segments once memory has been - // initialized. The generated code is as follows: - // - // (func $__wasm_init_memory - // (if - // (i32.atomic.rmw.cmpxchg align=2 offset=0 - // (i32.const $__init_memory_flag) - // (i32.const 0) - // (i32.const 1) - // ) - // (then - // (drop - // (i32.atomic.wait align=2 offset=0 - // (i32.const $__init_memory_flag) - // (i32.const 1) - // (i32.const -1) - // ) - // ) - // ) - // (else - // ( ... initialize data segments ... ) - // (i32.atomic.store align=2 offset=0 - // (i32.const $__init_memory_flag) - // (i32.const 2) - // ) - // (drop - // (i32.atomic.notify align=2 offset=0 - // (i32.const $__init_memory_flag) - // (i32.const -1u) - // ) - // ) - // ) - // ) - // ( ... drop data segments ... ) - // ) - // - // When we are building with PIC, calculate the flag location using: - // - // (global.get $__memory_base) - // (i32.const $__init_memory_flag) - // (i32.const 1) - - auto writeGetFlagAddress = [&]() { + auto writeGetFlagAddress = [&]() { + if (config->isPic) { + writeU8(os, WASM_OPCODE_LOCAL_GET, "local.get"); + writeUleb128(os, 0, "local 0"); + } else { + writePtrConst(os, flagAddress, is64, "flag address"); + } + }; + + // Atomically check whether this is the main thread. + writeGetFlagAddress(); + writeI32Const(os, 0, "expected flag value"); + writeI32Const(os, 1, "flag value"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_RMW_CMPXCHG, "i32.atomic.rmw.cmpxchg"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_IF, "IF"); + writeU8(os, WASM_TYPE_NORESULT, "blocktype"); + + // Did not increment 0, so wait for main thread to initialize memory + writeGetFlagAddress(); + writeI32Const(os, 1, "expected flag value"); + writeI64Const(os, -1, "timeout"); + + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_ATOMIC_WAIT, "i32.atomic.wait"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_DROP, "drop"); + + writeU8(os, WASM_OPCODE_ELSE, "ELSE"); + + // Did increment 0, so conditionally initialize passive data segments + for (const OutputSegment *s : segments) { + if (needsPassiveInitialization(s)) { + // destination address + writePtrConst(os, s->startVA, is64, "destination address"); if (config->isPic) { - writeU8(os, WASM_OPCODE_LOCAL_GET, "local.get"); - writeUleb128(os, 0, "local 0"); - } else { - writePtrConst(os, flagAddress, is64, "flag address"); - } - }; - - // Atomically check whether this is the main thread. - writeGetFlagAddress(); - writeI32Const(os, 0, "expected flag value"); - writeI32Const(os, 1, "flag value"); - writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); - writeUleb128(os, WASM_OPCODE_I32_RMW_CMPXCHG, "i32.atomic.rmw.cmpxchg"); - writeMemArg(os, 2, 0); - writeU8(os, WASM_OPCODE_IF, "IF"); - writeU8(os, WASM_TYPE_NORESULT, "blocktype"); - - // Did not increment 0, so wait for main thread to initialize memory - writeGetFlagAddress(); - writeI32Const(os, 1, "expected flag value"); - writeI64Const(os, -1, "timeout"); - - writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); - writeUleb128(os, WASM_OPCODE_I32_ATOMIC_WAIT, "i32.atomic.wait"); - writeMemArg(os, 2, 0); - writeU8(os, WASM_OPCODE_DROP, "drop"); - - writeU8(os, WASM_OPCODE_ELSE, "ELSE"); - - // Did increment 0, so conditionally initialize passive data segments - for (const OutputSegment *s : segments) { - if (needsPassiveInitialization(s)) { - // destination address - writePtrConst(os, s->startVA, is64, "destination address"); - if (config->isPic) { - writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET"); - writeUleb128(os, WasmSym::memoryBase->getGlobalIndex(), - "memory_base"); - writeU8(os, WASM_OPCODE_I32_ADD, "i32.add"); - } - // source segment offset - writeI32Const(os, 0, "segment offset"); - // memory region size - writeI32Const(os, s->size, "memory region size"); - // memory.init instruction - writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); - writeUleb128(os, WASM_OPCODE_MEMORY_INIT, "memory.init"); - writeUleb128(os, s->index, "segment index immediate"); - writeU8(os, 0, "memory index immediate"); + writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET"); + writeUleb128(os, WasmSym::memoryBase->getGlobalIndex(), + "memory_base"); + writeU8(os, WASM_OPCODE_I32_ADD, "i32.add"); } + // source segment offset + writeI32Const(os, 0, "segment offset"); + // memory region size + writeI32Const(os, s->size, "memory region size"); + // memory.init instruction + writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); + writeUleb128(os, WASM_OPCODE_MEMORY_INIT, "memory.init"); + writeUleb128(os, s->index, "segment index immediate"); + writeU8(os, 0, "memory index immediate"); } + } - // Set flag to 2 to mark end of initialization - writeGetFlagAddress(); - writeI32Const(os, 2, "flag value"); - writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); - writeUleb128(os, WASM_OPCODE_I32_ATOMIC_STORE, "i32.atomic.store"); - writeMemArg(os, 2, 0); - - // Notify any waiters that memory initialization is complete - writeGetFlagAddress(); - writeI32Const(os, -1, "number of waiters"); - writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); - writeUleb128(os, WASM_OPCODE_ATOMIC_NOTIFY, "atomic.notify"); - writeMemArg(os, 2, 0); - writeU8(os, WASM_OPCODE_DROP, "drop"); - - writeU8(os, WASM_OPCODE_END, "END"); - - // Unconditionally drop passive data segments - for (const OutputSegment *s : segments) { - if (needsPassiveInitialization(s)) { - // data.drop instruction - writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); - writeUleb128(os, WASM_OPCODE_DATA_DROP, "data.drop"); - writeUleb128(os, s->index, "segment index immediate"); - } + // Set flag to 2 to mark end of initialization + writeGetFlagAddress(); + writeI32Const(os, 2, "flag value"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_ATOMIC_STORE, "i32.atomic.store"); + writeMemArg(os, 2, 0); + + // Notify any waiters that memory initialization is complete + writeGetFlagAddress(); + writeI32Const(os, -1, "number of waiters"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_ATOMIC_NOTIFY, "atomic.notify"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_DROP, "drop"); + + writeU8(os, WASM_OPCODE_END, "END"); + + // Unconditionally drop passive data segments + for (const OutputSegment *s : segments) { + if (needsPassiveInitialization(s)) { + // data.drop instruction + writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix"); + writeUleb128(os, WASM_OPCODE_DATA_DROP, "data.drop"); + writeUleb128(os, s->index, "segment index immediate"); } } writeU8(os, WASM_OPCODE_END, "END"); @@ -1206,7 +1224,7 @@ out.eventSec = make(); out.globalSec = make(); out.exportSec = make(); - out.startSec = make(hasPassiveInitializedSegments()); + out.startSec = make(); out.elemSec = make(); out.dataCountSec = make(segments); out.linkingSec = make(initFunctions, segments); @@ -1235,6 +1253,8 @@ populateProducers(); log("-- calculateImports"); calculateImports(); + log("-- createSyntheticInitFunctions"); + createSyntheticInitFunctions(); log("-- layoutMemory"); layoutMemory();