diff --git a/lld/test/wasm/init-fini-gc.ll b/lld/test/wasm/init-fini-gc.ll new file mode 100644 --- /dev/null +++ b/lld/test/wasm/init-fini-gc.ll @@ -0,0 +1,84 @@ +; RUN: llc -filetype=obj -o %t.o %s +; RUN: wasm-ld %t.o -o %t.wasm +; RUN: obj2yaml %t.wasm | FileCheck %s + +; RUN: wasm-ld --export=__wasm_call_ctors %t.o -o %t.export.wasm +; RUN: obj2yaml %t.export.wasm | FileCheck %s -check-prefix=EXPORT + +; Test that we don't emit wrappers or call __wasm_call_ctor even when not +; referenced. + +target triple = "wasm32-unknown-unknown" + +define hidden void @_start() { +entry: + ret void +} + +define hidden void @func1() { +entry: + ret void +} + +define hidden void @func2() { +entry: + ret void +} + +define hidden i32 @__cxa_atexit(i32 %func, i32 %arg, i32 %dso_handle) { + ret i32 0 +} + +@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [ + { i32, void ()*, i8* } { i32 1, void ()* @func1, i8* null } +] + +@llvm.global_dtors = appending global [1 x { i32, void ()*, i8* }] [ + { i32, void ()*, i8* } { i32 1, void ()* @func2, i8* null } +] + +; Check that we have exactly the needed exports: `memory` because that's +; currently on by default, and `_start`, because that's the default entrypoint. + +; CHECK: - Type: EXPORT +; CHECK-NEXT: Exports: +; CHECK-NEXT: - Name: memory +; CHECK-NEXT: Kind: MEMORY +; CHECK-NEXT: Index: 0 +; CHECK-NEXT: - Name: _start +; CHECK-NEXT: Kind: FUNCTION +; CHECK-NEXT: Index: 1 + +; Check the body of `_start`'s command-export wrapper. + +; CHECK: - Type: CODE + +; CHECK: - Index: 1 +; CHECK-NEXT: Locals: [] +; CHECK-NEXT: Body: 0B + +; Check the symbol table to ensure all the functions are here, and that +; index 7 above refers to the function we think it does. + +; CHECK: - Type: CUSTOM +; CHECK-NEXT: Name: name +; CHECK-NEXT: FunctionNames: +; CHECK-NEXT: - Index: 0 +; CHECK-NEXT: Name: __wasm_call_ctors +; CHECK-NEXT: - Index: 1 +; CHECK-NEXT: Name: _start +; CHECK-NEXT: - Index: 2 +; CHECK-NEXT: Name: func1 +; CHECK-NEXT: - Index: 3 +; CHECK-NEXT: Name: func2 +; CHECK-NEXT: - Index: 4 +; CHECK-NEXT: Name: __cxa_atexit +; CHECK-NEXT: - Index: 5 +; CHECK-NEXT: Name: .Lcall_dtors.1 +; CHECK-NEXT: - Index: 6 +; CHECK-NEXT: Name: .Lregister_call_dtors.1 + +; EXPORT: __wasm_call_ctors +; EXPORT: func1 +; EXPORT: func2 +; EXPORT: __cxa_atexit diff --git a/lld/test/wasm/init-fini-no-gc.ll b/lld/test/wasm/init-fini-no-gc.ll --- a/lld/test/wasm/init-fini-no-gc.ll +++ b/lld/test/wasm/init-fini-no-gc.ll @@ -1,15 +1,17 @@ ; RUN: llc -filetype=obj -o %t.o %s -; RUN: wasm-ld %t.o -o %t.wasm +; RUN: wasm-ld --no-entry --export=export_func %t.o -o %t.wasm +; RUN: llvm-objdump -d --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck --check-prefix=ASM %s -- ; RUN: obj2yaml %t.wasm | FileCheck %s -; RUN: wasm-ld --export=__wasm_call_ctors %t.o -o %t.export.wasm -; RUN: obj2yaml %t.export.wasm | FileCheck %s -check-prefix=EXPORT +; RUN: llc -mattr=+atomics,+bulk-memory -filetype=obj -o %t.o %s +; RUN: wasm-ld --shared-memory --no-entry --export=export_func %t.o -o %t.wasm +; RUN: llvm-objdump -d --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck --check-prefix=SHARED-ASM %s -- -; Test that we emit wrappers and call __wasm_call_ctor when not referenced. +; Test that we emit wrappers which call __wasm_call_ctor. target triple = "wasm32-unknown-unknown" -define hidden void @_start() { +define hidden void @export_func() { entry: ret void } @@ -37,24 +39,24 @@ ] ; Check that we have exactly the needed exports: `memory` because that's -; currently on by default, and `_start`, because that's the default entrypoint. +; currently on by default, and `export_func`, because we asked for it. ; CHECK: - Type: EXPORT ; CHECK-NEXT: Exports: ; CHECK-NEXT: - Name: memory ; CHECK-NEXT: Kind: MEMORY ; CHECK-NEXT: Index: 0 -; CHECK-NEXT: - Name: _start +; CHECK-NEXT: - Name: export_func ; CHECK-NEXT: Kind: FUNCTION ; CHECK-NEXT: Index: 7 -; Check the body of `_start`'s command-export wrapper. +; Check the body of `export_func`'s reactor-export wrapper. ; CHECK: - Type: CODE ; CHECK: - Index: 7 ; CHECK-NEXT: Locals: [] -; CHECK-NEXT: Body: 100010010B +; CHECK-NEXT: Body: 02404180082802000D00100041800841013602000B10010B ; Check the symbol table to ensure all the functions are here, and that ; index 7 above refers to the function we think it does. @@ -65,7 +67,7 @@ ; CHECK-NEXT: - Index: 0 ; CHECK-NEXT: Name: __wasm_call_ctors ; CHECK-NEXT: - Index: 1 -; CHECK-NEXT: Name: _start +; CHECK-NEXT: Name: export_func ; CHECK-NEXT: - Index: 2 ; CHECK-NEXT: Name: func1 ; CHECK-NEXT: - Index: 3 @@ -77,9 +79,53 @@ ; CHECK-NEXT: - Index: 6 ; CHECK-NEXT: Name: .Lregister_call_dtors.1 ; CHECK-NEXT: - Index: 7 -; CHECK-NEXT: Name: _start.command_export - -; EXPORT: __wasm_call_ctors -; EXPORT: func1 -; EXPORT: func2 -; EXPORT: __cxa_atexit +; CHECK-NEXT: Name: export_func.reactor_export + +; Check the body of the generated wrapper. + +; ASM: : +; ASM-EMPTY: +; ASM-NEXT: block +; ASM-NEXT: i32.const 1024 +; ASM-NEXT: i32.load 0 +; ASM-NEXT: br_if 0 # 0: down to label1 +; ASM-NEXT: call 0 +; ASM-NEXT: i32.const 1024 +; ASM-NEXT: i32.const 1 +; ASM-NEXT: i32.store 0 +; ASM-NEXT: end +; ASM-NEXT: call 1 +; ASM-NEXT: end + +; Check the body of the generated wrapper in shared-memory mode. + +; SHARED-ASM: : +; SHARED-ASM-EMPTY: +; SHARED-ASM-NEXT: block +; SHARED-ASM-NEXT: block +; SHARED-ASM-NEXT: block +; SHARED-ASM-NEXT: i32.const 1024 +; SHARED-ASM-NEXT: i32.const 0 +; SHARED-ASM-NEXT: i32.const 1 +; SHARED-ASM-NEXT: i32.atomic.rmw.cmpxchg 0 +; SHARED-ASM-NEXT: br_table {0, 1, 2} # 1: down to label2 +; SHARED-ASM-NEXT: # 2: down to label1 +; SHARED-ASM-NEXT: end +; SHARED-ASM-NEXT: call 0 +; SHARED-ASM-NEXT: i32.const 1024 +; SHARED-ASM-NEXT: i32.const 2 +; SHARED-ASM-NEXT: i32.atomic.store 0 +; SHARED-ASM-NEXT: i32.const 1024 +; SHARED-ASM-NEXT: i32.const -1 +; SHARED-ASM-NEXT: memory.atomic.notify 0 +; SHARED-ASM-NEXT: drop +; SHARED-ASM-NEXT: br 1 # 1: down to label2 +; SHARED-ASM-NEXT: end +; SHARED-ASM-NEXT: i32.const 1024 +; SHARED-ASM-NEXT: i32.const 1 +; SHARED-ASM-NEXT: i64.const -1 +; SHARED-ASM-NEXT: memory.atomic.wait32 0 +; SHARED-ASM-NEXT: drop +; SHARED-ASM-NEXT: end +; SHARED-ASM-NEXT: call 1 +; SHARED-ASM-NEXT: end diff --git a/lld/test/wasm/command-exports-no-tors.s b/lld/test/wasm/reactor-exports-no-tors.s rename from lld/test/wasm/command-exports-no-tors.s rename to lld/test/wasm/reactor-exports-no-tors.s --- a/lld/test/wasm/command-exports-no-tors.s +++ b/lld/test/wasm/reactor-exports-no-tors.s @@ -2,7 +2,7 @@ # RUN: wasm-ld --no-entry %t.o -o %t.wasm # RUN: obj2yaml %t.wasm | FileCheck %s -# Like command-exports.s, but with no ctors or dtors, so there should be no +# Like reactor-exports.s, but with no ctors or dtors, so there should be no # __wasm_call_ctors, __cxa_atexit, or wrappers. .globl foo_i32 diff --git a/lld/test/wasm/command-exports.s b/lld/test/wasm/reactor-exports.s rename from lld/test/wasm/command-exports.s rename to lld/test/wasm/reactor-exports.s --- a/lld/test/wasm/command-exports.s +++ b/lld/test/wasm/reactor-exports.s @@ -1,8 +1,9 @@ # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s # RUN: wasm-ld --no-entry %t.o -o %t.wasm +# RUN: llvm-objdump -d --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck --check-prefix=ASM %s -- # RUN: obj2yaml %t.wasm | FileCheck %s -# This test defines a command with two exported functions, as well as a static +# This test defines a reactor with two exported functions, as well as a static # constructor and a static destructor. Check that the exports, constructor, and # destructor are all set up properly. @@ -83,10 +84,10 @@ # CHECK: - Index: 8 # CHECK-NEXT: Locals: [] -# CHECK-NEXT: Body: 10002000200110010B +# CHECK-NEXT: Body: 02404180082802000D00100041800841013602000B2000200110010B # CHECK-NEXT: - Index: 9 # CHECK-NEXT: Locals: [] -# CHECK-NEXT: Body: 10002000200110020B +# CHECK-NEXT: Body: 02404180082802000D00100041800841013602000B2000200110020B # CHECK: - Type: CUSTOM # CHECK-NEXT: Name: name @@ -108,6 +109,40 @@ # CHECK-NEXT: - Index: 7 # CHECK-NEXT: Name: .Lregister_call_dtors.1 # CHECK-NEXT: - Index: 8 -# CHECK-NEXT: Name: foo_i32.command_export +# CHECK-NEXT: Name: foo_i32.reactor_export # CHECK-NEXT: - Index: 9 -# CHECK-NEXT: Name: foo_f64.command_export +# CHECK-NEXT: Name: foo_f64.reactor_export + +# Check the body of the generated wrapper. + +# ASM: : +# ASM-EMPTY: +# ASM-NEXT: block +# ASM-NEXT: i32.const 1024 +# ASM-NEXT: i32.load 0 +# ASM-NEXT: br_if 0 # 0: down to label1 +# ASM-NEXT: call 0 +# ASM-NEXT: i32.const 1024 +# ASM-NEXT: i32.const 1 +# ASM-NEXT: i32.store 0 +# ASM-NEXT: end +# ASM-NEXT: local.get 0 +# ASM-NEXT: local.get 1 +# ASM-NEXT: call 1 +# ASM-NEXT: end + +# ASM: : +# ASM-EMPTY: +# ASM-NEXT: block +# ASM-NEXT: i32.const 1024 +# ASM-NEXT: i32.load 0 +# ASM-NEXT: br_if 0 # 0: down to label2 +# ASM-NEXT: call 0 +# ASM-NEXT: i32.const 1024 +# ASM-NEXT: i32.const 1 +# ASM-NEXT: i32.store 0 +# ASM-NEXT: end +# ASM-NEXT: local.get 0 +# ASM-NEXT: local.get 1 +# ASM-NEXT: call 2 +# ASM-NEXT: end diff --git a/lld/wasm/Symbols.h b/lld/wasm/Symbols.h --- a/lld/wasm/Symbols.h +++ b/lld/wasm/Symbols.h @@ -548,6 +548,10 @@ // Symbol whose contents are nonzero iff memory has already been initialized. static DefinedData *initMemoryFlag; + // __ctors_done + // Symbol whose value is non-zero if the global ctors have been called. + static DefinedData *ctorsDone; + // __wasm_init_memory // Function that initializes passive data segments during instantiation. static DefinedFunction *initMemory; diff --git a/lld/wasm/Symbols.cpp b/lld/wasm/Symbols.cpp --- a/lld/wasm/Symbols.cpp +++ b/lld/wasm/Symbols.cpp @@ -84,6 +84,7 @@ DefinedData *WasmSym::globalBase; DefinedData *WasmSym::heapBase; DefinedData *WasmSym::initMemoryFlag; +DefinedData *WasmSym::ctorsDone; GlobalSymbol *WasmSym::stackPointer; GlobalSymbol *WasmSym::tlsBase; GlobalSymbol *WasmSym::tlsSize; diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -65,8 +65,8 @@ void createApplyGlobalTLSRelocationsFunction(); void createCallCtorsFunction(); void createInitTLSFunction(); - void createCommandExportWrappers(); - void createCommandExportWrapper(uint32_t functionIndex, DefinedFunction *f); + void createReactorExportWrappers(); + void createReactorExportWrapper(uint32_t functionIndex, DefinedFunction *f); void assignIndexes(); void populateSymtab(); @@ -107,8 +107,8 @@ std::vector initFunctions; llvm::StringMap> customSectionMapping; - // Stable storage for command export wrapper function name strings. - std::list commandExportWrapperNames; + // Stable storage for reactor export wrapper function name strings. + std::list reactorExportWrapperNames; // Elements that are used to construct the final output std::string header; @@ -306,6 +306,34 @@ memoryPtr += 4; } + // If we need a `__ctors_done` flag, create one. + if (!config->relocatable && !config->isPic && config->entry.empty() && + !WasmSym::callCtors->isUsedInRegularObj && + !WasmSym::callCtors->isExported()) { + bool needCtorsDone = false; + for (ObjFile *file : symtab->objectFiles) { + const WasmLinkingData &l = file->getWasmObj()->linkingData(); + for (const WasmInitFunc &f : l.InitFunctions) { + FunctionSymbol *sym = file->getFunctionSymbol(f.Symbol); + // comdat exclusions can cause init functions be discarded. + if (sym->isDiscarded() || !sym->isLive()) + continue; + needCtorsDone = true; + break; + } + } + if (needCtorsDone) { + memoryPtr = alignTo(memoryPtr, 4); + WasmSym::ctorsDone = symtab->addSyntheticDataSymbol( + "__ctors_done", WASM_SYMBOL_VISIBILITY_HIDDEN); + WasmSym::ctorsDone->markLive(); + WasmSym::ctorsDone->setVA(memoryPtr); + log(formatv("mem: {0,-15} offset={1,-8} size={2,-8} align={3}", + "__ctors_done", memoryPtr, 4, 4)); + memoryPtr += 4; + } + } + if (WasmSym::dataEnd) WasmSym::dataEnd->setVA(memoryPtr); @@ -735,15 +763,14 @@ out.typeSec->registerType(t->signature); } -// In a command-style link, create a wrapper for each exported symbol +// In a reactor-style link, create a wrapper for each exported symbol // which calls the constructors and destructors. -void Writer::createCommandExportWrappers() { +void Writer::createReactorExportWrappers() { // This logic doesn't currently support Emscripten-style PIC mode. assert(!config->isPic); - // If there are no ctors and there's no libc `__wasm_call_dtors` to - // call, don't wrap the exports. - if (initFunctions.empty() && WasmSym::callDtors == nullptr) + // If there are no ctors, don't wrap the exports. + if (initFunctions.empty()) return; std::vector toWrap; @@ -754,9 +781,9 @@ toWrap.push_back(f); for (auto *f : toWrap) { - auto funcNameStr = (f->getName() + ".command_export").str(); - commandExportWrapperNames.push_back(funcNameStr); - const std::string &funcName = commandExportWrapperNames.back(); + auto funcNameStr = (f->getName() + ".reactor_export").str(); + reactorExportWrapperNames.push_back(funcNameStr); + const std::string &funcName = reactorExportWrapperNames.back(); auto func = make(*f->getSignature(), funcName); if (f->function->getExportName()) @@ -778,7 +805,7 @@ out.functionSec->addFunction(func); - createCommandExportWrapper(f->getFunctionIndex(), def); + createReactorExportWrapper(f->getFunctionIndex(), def); } } @@ -1374,7 +1401,7 @@ // Create a wrapper around a function export which calls the // static constructors and destructors. -void Writer::createCommandExportWrapper(uint32_t functionIndex, +void Writer::createReactorExportWrapper(uint32_t functionIndex, DefinedFunction *f) { // First write the body's contents to a string. std::string bodyContent; @@ -1385,9 +1412,91 @@ // Call `__wasm_call_ctors` which call static constructors (and // applies any runtime relocations in Emscripten-style PIC mode) if (WasmSym::callCtors->isLive()) { + bool is64 = config->is64.value_or(false); + uint64_t flagAddress = WasmSym::ctorsDone->getVA(); + if (config->sharedMemory) { + // Set up destination blocks + writeU8(os, WASM_OPCODE_BLOCK, "block $done"); + writeU8(os, WASM_TYPE_NORESULT, "block type"); + writeU8(os, WASM_OPCODE_BLOCK, "block $wait"); + writeU8(os, WASM_TYPE_NORESULT, "block type"); + writeU8(os, WASM_OPCODE_BLOCK, "block $init"); + writeU8(os, WASM_TYPE_NORESULT, "block type"); + + // Atomically check whether we win the race. + writePtrConst(os, flagAddress, is64, "flag address"); + writeI32Const(os, 0, "expected flag value"); + writeI32Const(os, 1, "new flag value"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_RMW_CMPXCHG, "i32.atomic.rmw.cmpxchg"); + writeMemArg(os, 2, 0); + + // Based on the value, decide what to do next. + writeU8(os, WASM_OPCODE_BR_TABLE, "br_table"); + writeUleb128(os, 2, "label vector length"); + writeUleb128(os, 0, "label $init"); + writeUleb128(os, 1, "label $wait"); + writeUleb128(os, 2, "default label $done"); + + writeU8(os, WASM_OPCODE_END, "end $init"); + } else { + writeU8(os, WASM_OPCODE_BLOCK, "BLOCK"); + writeU8(os, WASM_TYPE_NORESULT, "no result"); + + // If `__ctors_done` is true, skip the rest of the code here. + writePtrConst(os, flagAddress, is64, "flag address"); + writeU8(os, WASM_OPCODE_I32_LOAD, "LOAD"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_BR_IF, "BR_IF"); + writeUleb128(os, 0, "BR_IF immediate"); + } + + // Call `__wasm_call_ctors`. writeU8(os, WASM_OPCODE_CALL, "CALL"); writeUleb128(os, WasmSym::callCtors->getFunctionIndex(), - "function index"); + "function index"); + + if (config->sharedMemory) { + // Set flag to 2 to mark end of initialization + writePtrConst(os, flagAddress, is64, "flag address"); + writeI32Const(os, 2, "flag value"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_ATOMIC_STORE, "i32.atomic.store"); + writeMemArg(os, 2, 0); + + // Notify any waiters that memory initialization is complete + writePtrConst(os, flagAddress, is64, "flag address"); + writeI32Const(os, -1, "number of waiters"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_ATOMIC_NOTIFY, "atomic.notify"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_DROP, "drop"); + + // Branch to the done label + writeU8(os, WASM_OPCODE_BR, "br"); + writeUleb128(os, 1, "label $done"); + + writeU8(os, WASM_OPCODE_END, "end $wait"); + + // Wait for the winning thread to initialize memory + writePtrConst(os, flagAddress, is64, "flag address"); + writeI32Const(os, 1, "expected flag value"); + writeI64Const(os, -1, "timeout"); + writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix"); + writeUleb128(os, WASM_OPCODE_I32_ATOMIC_WAIT, "i32.atomic.wait"); + writeMemArg(os, 2, 0); + writeU8(os, WASM_OPCODE_DROP, "drop"); + + writeU8(os, WASM_OPCODE_END, "end $done"); + } else { + // Set `__ctors_done` to true. + writePtrConst(os, flagAddress, is64, "flag address"); + writeI32Const(os, 1, "new flag value"); + writeU8(os, WASM_OPCODE_I32_STORE, "STORE"); + writeMemArg(os, 2, 0); + + writeU8(os, WASM_OPCODE_END, "END"); + } } // Call the user's code, leaving any return values on the operand stack. @@ -1600,16 +1709,16 @@ createCallCtorsFunction(); - // Create export wrappers for commands if needed. + // Create export wrappers for reactors if needed. // // If the input contains a call to `__wasm_call_ctors`, either in one of // the input objects or an explicit export from the command-line, we // assume ctors and dtors are taken care of already. - if (!config->relocatable && !config->isPic && + if (!config->isPic && config->entry.empty() && !WasmSym::callCtors->isUsedInRegularObj && !WasmSym::callCtors->isExported()) { - log("-- createCommandExportWrappers"); - createCommandExportWrappers(); + log("-- createReactorExportWrappers"); + createReactorExportWrappers(); } } diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h --- a/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/llvm/include/llvm/BinaryFormat/Wasm.h @@ -287,12 +287,14 @@ WASM_OPCODE_LOCAL_TEE = 0x22, WASM_OPCODE_GLOBAL_GET = 0x23, WASM_OPCODE_GLOBAL_SET = 0x24, + WASM_OPCODE_I32_LOAD = 0x28, WASM_OPCODE_I32_STORE = 0x36, WASM_OPCODE_I64_STORE = 0x37, WASM_OPCODE_I32_CONST = 0x41, WASM_OPCODE_I64_CONST = 0x42, WASM_OPCODE_F32_CONST = 0x43, WASM_OPCODE_F64_CONST = 0x44, + WASM_OPCODE_I32_EQZ = 0x45, WASM_OPCODE_I32_ADD = 0x6a, WASM_OPCODE_I32_SUB = 0x6b, WASM_OPCODE_I32_MUL = 0x6c, @@ -306,6 +308,7 @@ enum : unsigned { WASM_OPCODE_BLOCK = 0x02, WASM_OPCODE_BR = 0x0c, + WASM_OPCODE_BR_IF = 0x0d, WASM_OPCODE_BR_TABLE = 0x0e, WASM_OPCODE_RETURN = 0x0f, WASM_OPCODE_DROP = 0x1a,