diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp --- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp @@ -239,8 +239,10 @@ Changed = true; } - // Start assigning local numbers after the last parameter. + // Start assigning local numbers after the last parameter and after any + // already-assigned locals. unsigned CurLocal = static_cast(MFI.getParams().size()); + CurLocal += static_cast(MFI.getLocals().size()); // Precompute the set of registers that are unused, so that we can insert // drops to their defs. diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -17,6 +17,8 @@ HANDLE_NODETYPE(RET_CALL) HANDLE_NODETYPE(RETURN) HANDLE_NODETYPE(ARGUMENT) +HANDLE_NODETYPE(LOCAL_GET) +HANDLE_NODETYPE(LOCAL_SET) // A wrapper node for TargetExternalSymbol, TargetGlobalAddress, and MCSymbol HANDLE_NODETYPE(Wrapper) // A special wapper used in PIC code for __memory_base/__table_base relative diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h @@ -45,27 +45,9 @@ WebAssemblyTargetLowering(const TargetMachine &TM, const WebAssemblySubtarget &STI); - enum WasmAddressSpace { - DEFAULT = 0, - EXTERNREF = 1, - EXTERNREF_GLOBAL = 2, - FUNCREF = 3, - FUNCREF_GLOBAL = 4, - }; - - // WebAssembly uses 5 different address spaces [0, 4]: - // AS 0 : is the default address space - // AS 1 : is a non-integral address space for externref values - // AS 2 : is an integral address space for globals of externref values - // AS 3 : is an non-integral address space for funcref values - // AS 4 : is an integral address spaces for globals of funcref values - MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override { - if (AS == WasmAddressSpace::EXTERNREF) - return MVT::externref; - else if (AS == WasmAddressSpace::FUNCREF) - return MVT::funcref; - return TargetLowering::getPointerTy(DL, AS); - } + // Pointers in well-known non-default address spaces lower to reference types, + // not linear memory pointers. + MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override; private: /// Keep a pointer to the WebAssemblySubtarget around so that we can make the @@ -88,9 +70,6 @@ bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I = nullptr) const override; - bool isExternrefGlobal(SDValue Op) const; - bool isFuncrefGlobal(SDValue Op) const; - bool isFuncref(const Value *Op) const; bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const override; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -313,6 +313,42 @@ setMinimumJumpTableEntries(2); } +// With the reference-types feature, WebAssembly programs can have a mix of +// pointers to data in linear memory and "opaque" data managed by the run-time. +// For example, in a web embedding, these values might be garbage-collected +// JavaScript values from the host. In LLVM we allow references to +// reference-typed values to be represented as pointers on the IR level, in +// order to represent mutable global variables (which lower to WebAssembly +// globals), global arrays (which lower to WebAssembly tables), and local +// variables (which lower to WebAssembly locals). +// +// On the IR level, we distinguish these different kinds of +// pointers-to-reference-types by putting the pointers into different address +// spaces. When translating an IR pointer to an MVT, the address space is the +// only way to choose an MVT; notably, we don't have the llvm::PointerType. +MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL, + uint32_t AS) const { + switch (AS) { + case WebAssembly::WASM_ADDRESS_SPACE_EXTERNREF: + // Externref values, punned as pointers in a specific address space. + return MVT::externref; + case WebAssembly::WASM_ADDRESS_SPACE_EXTERNREF_GLOBAL: + case WebAssembly::WASM_ADDRESS_SPACE_EXTERNREF_LOCAL: + // Externref locations. Abuse MVT::externref because they really aren't + // normal pointers. + return MVT::externref; + case WebAssembly::WASM_ADDRESS_SPACE_FUNCREF: + // Funcref values. + return MVT::funcref; + case WebAssembly::WASM_ADDRESS_SPACE_FUNCREF_GLOBAL: + case WebAssembly::WASM_ADDRESS_SPACE_FUNCREF_LOCAL: + // Funcref locations, as in EXTERNREF_GLOBAL / EXTERNREF_LOCAL. + return MVT::funcref; + default: + return TargetLowering::getPointerTy(DL, AS); + } +} + TargetLowering::AtomicExpansionKind WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { // We have wasm instructions for these @@ -833,6 +869,13 @@ CallConv == CallingConv::Swift; } +static bool IsFuncref(const Value *Op) { + const Type *Ty = Op->getType(); + + return isa(Ty) && Ty->getPointerAddressSpace() == + WebAssembly::WASM_ADDRESS_SPACE_FUNCREF; +} + SDValue WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI, SmallVectorImpl &InVals) const { @@ -1069,7 +1112,7 @@ // Lastly, if this is a call to a funcref we need to add an instruction // table.set to the chain and transform the call. - if (CLI.CB && isFuncref(CLI.CB->getCalledOperand())) { + if (CLI.CB && IsFuncref(CLI.CB->getCalledOperand())) { // In the absence of function references proposal where a funcref call is // lowered to call_ref, using reference types we generate a table.set to set // the funcref to a special table used solely for this purpose, followed by @@ -1309,27 +1352,51 @@ } } -bool WebAssemblyTargetLowering::isExternrefGlobal(SDValue Op) const { +static Optional GetGlobalReferenceType(SDValue Op) { const GlobalAddressSDNode *GA = dyn_cast(Op); - if (!GA || GA->getAddressSpace() != WasmAddressSpace::EXTERNREF_GLOBAL) - return false; - - return true; + if (!GA) + return None; + + switch (GA->getAddressSpace()) { + case WebAssembly::WASM_ADDRESS_SPACE_FUNCREF_GLOBAL: + return MVT(MVT::funcref); + case WebAssembly::WASM_ADDRESS_SPACE_EXTERNREF_GLOBAL: + return MVT(MVT::externref); + default: + return None; + } } -bool WebAssemblyTargetLowering::isFuncrefGlobal(SDValue Op) const { - const GlobalAddressSDNode *GA = dyn_cast(Op); - if (!GA || GA->getAddressSpace() != WasmAddressSpace::FUNCREF_GLOBAL) +static bool ResolveLocal(SDValue Op, SelectionDAG &DAG, MVT *VT, + unsigned *Local) { + const FrameIndexSDNode *FI = dyn_cast(Op); + if (!FI) return false; - return true; -} + int FrameIndex = FI->getIndex(); + auto &MFI = DAG.getMachineFunction().getFrameInfo(); + const AllocaInst *AI = MFI.getObjectAllocation(FrameIndex); -bool WebAssemblyTargetLowering::isFuncref(const Value *Op) const { - const Type *Ty = Op->getType(); + if (!AI) + return false; + + switch (AI->getType()->getAddressSpace()) { + case WebAssembly::WASM_ADDRESS_SPACE_FUNCREF_LOCAL: + *VT = MVT::funcref; + break; + case WebAssembly::WASM_ADDRESS_SPACE_EXTERNREF_LOCAL: + *VT = MVT::externref; + break; + default: + return false; + } - return isa(Ty) && - Ty->getPointerAddressSpace() == WasmAddressSpace::FUNCREF; + auto *FuncInfo = DAG.getMachineFunction().getInfo(); + *Local = FuncInfo->getLocalForStackObject(FrameIndex, *VT); + // Mark stack object as dead, as it will be replaced by the local. + if (!MFI.isDeadObjectIndex(FrameIndex)) + MFI.RemoveStackObject(FrameIndex); + return true; } SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op, @@ -1337,23 +1404,28 @@ SDLoc DL(Op); StoreSDNode *SN = cast(Op.getNode()); - const SDValue &Value = SN->getValue(); + const SDValue &Base = SN->getBasePtr(); - // Expect Offset to be undef for externref global stores - const SDValue &Offset = SN->getOffset(); - if (!Offset->isUndef()) - return Op; + if (SN->getOffset()->isUndef()) { + if (Optional VT = GetGlobalReferenceType(Base)) { + SDVTList Tys = DAG.getVTList(MVT::Other); + SDValue Ops[] = {SN->getChain(), Value, Base}; + return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops, + SN->getMemoryVT(), SN->getMemOperand()); + } - // Expect Base to be an externref GlobalAddress - const SDValue &Base = SN->getBasePtr(); - if (!isExternrefGlobal(Base) && !isFuncrefGlobal(Base)) - return Op; + MVT LocalVT; + unsigned LocalIdx; + if (ResolveLocal(Base, DAG, &LocalVT, &LocalIdx)) { + SDValue Idx = DAG.getTargetConstant(LocalIdx, Base, MVT::i32); + SDVTList Tys = DAG.getVTList(MVT::Other); // The chain. + SDValue Ops[] = {SN->getChain(), Idx, Value}; + return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops); + } + } - SDVTList Tys = DAG.getVTList(MVT::Other); - SDValue Ops[] = {SN->getChain(), Value, Base}; - return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops, - SN->getMemoryVT(), SN->getMemOperand()); + return Op; } SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op, @@ -1363,24 +1435,29 @@ LoadSDNode *LN = cast(Op.getNode()); const SDValue &Base = LN->getBasePtr(); - // Check Base is a Global Address - if (!isExternrefGlobal(Base) && !isFuncrefGlobal(Base)) - return Op; - - // Check Offset if undef - const SDValue &Offset = LN->getOffset(); - if (!Offset->isUndef()) - return Op; - - EVT VT = isExternrefGlobal(Base) ? MVT::externref : MVT::funcref; + if (LN->getOffset()->isUndef()) { + if (Optional VT = GetGlobalReferenceType(Base)) { + SDValue GlobalGet = DAG.getMemIntrinsicNode( + WebAssemblyISD::GLOBAL_GET, DL, DAG.getVTList(*VT), + {LN->getChain(), Base}, LN->getMemoryVT(), LN->getMemOperand()); + SDValue Result = DAG.getMergeValues({GlobalGet, LN->getChain()}, DL); + assert(Result->getNumValues() == 2 && "Loads must carry a chain!"); + return Result; + } - SDValue GlobalGet = DAG.getMemIntrinsicNode( - WebAssemblyISD::GLOBAL_GET, DL, DAG.getVTList(VT), {LN->getChain(), Base}, - LN->getMemoryVT(), LN->getMemOperand()); - SDValue Result = DAG.getMergeValues({GlobalGet, LN->getChain()}, DL); - assert(Result->getNumValues() == 2 && "Loads must carry a chain!"); + MVT LocalVT; + unsigned LocalIdx; + if (ResolveLocal(Base, DAG, &LocalVT, &LocalIdx)) { + SDValue Idx = DAG.getTargetConstant(LocalIdx, Base, MVT::i32); + SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT, + {LN->getChain(), Idx}); + SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL); + assert(Result->getNumValues() == 2 && "Loads must carry a chain!"); + return Result; + } + } - return Result; + return Op; } SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op, @@ -1412,6 +1489,7 @@ SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op, SelectionDAG &DAG) const { int FI = cast(Op)->getIndex(); + assert(!DAG.getMachineFunction().getFrameInfo().isDeadObjectIndex(FI)); return DAG.getTargetFrameIndex(FI, Op.getValueType()); } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td @@ -73,6 +73,8 @@ SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>; def SDT_WebAssemblyBrTable : SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>; def SDT_WebAssemblyArgument : SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>; +def SDT_WebAssemblyLocalGet : SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>; +def SDT_WebAssemblyLocalSet : SDTypeProfile<0, 2, [SDTCisVT<0, i32>]>; def SDT_WebAssemblyReturn : SDTypeProfile<0, -1, []>; def SDT_WebAssemblyWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisPtrTy<0>]>; @@ -96,6 +98,12 @@ [SDNPHasChain, SDNPVariadic]>; def WebAssemblyargument : SDNode<"WebAssemblyISD::ARGUMENT", SDT_WebAssemblyArgument>; +def WebAssemblylocalGet : SDNode<"WebAssemblyISD::LOCAL_GET", + SDT_WebAssemblyLocalGet, + [SDNPHasChain, SDNPMayLoad]>; +def WebAssemblylocalSet : SDNode<"WebAssemblyISD::LOCAL_SET", + SDT_WebAssemblyLocalSet, + [SDNPHasChain, SDNPMayStore]>; def WebAssemblyreturn : SDNode<"WebAssemblyISD::RETURN", SDT_WebAssemblyReturn, [SDNPHasChain, SDNPVariadic]>; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td @@ -26,6 +26,7 @@ vt#".select\t$dst, $lhs, $rhs, $cond", vt#".select", 0x1b>, Requires<[HasReferenceTypes]>; + } defm "" : REF_I; @@ -36,4 +37,18 @@ (!cast("SELECT_"#reg) reg:$lhs, reg:$rhs, I32:$cond)>; def : Pat<(select (i32 (seteq I32:$cond, 0)), reg:$lhs, reg:$rhs), (!cast("SELECT_"#reg) reg:$rhs, reg:$lhs, I32:$cond)>; + + // Legalization will lower reference-typed alloca to locals, and lower + // their uses to WebAssemblyISD::LOCAL_GET / + // WebAssemblyISD::LOCAL_SET; see WebAssemblyTargetLowering::LowerLoad + // and WebAssemblyTargetLowering::LowerStore. These patterns then + // selct the corresponding machine instructions. + foreach vt = reg.RegTypes in { + def : Pat<(vt (WebAssemblylocalGet (i32 timm:$local))), + (!cast("LOCAL_GET_" # reg) timm:$local)>, + Requires<[HasReferenceTypes]>; + def : Pat<(WebAssemblylocalSet timm:$local, vt:$src), + (!cast("LOCAL_SET_" # reg) timm:$local, reg:$src)>, + Requires<[HasReferenceTypes]>; + } } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h --- a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h @@ -37,6 +37,8 @@ std::vector Results; std::vector Locals; + DenseMap FrameIndex2Local; + /// A mapping from CodeGen vreg index to WebAssembly register number. std::vector WARegs; @@ -94,6 +96,22 @@ void addLocal(MVT VT) { Locals.push_back(VT); } const std::vector &getLocals() const { return Locals; } + unsigned getLocalForStackObject(int FrameIndex, MVT VT) { + const auto &Iter = FrameIndex2Local.find(FrameIndex); + unsigned LocalIdx; + if (Iter == FrameIndex2Local.end()) { + LocalIdx = Locals.size(); + // Add an entry that records the stack object at FrameIndex as being + // replaced by the WebAssembly local with index LocalIdx. The caller is + // responsible for marking the stack object as dead in the frame. + FrameIndex2Local.insert(std::make_pair(FrameIndex, LocalIdx)); + addLocal(VT); + } else { + LocalIdx = Iter->second; + } + return LocalIdx + Params.size(); + } + unsigned getVarargBufferVreg() const { assert(VarargVreg != -1U && "Vararg vreg hasn't been set"); return VarargVreg; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -124,16 +124,17 @@ const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, Optional RM, Optional CM, CodeGenOpt::Level OL, bool JIT) - : LLVMTargetMachine(T, - TT.isArch64Bit() - ? (hasReferenceTypes(FS) - ? "e-m:e-p:64:64-i64:64-n32:64-S128-ni:1:3" - : "e-m:e-p:64:64-i64:64-n32:64-S128") - : (hasReferenceTypes(FS) - ? "e-m:e-p:32:32-i64:64-n32:64-S128-ni:1:3" - : "e-m:e-p:32:32-i64:64-n32:64-S128"), - TT, CPU, FS, Options, getEffectiveRelocModel(RM, TT), - getEffectiveCodeModel(CM, CodeModel::Large), OL), + : LLVMTargetMachine( + T, + TT.isArch64Bit() + ? (hasReferenceTypes(FS) + ? "e-m:e-p:64:64-i64:64-n32:64-S128-ni:1:3:5:6-A0:u" + : "e-m:e-p:64:64-i64:64-n32:64-S128") + : (hasReferenceTypes(FS) + ? "e-m:e-p:32:32-i64:64-n32:64-S128-ni:1:3:5:6-A0:u" + : "e-m:e-p:32:32-i64:64-n32:64-S128"), + TT, CPU, FS, Options, getEffectiveRelocModel(RM, TT), + getEffectiveCodeModel(CM, CodeModel::Large), OL), TLOF(new WebAssemblyTargetObjectFile()) { // WebAssembly type-checks instructions, but a noreturn function with a return // type that doesn't match the context will cause a check failure. So we lower diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h --- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h @@ -28,6 +28,33 @@ namespace WebAssembly { +// FIXME: Renumber so that EXTERNREF_LOCAL is contiguous with EXTERNREF and +// EXTERNREF_GLOBAL. +// An unscoped enum, as address spaces are often produced and consumed as +// uint32_t or unsigned. +enum WasmAddressSpace { + // Default address space, for pointers to unmanaged data in linear memory + // (stack, heap, data). + WASM_ADDRESS_SPACE_DEFAULT = 0, + // A non-integral address space used to represent externref values. We use a + // separate address space to prevent LLVM from attempting to write them to + // linear memory. Because you can't add an offset to one externref value to + // get a new externref value, we mark this address space as non-integral. + WASM_ADDRESS_SPACE_EXTERNREF = 1, + // An integral address space for static-storage-duration locations of + // externref values. This address space is integral, to allow for tables to + // be represented as arrays in IR. + WASM_ADDRESS_SPACE_EXTERNREF_GLOBAL = 2, + // A non-integral address space for automatic-storage-duration locations of + // externref values (local variables). Non-integral because having the + // location of one local doesn't give you access to any other local. + WASM_ADDRESS_SPACE_EXTERNREF_LOCAL = 5, + // Same as above, but for funcref values. + WASM_ADDRESS_SPACE_FUNCREF = 3, + WASM_ADDRESS_SPACE_FUNCREF_GLOBAL = 4, + WASM_ADDRESS_SPACE_FUNCREF_LOCAL = 6, +}; + bool isChild(const MachineInstr &MI, const WebAssemblyFunctionInfo &MFI); bool mayThrow(const MachineInstr &MI); diff --git a/llvm/test/CodeGen/WebAssembly/externref-alloca.ll b/llvm/test/CodeGen/WebAssembly/externref-alloca.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/externref-alloca.ll @@ -0,0 +1,22 @@ +; RUN: llc < %s -mtriple=wasm32-unknown-unknown -asm-verbose=false -mattr=+reference-types | FileCheck %s + +;; Address space 1 is for externref values, and address space 5 is for +;; externref-valued local variables. + +%extern = type opaque +%externref = type %extern addrspace(1)* +%externref_local = type %externref addrspace(5)* + +; CHECK-LABEL: alloca_externref: +define %externref @alloca_externref(%externref %param) { + ; CHECK-NEXT: .functype alloca_externref (externref) -> (externref) + %retval = alloca %externref, addrspace(5) + ; CHECK-NEXT: .local externref + ; CHECK-NEXT: local.get 0 + ; CHECK-NEXT: local.set 1 + store %externref %param, %externref_local %retval + ; CHECK-NEXT: local.get 1 + %reloaded = load %externref, %externref_local %retval + ; CHECK-NEXT: end_function + ret %externref %reloaded +}