diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td --- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td @@ -212,7 +212,8 @@ def int_wasm_memory_init : Intrinsic<[], - [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_anyint_ty, + llvm_anyint_ty], [IntrWriteMem, IntrInaccessibleMemOrArgMemOnly, WriteOnly>, IntrHasSideEffects, ImmArg>, ImmArg>]>; def int_wasm_data_drop : diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -33,39 +33,44 @@ def wasm_memset : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; +multiclass BulkMemoryOps { let mayStore = 1, hasSideEffects = 1 in -defm MEMORY_INIT : +defm MEMORY_INIT_A#B : BULK_I<(outs), - (ins i32imm_op:$seg, i32imm_op:$idx, I32:$dest, - I32:$offset, I32:$size), + (ins i32imm_op:$seg, i32imm_op:$idx, rc:$dest, + rc:$offset, rc:$size), (outs), (ins i32imm_op:$seg, i32imm_op:$idx), - [(int_wasm_memory_init (i32 timm:$seg), (i32 timm:$idx), I32:$dest, - I32:$offset, I32:$size + [(int_wasm_memory_init (i32 timm:$seg), (i32 timm:$idx), rc:$dest, + rc:$offset, rc:$size )], "memory.init\t$seg, $idx, $dest, $offset, $size", "memory.init\t$seg, $idx", 0x08>; let hasSideEffects = 1 in -defm DATA_DROP : +defm DATA_DROP_A#B : BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg), [(int_wasm_data_drop (i32 timm:$seg))], "data.drop\t$seg", "data.drop\t$seg", 0x09>; let mayLoad = 1, mayStore = 1 in -defm MEMORY_COPY : +defm MEMORY_COPY_A#B : BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - I32:$dst, I32:$src, I32:$len), + rc:$dst, rc:$src, rc:$len), (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - I32:$dst, I32:$src, I32:$len + rc:$dst, rc:$src, rc:$len )], "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", "memory.copy\t$src_idx, $dst_idx", 0x0a>; let mayStore = 1 in -defm MEMORY_FILL : - BULK_I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size), +defm MEMORY_FILL_A#B : + BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)], + [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], "memory.fill\t$idx, $dst, $value, $size", "memory.fill\t$idx", 0x0b>; +} + +defm : BulkMemoryOps; +defm : BulkMemoryOps; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td @@ -365,19 +365,24 @@ defm : StorePatGlobalAddrOffOnly; defm : StorePatGlobalAddrOffOnly; +multiclass MemoryOps { // Current memory size. -defm MEMORY_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags), +defm MEMORY_SIZE_A#B : I<(outs rc:$dst), (ins i32imm:$flags), (outs), (ins i32imm:$flags), - [(set I32:$dst, + [(set rc:$dst, (int_wasm_memory_size (i32 imm:$flags)))], "memory.size\t$dst, $flags", "memory.size\t$flags", 0x3f>; // Grow memory. -defm MEMORY_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta), +defm MEMORY_GROW_A#B : I<(outs rc:$dst), (ins i32imm:$flags, rc:$delta), (outs), (ins i32imm:$flags), - [(set I32:$dst, + [(set rc:$dst, (int_wasm_memory_grow (i32 imm:$flags), - I32:$delta))], + rc:$delta))], "memory.grow\t$dst, $flags, $delta", "memory.grow\t$flags", 0x40>; +} + +defm : MemoryOps; +defm : MemoryOps; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp --- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp @@ -22,15 +22,15 @@ SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { - if (!DAG.getMachineFunction() - .getSubtarget() - .hasBulkMemory()) + auto &ST = DAG.getMachineFunction().getSubtarget(); + if (!ST.hasBulkMemory()) return SDValue(); SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32); + auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32; return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other, {Chain, MemIdx, MemIdx, Dst, Src, - DAG.getZExtOrTrunc(Size, DL, MVT::i32)}); + DAG.getZExtOrTrunc(Size, DL, LenMVT)}); } SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove( @@ -46,14 +46,14 @@ SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Val, SDValue Size, unsigned Align, bool IsVolatile, MachinePointerInfo DstPtrInfo) const { - if (!DAG.getMachineFunction() - .getSubtarget() - .hasBulkMemory()) + auto &ST = DAG.getMachineFunction().getSubtarget(); + if (!ST.hasBulkMemory()) return SDValue(); SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32); + auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32; // Only low byte matters for val argument, so anyext the i8 return DAG.getNode(WebAssemblyISD::MEMORY_FILL, DL, MVT::Other, Chain, MemIdx, Dst, DAG.getAnyExtOrTrunc(Val, DL, MVT::i32), - DAG.getZExtOrTrunc(Size, DL, MVT::i32)); + DAG.getZExtOrTrunc(Size, DL, LenMVT)); } diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll @@ -0,0 +1,210 @@ +; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s --check-prefixes CHECK,BULK-MEM +; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-bulk-memory | FileCheck %s --check-prefixes CHECK,NO-BULK-MEM + +; Test that basic bulk memory codegen works correctly + +target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128" +target triple = "wasm64-unknown-unknown" + +declare void @llvm.memcpy.p0i8.p0i8.i8(i8*, i8*, i8, i1) +declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1) +declare void @llvm.memcpy.p0i32.p0i32.i64(i32*, i32*, i64, i1) + +declare void @llvm.memmove.p0i8.p0i8.i8(i8*, i8*, i8, i1) +declare void @llvm.memmove.p0i8.p0i8.i64(i8*, i8*, i64, i1) +declare void @llvm.memmove.p0i32.p0i32.i64(i32*, i32*, i64, i1) + +declare void @llvm.memset.p0i8.i8(i8*, i8, i8, i1) +declare void @llvm.memset.p0i8.i64(i8*, i8, i64, i1) +declare void @llvm.memset.p0i32.i64(i32*, i8, i64, i1) + +; CHECK-LABEL: memcpy_i8: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memcpy_i8 (i64, i64, i32) -> () +; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0 +; BULK-MEM-NEXT: return +define void @memcpy_i8(i8* %dest, i8* %src, i8 zeroext %len) { + call void @llvm.memcpy.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0) + ret void +} + +; CHECK-LABEL: memmove_i8: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memmove_i8 (i64, i64, i32) -> () +; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0 +; BULK-MEM-NEXT: return +define void @memmove_i8(i8* %dest, i8* %src, i8 zeroext %len) { + call void @llvm.memmove.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0) + ret void +} + +; CHECK-LABEL: memset_i8: +; NO-BULK-MEM-NOT: memory.fill +; BULK-MEM-NEXT: .functype memset_i8 (i64, i32, i32) -> () +; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop0 +; BULK-MEM-NEXT: return +define void @memset_i8(i8* %dest, i8 %val, i8 zeroext %len) { + call void @llvm.memset.p0i8.i8(i8* %dest, i8 %val, i8 %len, i1 0) + ret void +} + +; CHECK-LABEL: memcpy_i32: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memcpy_i32 (i64, i64, i64) -> () +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: return +define void @memcpy_i32(i32* %dest, i32* %src, i64 %len) { + call void @llvm.memcpy.p0i32.p0i32.i64(i32* %dest, i32* %src, i64 %len, i1 0) + ret void +} + +; CHECK-LABEL: memmove_i32: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memmove_i32 (i64, i64, i64) -> () +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: return +define void @memmove_i32(i32* %dest, i32* %src, i64 %len) { + call void @llvm.memmove.p0i32.p0i32.i64(i32* %dest, i32* %src, i64 %len, i1 0) + ret void +} + +; CHECK-LABEL: memset_i32: +; NO-BULK-MEM-NOT: memory.fill +; BULK-MEM-NEXT: .functype memset_i32 (i64, i32, i64) -> () +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2 +; BULK-MEM-NEXT: return +define void @memset_i32(i32* %dest, i8 %val, i64 %len) { + call void @llvm.memset.p0i32.i64(i32* %dest, i8 %val, i64 %len, i1 0) + ret void +} + +; CHECK-LABEL: memcpy_1: +; CHECK-NEXT: .functype memcpy_1 (i64, i64) -> () +; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1) +; CHECK-NEXT: i32.store8 0($0), $pop[[L0]] +; CHECK-NEXT: return +define void @memcpy_1(i8* %dest, i8* %src) { + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1, i1 0) + ret void +} + +; CHECK-LABEL: memmove_1: +; CHECK-NEXT: .functype memmove_1 (i64, i64) -> () +; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1) +; CHECK-NEXT: i32.store8 0($0), $pop[[L0]] +; CHECK-NEXT: return +define void @memmove_1(i8* %dest, i8* %src) { + call void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1, i1 0) + ret void +} + +; CHECK-LABEL: memset_1: +; NO-BULK-MEM-NOT: memory.fill +; BULK-MEM-NEXT: .functype memset_1 (i64, i32) -> () +; BULK-MEM-NEXT: i32.store8 0($0), $1 +; BULK-MEM-NEXT: return +define void @memset_1(i8* %dest, i8 %val) { + call void @llvm.memset.p0i8.i64(i8* %dest, i8 %val, i64 1, i1 0) + ret void +} + +; CHECK-LABEL: memcpy_1024: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memcpy_1024 (i64, i64) -> () +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: return +define void @memcpy_1024(i8* %dest, i8* %src) { + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1024, i1 0) + ret void +} + +; CHECK-LABEL: memmove_1024: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memmove_1024 (i64, i64) -> () +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: return +define void @memmove_1024(i8* %dest, i8* %src) { + call void @llvm.memmove.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1024, i1 0) + ret void +} + +; CHECK-LABEL: memset_1024: +; NO-BULK-MEM-NOT: memory.fill +; BULK-MEM-NEXT: .functype memset_1024 (i64, i32) -> () +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: return +define void @memset_1024(i8* %dest, i8 %val) { + call void @llvm.memset.p0i8.i64(i8* %dest, i8 %val, i64 1024, i1 0) + ret void +} + +; The following tests check that frame index elimination works for +; bulk memory instructions. The stack pointer is bumped by 112 instead +; of 100 because the stack pointer in WebAssembly is currently always +; 16-byte aligned, even in leaf functions, although it is not written +; back to the global in this case. + +; TODO: Change TransientStackAlignment to 1 to avoid this extra +; arithmetic. This will require forcing the use of StackAlignment in +; PrologEpilogEmitter.cpp when +; WebAssemblyFrameLowering::needsSPWriteback would be true. + +; CHECK-LABEL: memcpy_alloca_src: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memcpy_alloca_src (i64) -> () +; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] +; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]] +; BULK-MEM-NEXT: return +define void @memcpy_alloca_src(i8* %dst) { + %a = alloca [100 x i8] + %p = bitcast [100 x i8]* %a to i8* + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 100, i1 false) + ret void +} + +; CHECK-LABEL: memcpy_alloca_dst: +; NO-BULK-MEM-NOT: memory.copy +; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i64) -> () +; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] +; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: return +define void @memcpy_alloca_dst(i8* %src) { + %a = alloca [100 x i8] + %p = bitcast [100 x i8]* %a to i8* + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %src, i64 100, i1 false) + ret void +} + +; CHECK-LABEL: memset_alloca: +; NO-BULK-MEM-NOT: memory.fill +; BULK-MEM-NEXT: .functype memset_alloca (i32) -> () +; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] +; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: return +define void @memset_alloca(i8 %val) { + %a = alloca [100 x i8] + %p = bitcast [100 x i8]* %a to i8* + call void @llvm.memset.p0i8.i64(i8* %p, i8 %val, i64 100, i1 false) + ret void +} diff --git a/llvm/test/CodeGen/WebAssembly/memory-addr64.ll b/llvm/test/CodeGen/WebAssembly/memory-addr64.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/memory-addr64.ll @@ -0,0 +1,27 @@ +; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s + +; Test that basic memory operations assemble as expected with 32-bit addresses. + +target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128" +target triple = "wasm64-unknown-unknown" + +declare i64 @llvm.wasm.memory.size.i64(i32) nounwind readonly +declare i64 @llvm.wasm.memory.grow.i64(i32, i64) nounwind + +; CHECK-LABEL: memory_size: +; CHECK-NEXT: .functype memory_size () -> (i64){{$}} +; CHECK-NEXT: memory.size $push0=, 0{{$}} +; CHECK-NEXT: return $pop0{{$}} +define i64 @memory_size() { + %a = call i64 @llvm.wasm.memory.size.i64(i32 0) + ret i64 %a +} + +; CHECK-LABEL: memory_grow: +; CHECK-NEXT: .functype memory_grow (i64) -> (i64){{$}} +; CHECK: memory.grow $push0=, 0, $0{{$}} +; CHECK-NEXT: return $pop0{{$}} +define i64 @memory_grow(i64 %n) { + %a = call i64 @llvm.wasm.memory.grow.i64(i32 0, i64 %n) + ret i64 %a +}