[WebAssembly] memory.copy

[tlively]

Summary

Depends on D57495.

[tlively]

• Clean up pattern

[dschuff]

llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

248

Is it smaller than even 1 store? Even if it is, stores can have constant offsets. Probably best to set this to at least 2.

[aheejin]

llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp

31

Do we need WebAssemblyISD::MEMORY_COPY node? Can we possibly lower it directly using DAG.getMachineNode here? In case that involves more work, nevermind. But if we are just transferring operands one-to-one that also looks as simple.

llvm/test/CodeGen/WebAssembly/bulk-memory.ll

66

Are there any difference between memcpy_1, memory_4, and memory_1024 other than the argument to i32.const? Do we need all these three tests?

[tlively]

• Prefer to use single load-store pairs

[tlively]

llvm/test/CodeGen/WebAssembly/bulk-memory.ll

66

No, we can probably get away with one test for the load-store pair and one for memory.copy.

[tlively]

• Address comments

[tlively]

llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

248

We agreed offline that setting this to 1 is fine for now (it was originally 0). Once bulk memory is correct throughout the toolchain, we will take measurements and figure out where these thresholds should be set.

llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp

31

After looking into it for a bit, I wasn't able to get this to work :\

llvm/lib/Target/WebAssembly/WebAssemblyISD.def

 HANDLE_NODETYPE(Wrapper)
 HANDLE_NODETYPE(BR_IF)
 HANDLE_NODETYPE(BR_TABLE)
 HANDLE_NODETYPE(SHUFFLE)
 HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
 HANDLE_NODETYPE(THROW)
+HANDLE_NODETYPE(MEMORY_COPY)
 
 // add memory opcodes starting at ISD::FIRST_TARGET_MEMORY_OPCODE here...

llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   // Trap lowers to wasm unreachable
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // Exception handling intrinsics
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
   setMaxAtomicSizeInBitsSupported(64);
+
+  if (Subtarget->hasBulkMemory()) {
+    // Using memory.copy is always better than using multiple loads and stores
+    MaxStoresPerMemcpy = 1;
+    MaxStoresPerMemcpyOptSize = 1;
+  }
 }
 
 TargetLowering::AtomicExpansionKind
 WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // We have wasm instructions for these
   switch (AI->getOperation()) {
   case AtomicRMWInst::Add:
   case AtomicRMWInst::Sub:

llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td

+// WebAssemblyInstrBulkMemory.td - bulk memory codegen support --*- tablegen -*-
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// WebAssembly bulk memory codegen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+// Instruction requiring HasBulkMemory and the bulk memory prefix byte
+multiclass BULK_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
+                  list<dag> pattern_r, string asmstr_r = "",
+                  string asmstr_s = "", bits<32> simdop = -1> {
+  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
+              !or(0xfc00, !and(0xff, simdop))>,
+            Requires<[HasBulkMemory]>;
+}
+
+// Bespoke types and nodes for bulk memory ops
+def wasm_memcpy_t : SDTypeProfile<0, 3,
+  [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisInt<2>]
+>;
+def wasm_memcpy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t,
+                         [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>;
+
+//===----------------------------------------------------------------------===//
+// memory.copy
+//===----------------------------------------------------------------------===//
+
+let mayLoad = 1, mayStore = 1 in
+defm MEMORY_COPY : BULK_I<(outs), (ins I32:$dst, I32:$src, I32:$len),
+                          (outs), (ins),
+                          [(wasm_memcpy I32:$dst, I32:$src, I32:$len)],
+                          "memory.copy\t$dst, $src, $len",
+                          "memory.copy", 0x0a>;

llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td

 include "WebAssemblyInstrCall.td"
 include "WebAssemblyInstrControl.td"
 include "WebAssemblyInstrInteger.td"
 include "WebAssemblyInstrConv.td"
 include "WebAssemblyInstrFloat.td"
 include "WebAssemblyInstrAtomics.td"
 include "WebAssemblyInstrSIMD.td"
 include "WebAssemblyInstrExceptRef.td"
+include "WebAssemblyInstrBulkMemory.td"

llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h

 
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 
 namespace llvm {
 
 class WebAssemblySelectionDAGInfo final : public SelectionDAGTargetInfo {
 public:
   ~WebAssemblySelectionDAGInfo() override;
+  SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
+                                  SDValue Chain, SDValue Op1, SDValue Op2,
+                                  SDValue Op3, unsigned Align, bool isVolatile,
+                                  bool AlwaysInline,
+                                  MachinePointerInfo DstPtrInfo,
+                                  MachinePointerInfo SrcPtrInfo) const override;
 };
 
 } // end namespace llvm
 
 #endif

llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp

 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyTargetMachine.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "wasm-selectiondag-info"
 
 WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() {}
+
+SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
+    SDValue Op3, unsigned Align, bool isVolatile, bool AlwaysInline,
+    MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
+  if (!DAG.getMachineFunction()
+           .getSubtarget<WebAssemblySubtarget>()
+           .hasBulkMemory())
+    return SDValue();
+
+  return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other, Chain, Op1,
+                     Op2, Op3);
+}

llvm/test/CodeGen/WebAssembly/bulk-memory.ll

-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s --check-prefixes CHECK,BULK-MEM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-bulk-memory | FileCheck %s --check-prefixes CHECK,NO-BULK-MEM
 
 ; Test that basic bulk memory codegen works correctly
-; TODO: implement basic bulk memory codegen
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: memcpy_i8:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy $0, $1, $2
+; BULK-MEM-NEXT: return
+declare void @llvm.memcpy.p0i8.p0i8.i32(
+  i8* %dest, i8* %src, i32 %len, i1 %volatile
+)
+define void @memcpy_i8(i8* %dest, i8* %src, i32 %len) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_i32:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy $0, $1, $2
+; BULK-MEM-NEXT: return
+declare void @llvm.memcpy.p0i32.p0i32.i32(
+  i32* %dest, i32* %src, i32 %len, i1 %volatile
+)
+define void @memcpy_i32(i32* %dest, i32* %src, i32 %len) {
+  call void @llvm.memcpy.p0i32.p0i32.i32(i32* %dest, i32* %src, i32 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1:
+; CHECK-NEXT: .functype memcpy_1 (i32, i32) -> ()
+; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1)
+; CHECK-NEXT: i32.store8 0($0), $pop[[L0]]
+; CHECK-NEXT: return
+define void @memcpy_1(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_8:
+; CHECK-NEXT: .functype memcpy_8 (i32, i32) -> ()
+; CHECK-NEXT: i64.load $push[[L0:[0-9]+]]=, 0($1):p2align=0
+; CHECK-NEXT: i64.store 0($0):p2align=0, $pop[[L0]]
+; CHECK-NEXT: return
+define void @memcpy_8(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 8, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_16:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_16 (i32, i32) -> ()
+; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 16
+; BULK-MEM-NEXT: memory.copy $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memcpy_16(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 16, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1024:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memcpy_1024(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1024, i1 0)
+  ret void
+}