[WebAssembly] Prototype prefetch instructions

[tlively]

Summary

As proposed in https://github.com/WebAssembly/simd/pull/352 and using the
opcodes used in the V8 prototype:
https://chromium-review.googlesource.com/c/v8/v8/+/2543167. These instructions
are only usable via intrinsics and clang builtins to make them opt-in while they
are being benchmarked.

[aheejin]

llvm/include/llvm/IR/IntrinsicsWebAssembly.td

318

It looks IntrWillReturn is applied by default?

llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll

15

127

[aheejin]

llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll

127

Sorry the previous suggestion had an extra ; and there seems no way to delete it

[tlively]

• Fix section headers in test file

llvm/include/llvm/IR/IntrinsicsWebAssembly.td

318

It looks like this is only true for Intrinsics that inherit DefaultAttrsIntrinsic. Otherwise the default-enabled attributes are confusingly disabled by default https://github.com/llvm/llvm-project/blob/5abfeccf10bcbc0d673ece21ddd8d4ac4a0e7594/llvm/include/llvm/IR/Intrinsics.td#L346. As a separate change it would be a good idea to change most of our target intrinsics to use DefaultAttrsIntrinsics.

clang/include/clang/Basic/BuiltinsWebAssembly.def

 TARGET_BUILTIN(__builtin_wasm_load64_lane, "V2LLiLLi*V2LLiIi", "n", "simd128")
 TARGET_BUILTIN(__builtin_wasm_store8_lane, "vSc*V16ScIi", "n", "simd128")
 TARGET_BUILTIN(__builtin_wasm_store16_lane, "vs*V8sIi", "n", "simd128")
 TARGET_BUILTIN(__builtin_wasm_store32_lane, "vi*V4iIi", "n", "simd128")
 TARGET_BUILTIN(__builtin_wasm_store64_lane, "vLLi*V2LLiIi", "n", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_eq_i64x2, "V2LLiV2LLiV2LLi", "nc", "simd128")
 
+TARGET_BUILTIN(__builtin_wasm_prefetch_t, "vv*", "n", "simd128")
+TARGET_BUILTIN(__builtin_wasm_prefetch_nt, "vv*", "n", "simd128")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN

clang/lib/CodeGen/CGBuiltin.cpp

while (OpIdx < 18) {
       Optional<llvm::APSInt> LaneConst =
           E->getArg(OpIdx)->getIntegerConstantExpr(getContext());
       assert(LaneConst && "Constant arg isn't actually constant?");
       Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst);
     }
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
     return Builder.CreateCall(Callee, Ops);
   }
+  case WebAssembly::BI__builtin_wasm_prefetch_t: {
+    Value *Ptr = EmitScalarExpr(E->getArg(0));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_prefetch_t);
+    return Builder.CreateCall(Callee, Ptr);
+  }
+  case WebAssembly::BI__builtin_wasm_prefetch_nt: {
+    Value *Ptr = EmitScalarExpr(E->getArg(0));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_prefetch_nt);
+    return Builder.CreateCall(Callee, Ptr);
+  }
   default:
     return nullptr;
   }
 }
 
 static std::pair<Intrinsic::ID, unsigned>
 getIntrinsicForHexagonNonGCCBuiltin(unsigned BuiltinID) {
   struct Info {

clang/test/CodeGen/builtins-wasm.c

i8x16 shuffle(i8x16 x, i8x16 y) {
   return __builtin_wasm_shuffle_v8x16(x, y, 0, 1, 2, 3, 4, 5, 6, 7,
                                       8, 9, 10, 11, 12, 13, 14, 15);
   // WEBASSEMBLY: call <16 x i8> @llvm.wasm.shuffle(<16 x i8> %x, <16 x i8> %y,
   // WEBASSEMBLY-SAME: i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
   // WEBASSEMBLY-SAME: i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14,
   // WEBASSEMBLY-SAME: i32 15
   // WEBASSEMBLY-NEXT: ret
 }
+
+void prefetch_t(void *p) {
+  return __builtin_wasm_prefetch_t(p);
+  // WEBASSEMBLY: call void @llvm.wasm.prefetch.t(i8* %p)
+}
+
+void prefetch_nt(void *p) {
+  return __builtin_wasm_prefetch_nt(p);
+  // WEBASSEMBLY: call void @llvm.wasm.prefetch.nt(i8* %p)
+}

llvm/include/llvm/IR/IntrinsicsWebAssembly.td

 
 // TODO: Remove this intrinsic and the associated builtin if i64x2.eq gets
 // merged to the proposal.
 def int_wasm_eq :
   Intrinsic<[llvm_v2i64_ty],
             [llvm_v2i64_ty, llvm_v2i64_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
+// TODO: Remove this after experiments have been run. Use the target-agnostic
+// int_prefetch if this becomes specified at some point.
+def int_wasm_prefetch_t :
+  Intrinsic<[], [llvm_ptr_ty],
+            [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn,
+             ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
+            "", [SDNPMemOperand]>;
+
+def int_wasm_prefetch_nt :
+  Intrinsic<[], [llvm_ptr_ty],
+            [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn,
+             ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
+            "", [SDNPMemOperand]>;
+
 //===----------------------------------------------------------------------===//
 // Thread-local storage intrinsics
 //===----------------------------------------------------------------------===//
 
 def int_wasm_tls_size :
   Intrinsic<[llvm_anyint_ty],
             [],
             [IntrNoMem, IntrSpeculatable]>;

llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp

Operands.push_back(std::make_unique<WebAssemblyOperand>(
         WebAssemblyOperand::FltOp{Val}));
     Parser.Lex();
     return false;
   }
 
   bool checkForP2AlignIfLoadStore(OperandVector &Operands, StringRef InstName) {
     // FIXME: there is probably a cleaner way to do this.
     auto IsLoadStore = InstName.find(".load") != StringRef::npos ||
-                       InstName.find(".store") != StringRef::npos;
+                       InstName.find(".store") != StringRef::npos ||
+                       InstName.find("prefetch") != StringRef::npos;
     auto IsAtomic = InstName.find("atomic.") != StringRef::npos;
     if (IsLoadStore || IsAtomic) {
       // Parse load/store operands of the form: offset:p2align=align
       if (IsLoadStore && isNext(AsmToken::Colon)) {
         auto Id = expectIdent();
         if (Id != "p2align")
           return error("Expected p2align, instead got: " + Id);
         if (expect(AsmToken::Equal, "="))

llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h

#define WASM_LOAD_STORE(NAME) \
   WASM_LOAD_STORE(ATOMIC_RMW8_U_XOR_I64)
   WASM_LOAD_STORE(ATOMIC_RMW8_U_XCHG_I32)
   WASM_LOAD_STORE(ATOMIC_RMW8_U_XCHG_I64)
   WASM_LOAD_STORE(ATOMIC_RMW8_U_CMPXCHG_I32)
   WASM_LOAD_STORE(ATOMIC_RMW8_U_CMPXCHG_I64)
   WASM_LOAD_STORE(LOAD8_SPLAT)
   WASM_LOAD_STORE(LOAD_LANE_I8x16)
   WASM_LOAD_STORE(STORE_LANE_I8x16)
+  WASM_LOAD_STORE(PREFETCH_T)
+  WASM_LOAD_STORE(PREFETCH_NT)
   return 0;
   WASM_LOAD_STORE(LOAD16_S_I32)
   WASM_LOAD_STORE(LOAD16_U_I32)
   WASM_LOAD_STORE(LOAD16_S_I64)
   WASM_LOAD_STORE(LOAD16_U_I64)
   WASM_LOAD_STORE(ATOMIC_LOAD16_U_I32)
   WASM_LOAD_STORE(ATOMIC_LOAD16_U_I64)
   WASM_LOAD_STORE(STORE16_I32)

llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

if (Intrinsic == Intrinsic::wasm_load8_lane ||
       Info.flags = MachineMemOperand::MOStore;
     }
     Info.ptrVal = I.getArgOperand(0);
     Info.memVT = MemVT;
     Info.offset = 0;
     Info.align = MemAlign;
     return true;
   }
+  case Intrinsic::wasm_prefetch_t:
+  case Intrinsic::wasm_prefetch_nt: {
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::i8;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = Align(1);
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
+  }
   default:
     return false;
   }
 }
 
 //===----------------------------------------------------------------------===//
 // WebAssembly Lowering private implementation.
 //===----------------------------------------------------------------------===//

llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

 defm "" : SIMDQFM<F64x2, 254, 255>;
 
 //===----------------------------------------------------------------------===//
 // Saturating Rounding Q-Format Multiplication
 //===----------------------------------------------------------------------===//
 
 defm Q15MULR_SAT_S :
   SIMDBinary<I16x8, int_wasm_q15mulr_saturate_signed, "q15mulr_sat_s", 156>;
+
+//===----------------------------------------------------------------------===//
+// Experimental prefetch instructions: prefetch.t, prefetch.nt
+//===----------------------------------------------------------------------===//
+
+let mayLoad = true, UseNamedOperandTable = true in {
+defm PREFETCH_T_A32 :
+  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+         "prefetch.t\t${off}(${addr})$p2align",
+         "prefetch.t\t$off$p2align", 0xc5>;
+defm PREFETCH_T_A64 :
+  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
+         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
+         "prefetch.t\t${off}(${addr})$p2align",
+         "prefetch.t\t$off$p2align", 0xc5>;
+defm PREFETCH_NT_A32 :
+  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+         "prefetch.nt\t${off}(${addr})$p2align",
+         "prefetch.nt\t$off$p2align", 0xc6>;
+defm PREFETCH_NT_A64 :
+  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
+         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
+         "prefetch.nt\t${off}(${addr})$p2align",
+         "prefetch.nt\t$off$p2align", 0xc6>;
+} // mayLoad, UseNamedOperandTable
+
+multiclass PrefetchPatNoOffset<PatFrag kind, string inst> {
+  def : Pat<(kind I32:$addr), (!cast<NI>(inst # "_A32") 0, 0, $addr)>,
+        Requires<[HasAddr32]>;
+  def : Pat<(kind I64:$addr), (!cast<NI>(inst # "_A64") 0, 0, $addr)>,
+        Requires<[HasAddr64]>;
+}
+
+foreach inst = [["PREFETCH_T", "int_wasm_prefetch_t"],
+                ["PREFETCH_NT", "int_wasm_prefetch_nt"]] in {
+defvar node = !cast<Intrinsic>(inst[1]);
+defm : PrefetchPatNoOffset<node, inst[0]>;
+}

llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
+
+; Test experimental prefetch instructions
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare void @llvm.wasm.prefetch.t(i8*)
+declare void @llvm.wasm.prefetch.nt(i8*)
+@gv = global i8 0
+
+;===----------------------------------------------------------------------------
+; prefetch.t
+;===----------------------------------------------------------------------------
+
+define void @prefetch_t_no_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_no_offset:
+; CHECK:         .functype prefetch_t_no_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  tail call void @llvm.wasm.prefetch.t(i8* %p)
+  ret void
+}
+
+define void @prefetch_t_with_folded_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_with_folded_offset:
+; CHECK:         .functype prefetch_t_with_folded_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_with_folded_gep_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_with_folded_gep_offset:
+; CHECK:         .functype prefetch_t_with_folded_gep_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i8, i8* %p, i32 6
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_with_unfolded_gep_negative_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_with_unfolded_gep_negative_offset:
+; CHECK:         .functype prefetch_t_with_unfolded_gep_negative_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i8, i8* %p, i32 -6
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_with_unfolded_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_with_unfolded_offset:
+; CHECK:         .functype prefetch_t_with_unfolded_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i8* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_with_unfolded_gep_offset(i8* %p) {
+; CHECK-LABEL: prefetch_t_with_unfolded_gep_offset:
+; CHECK:         .functype prefetch_t_with_unfolded_gep_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr i8, i8* %p, i32 6
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_from_numeric_address() {
+; CHECK-LABEL: prefetch_t_from_numeric_address:
+; CHECK:         .functype prefetch_t_from_numeric_address () -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 42 to i8*
+  tail call void @llvm.wasm.prefetch.t(i8* %s)
+  ret void
+}
+
+define void @prefetch_t_from_global_address() {
+; CHECK-LABEL: prefetch_t_from_global_address:
+; CHECK:         .functype prefetch_t_from_global_address () -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const gv
+; CHECK-NEXT:    prefetch.t 0
+; CHECK-NEXT:    # fallthrough-return
+  tail call void @llvm.wasm.prefetch.t(i8* @gv)
+  ret void
+}
+
+;===----------------------------------------------------------------------------
+; prefetch.nt
+;===----------------------------------------------------------------------------
+
+define void @prefetch_nt_no_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_no_offset:
+; CHECK:         .functype prefetch_nt_no_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  tail call void @llvm.wasm.prefetch.nt(i8* %p)
+  ret void
+}
+
+define void @prefetch_nt_with_folded_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_with_folded_offset:
+; CHECK:         .functype prefetch_nt_with_folded_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_with_folded_gep_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_with_folded_gep_offset:
+; CHECK:         .functype prefetch_nt_with_folded_gep_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i8, i8* %p, i64 6
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_with_unfolded_gep_negative_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_with_unfolded_gep_negative_offset:
+; CHECK:         .functype prefetch_nt_with_unfolded_gep_negative_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i8, i8* %p, i64 -6
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_with_unfolded_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_with_unfolded_offset:
+; CHECK:         .functype prefetch_nt_with_unfolded_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i8* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_with_unfolded_gep_offset(i8* %p) {
+; CHECK-LABEL: prefetch_nt_with_unfolded_gep_offset:
+; CHECK:         .functype prefetch_nt_with_unfolded_gep_offset (i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 6
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr i8, i8* %p, i64 6
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_from_numeric_address() {
+; CHECK-LABEL: prefetch_nt_from_numeric_address:
+; CHECK:         .functype prefetch_nt_from_numeric_address () -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 42 to i8*
+  tail call void @llvm.wasm.prefetch.nt(i8* %s)
+  ret void
+}
+
+define void @prefetch_nt_from_global_address() {
+; CHECK-LABEL: prefetch_nt_from_global_address:
+; CHECK:         .functype prefetch_nt_from_global_address () -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const gv
+; CHECK-NEXT:    prefetch.nt 0
+; CHECK-NEXT:    # fallthrough-return
+  tail call void @llvm.wasm.prefetch.nt(i8* @gv)
+  ret void
+}

llvm/test/MC/WebAssembly/simd-encodings.s

main:
     i16x8.extadd_pairwise_i8x16_u
 
     # CHECK: i32x4.extadd_pairwise_i16x8_s # encoding: [0xfd,0xa5,0x01]
     i32x4.extadd_pairwise_i16x8_s
 
     # CHECK: i32x4.extadd_pairwise_i16x8_u # encoding: [0xfd,0xa6,0x01]
     i32x4.extadd_pairwise_i16x8_u
 
+    # CHECK: prefetch.t 16 # encoding: [0xfd,0xc5,0x01,0x00,0x10]
+    prefetch.t 16
+
+    # CHECK: prefetch.nt 16 # encoding: [0xfd,0xc6,0x01,0x00,0x10]
+    prefetch.nt 16
+
     end_function