[WebAssembly] Support for atomic.wait / atomic.wake instructions

[aheejin]

Summary

This adds support for [[ https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md#wait | atomic.wait ]] / [[ https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md#wake | atomic.wake ]] instructions in the wasm thread proposal.

[aheejin]

• Type fix

[aheejin]

Should've included all sorts of other memargs.

[aheejin]

• Add support for various memargs

[aheejin]

clang-format

[aheejin]

Does atomic.wake possibly load the given address? It is marked as such for now to be safe. Would it have any problems, or would it be unnecessary?

[aheejin]

Should make it return true on isAtomic() I guess

[aheejin]

I guess there's no sane way to make MachineMemOperand of wait and wake instructions return true on isAtomic() anyway. First, LLVM currently does not seem to provide atomic memory intrinsic other than few predefined target-independent intrinsics, and there's no code that depends on MachineMemOperand::isAtomic() in the backend passes anyway.

[dschuff]

Note that in JS, atomics.wake is now atomics.notify (https://github.com/tc39/ecma262/pull/1220). So we should use the new terminology for new and internal stuff. Since there's no legacy yet (except maybe in Binaryen's and WABT's text parser) hopefully LLVM can just use "notify" everywhere.

I think wake does not load the given address. the list of waiters is outside of the memory and invisible to LLVM, it just happens to share the same index space. So I think hasSideEffects is the right way to model that.

include/llvm/IR/IntrinsicsWebAssembly.td

75

Same signature adjustments as suggested in the clang CL.

[aheejin]

• wake -> notify / etc

[aheejin]

Do you think we should also suggest to change wake to notify in the thread proposal?

include/llvm/IR/IntrinsicsWebAssembly.td

75

I added some comments in that CL, PTAL.

[aheejin]

rebase

[dschuff]

I would imagine that's already on Ben's plate to do sometime, but yes it should be changed to match JS.

include/llvm/IR/IntrinsicsWebAssembly.td

75

Yeah, so this signature would be unchanged then, since there's no signed/unsigned distinction in the IR either.

[aheejin]

Added MOVolatile flags to instructions and added comments

[aheejin]

I added MOVolatile flags to instructions, because, currently LLVM atomic instructions are marked as such. References: 1 2
I also added some comments on why wake is marked as MOLoad. PTAL

[dschuff]

Still looks good 👍

include/llvm/IR/IntrinsicsWebAssembly.td

 
 // WebAssembly EH must maintain the landingpads in the order assigned to them
 // by WasmEHPrepare pass to generate landingpad table in EHStreamer. This is
 // used in order to give them the indices in WasmEHPrepare.
 def int_wasm_landingpad_index: Intrinsic<[], [llvm_i32_ty], [IntrNoMem]>;
 
 // Returns LSDA address of the current function.
 def int_wasm_lsda : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>;
+
+//===----------------------------------------------------------------------===//
+// Atomic intrinsics
+//===----------------------------------------------------------------------===//
+
+// wait / notify
+def int_wasm_atomic_wait_i32 :
+  Intrinsic<[llvm_i32_ty],
+            [LLVMPointerType<llvm_i32_ty>, llvm_i32_ty, llvm_i64_ty],
+            [IntrInaccessibleMemOrArgMemOnly, ReadOnly<0>, NoCapture<0>,
+             IntrHasSideEffects],
+             "", [SDNPMemOperand]>;
+def int_wasm_atomic_wait_i64 :
+  Intrinsic<[llvm_i32_ty],
+            [LLVMPointerType<llvm_i64_ty>, llvm_i64_ty, llvm_i64_ty],
+            [IntrInaccessibleMemOrArgMemOnly, ReadOnly<0>, NoCapture<0>,
+             IntrHasSideEffects],
+             "", [SDNPMemOperand]>;
+def int_wasm_atomic_notify:
+  Intrinsic<[llvm_i64_ty], [LLVMPointerType<llvm_i32_ty>, llvm_i64_ty],
+            [IntrInaccessibleMemOnly, NoCapture<0>, IntrHasSideEffects], "",
+            [SDNPMemOperand]>;
+
 }

lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h

inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW_XCHG_I32:
   case WebAssembly::ATOMIC_RMW_XCHG_I32_S:
   case WebAssembly::ATOMIC_RMW32_U_XCHG_I64:
   case WebAssembly::ATOMIC_RMW32_U_XCHG_I64_S:
   case WebAssembly::ATOMIC_RMW_CMPXCHG_I32:
   case WebAssembly::ATOMIC_RMW_CMPXCHG_I32_S:
   case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64:
   case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64_S:
+  case WebAssembly::ATOMIC_NOTIFY:
+  case WebAssembly::ATOMIC_NOTIFY_S:
+  case WebAssembly::ATOMIC_WAIT_I32:
+  case WebAssembly::ATOMIC_WAIT_I32_S:
     return 2;
   case WebAssembly::LOAD_I64:
   case WebAssembly::LOAD_I64_S:
   case WebAssembly::LOAD_F64:
   case WebAssembly::LOAD_F64_S:
   case WebAssembly::STORE_I64:
   case WebAssembly::STORE_I64_S:
   case WebAssembly::STORE_F64:
inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_RMW_OR_I64:
   case WebAssembly::ATOMIC_RMW_OR_I64_S:
   case WebAssembly::ATOMIC_RMW_XOR_I64:
   case WebAssembly::ATOMIC_RMW_XOR_I64_S:
   case WebAssembly::ATOMIC_RMW_XCHG_I64:
   case WebAssembly::ATOMIC_RMW_XCHG_I64_S:
   case WebAssembly::ATOMIC_RMW_CMPXCHG_I64:
   case WebAssembly::ATOMIC_RMW_CMPXCHG_I64_S:
+  case WebAssembly::ATOMIC_WAIT_I64:
+  case WebAssembly::ATOMIC_WAIT_I64_S:
     return 3;
   default:
     llvm_unreachable("Only loads and stores have p2align values");
   }
 }
 
 /// The operand number of the load or store address in load/store instructions.
 static const unsigned LoadAddressOperandNo = 3;

lib/Target/WebAssembly/WebAssemblyISelLowering.h

bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
                              unsigned AS,
                              Instruction *I = nullptr) const override;
   bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace, unsigned Align,
                                       bool *Fast) const override;
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
 
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+                          MachineFunction &MF,
+                          unsigned Intrinsic) const override;
 
   SDValue LowerCall(CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
   bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
                       bool isVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       LLVMContext &Context) const override;
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,

lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
                                                   LLVMContext &C,
                                                   EVT VT) const {
   if (VT.isVector())
     return VT.changeVectorElementTypeToInteger();
 
   return TargetLowering::getSetCCResultType(DL, C, VT);
 }
 
+bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
+                                                   const CallInst &I,
+                                                   MachineFunction &MF,
+                                                   unsigned Intrinsic) const {
+  switch (Intrinsic) {
+  case Intrinsic::wasm_atomic_notify:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i32;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 4;
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
+  case Intrinsic::wasm_atomic_wait_i32:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i32;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 4;
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
+  case Intrinsic::wasm_atomic_wait_i64:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::i64;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = 8;
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
+  default:
+    return false;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // WebAssembly Lowering private implementation.
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // Lowering Code
 //===----------------------------------------------------------------------===//
 

lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

 }
 
 let Predicates = [HasAtomics] in {
 defm : TerRMWTruncExtPattern<
   atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64,
   ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32,
   ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64,
   ATOMIC_RMW32_U_CMPXCHG_I64>;
+
+//===----------------------------------------------------------------------===//
+// Atomic wait / notify
+//===----------------------------------------------------------------------===//
+
+let Defs = [ARGUMENTS] in {
+let hasSideEffects = 1 in {
+defm ATOMIC_NOTIFY :
+  I<(outs I64:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$count),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count",
+    "atomic.notify \t${off}, ${p2align}", 0xfe00>;
+let mayLoad = 1 in {
+defm ATOMIC_WAIT_I32 :
+  I<(outs I32:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
+    "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>;
+defm ATOMIC_WAIT_I64 :
+  I<(outs I32:$dst),
+    (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout),
+    (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+    "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
+    "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>;
+} // mayLoad = 1
+} // hasSideEffects = 1
+} // Defs = [ARGUMENTS]
+
+let Predicates = [HasAtomics] in {
+// Select notifys with no constant offset.
+class NotifyPatNoOffset<Intrinsic kind> :
+  Pat<(i64 (kind I32:$addr, I64:$count)),
+      (ATOMIC_NOTIFY 0, 0, I32:$addr, I64:$count)>;
+def : NotifyPatNoOffset<int_wasm_atomic_notify>;
+
+// Select notifys with a constant offset.
+
+// Pattern with address + immediate offset
+class NotifyPatImmOff<Intrinsic kind, PatFrag operand> :
+  Pat<(i64 (kind (operand I32:$addr, imm:$off), I64:$count)),
+      (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I64:$count)>;
+def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>;
+def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>;
+
+class NotifyPatGlobalAddr<Intrinsic kind> :
+  Pat<(i64 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                 I64:$count)),
+      (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I64:$count)>;
+def : NotifyPatGlobalAddr<int_wasm_atomic_notify>;
+
+class NotifyPatExternalSym<Intrinsic kind> :
+  Pat<(i64 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                 I64:$count)),
+      (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I64:$count)>;
+def : NotifyPatExternalSym<int_wasm_atomic_notify>;
+
+// Select notifys with just a constant offset.
+class NotifyPatOffsetOnly<Intrinsic kind> :
+  Pat<(i64 (kind imm:$off, I64:$count)),
+      (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I64:$count)>;
+def : NotifyPatOffsetOnly<int_wasm_atomic_notify>;
+
+class NotifyPatGlobalAddrOffOnly<Intrinsic kind> :
+  Pat<(i64 (kind (WebAssemblywrapper tglobaladdr:$off), I64:$count)),
+      (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I64:$count)>;
+def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>;
+
+class NotifyPatExternSymOffOnly<Intrinsic kind> :
+  Pat<(i64 (kind (WebAssemblywrapper texternalsym:$off), I64:$count)),
+      (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I64:$count)>;
+def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>;
+
+// Select waits with no constant offset.
+class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)),
+      (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+// Select waits with a constant offset.
+
+// Pattern with address + immediate offset
+class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> :
+  Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)),
+      (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>;
+def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>;
+def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>;
+def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>;
+
+class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                 ty:$exp, I64:$timeout)),
+      (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                 ty:$exp, I64:$timeout)),
+      (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>;
+def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset.
+class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
+      (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)),
+      (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
+
+class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> :
+  Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp,
+                 I64:$timeout)),
+      (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>;
+def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
+def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
 } // Predicates = [HasAtomics]

lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp

for (auto &MI : MBB) {
       case WebAssembly::ATOMIC_RMW32_U_CMPXCHG_I64:
       case WebAssembly::ATOMIC_RMW_ADD_I64:
       case WebAssembly::ATOMIC_RMW_SUB_I64:
       case WebAssembly::ATOMIC_RMW_AND_I64:
       case WebAssembly::ATOMIC_RMW_OR_I64:
       case WebAssembly::ATOMIC_RMW_XOR_I64:
       case WebAssembly::ATOMIC_RMW_XCHG_I64:
       case WebAssembly::ATOMIC_RMW_CMPXCHG_I64:
+      case WebAssembly::ATOMIC_NOTIFY:
+      case WebAssembly::ATOMIC_WAIT_I32:
+      case WebAssembly::ATOMIC_WAIT_I64:
         RewriteP2Align(MI, WebAssembly::LoadP2AlignOperandNo);
         break;
       case WebAssembly::STORE_I32:
       case WebAssembly::STORE_I64:
       case WebAssembly::STORE_F32:
       case WebAssembly::STORE_F64:
       case WebAssembly::STORE8_I32:
       case WebAssembly::STORE16_I32:

test/CodeGen/WebAssembly/offset-atomics.ll

 define i32 @cmpxchg_i8_i32_z_from_global_address(i32 %exp, i32 %new) {
   %exp_t = trunc i32 %exp to i8
   %new_t = trunc i32 %new to i8
   %pair = cmpxchg i8* @gv8, i8 %exp_t, i8 %new_t seq_cst seq_cst
   %old = extractvalue { i8, i1 } %pair, 0
   %u = zext i8 %old to i32
   ret i32 %u
 }
+
+;===----------------------------------------------------------------------------
+; Waits: 32-bit
+;===----------------------------------------------------------------------------
+
+declare i32 @llvm.wasm.atomic.wait.i32(i32*, i32, i64)
+
+; Basic wait.
+
+; CHECK-LABEL: wait_i32_no_offset:
+; CHECK: i32.atomic.wait $push0=, 0($0), $1, $2{{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i32 @wait_i32_no_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %v = call i32 @llvm.wasm.atomic.wait.i32(i32* %p, i32 %exp, i64 %timeout)
+  ret i32 %v
+}
+
+; With an nuw add, we can fold an offset.
+
+; CHECK-LABEL: wait_i32_with_folded_offset:
+; CHECK: i32.atomic.wait $push0=, 24($0), $1, $2{{$}}
+define i32 @wait_i32_with_folded_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; With an inbounds gep, we can fold an offset.
+
+; CHECK-LABEL: wait_i32_with_folded_gep_offset:
+; CHECK: i32.atomic.wait $push0=, 24($0), $1, $2{{$}}
+define i32 @wait_i32_with_folded_gep_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; We can't fold a negative offset though, even with an inbounds gep.
+
+; CHECK-LABEL: wait_i32_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i32_with_unfolded_gep_negative_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; Without nuw, and even with nsw, we can't fold an offset.
+
+; CHECK-LABEL: wait_i32_with_unfolded_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i32_with_unfolded_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; Without inbounds, we can't fold a gep offset.
+
+; CHECK-LABEL: wait_i32_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i32_with_unfolded_gep_offset(i32* %p, i32 %exp, i64 %timeout) {
+  %s = getelementptr i32, i32* %p, i32 6
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; When waiting from a fixed address, materialize a zero.
+
+; CHECK-LABEL: wait_i32_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.wait $push1=, 42($pop0), $0, $1{{$}}
+define i32 @wait_i32_from_numeric_address(i32 %exp, i64 %timeout) {
+  %s = inttoptr i32 42 to i32*
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* %s, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; CHECK-LABEL: wait_i32_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.wait $push1=, gv($pop0), $0, $1{{$}}
+define i32 @wait_i32_from_global_address(i32 %exp, i64 %timeout) {
+  %t = call i32 @llvm.wasm.atomic.wait.i32(i32* @gv, i32 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+;===----------------------------------------------------------------------------
+; Waits: 64-bit
+;===----------------------------------------------------------------------------
+
+declare i32 @llvm.wasm.atomic.wait.i64(i64*, i64, i64)
+
+; Basic wait.
+
+; CHECK-LABEL: wait_i64_no_offset:
+; CHECK: i64.atomic.wait $push0=, 0($0), $1, $2{{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i32 @wait_i64_no_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %v = call i32 @llvm.wasm.atomic.wait.i64(i64* %p, i64 %exp, i64 %timeout)
+  ret i32 %v
+}
+
+; With an nuw add, we can fold an offset.
+
+; CHECK-LABEL: wait_i64_with_folded_offset:
+; CHECK: i64.atomic.wait $push0=, 24($0), $1, $2{{$}}
+define i32 @wait_i64_with_folded_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  %t = call i32 @llvm.wasm.atomic.wait.i64(i64* %s, i64 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; With an inbounds gep, we can fold an offset.
+
+; CHECK-LABEL: wait_i64_with_folded_gep_offset:
+; CHECK: i64.atomic.wait $push0=, 24($0), $1, $2{{$}}
+define i32 @wait_i64_with_folded_gep_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %s = getelementptr inbounds i64, i64* %p, i32 3
+  %t = call i32 @llvm.wasm.atomic.wait.i64(i64* %s, i64 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; We can't fold a negative offset though, even with an inbounds gep.
+
+; CHECK-LABEL: wait_i64_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i64_with_unfolded_gep_negative_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -3
+  %t = call i32 @llvm.wasm.atomic.wait.i64(i64* %s, i64 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; Without nuw, and even with nsw, we can't fold an offset.
+
+; CHECK-LABEL: wait_i64_with_unfolded_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i64_with_unfolded_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  %t = call i32 @llvm.wasm.atomic.wait.i64(i64* %s, i64 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+; Without inbounds, we can't fold a gep offset.
+
+; CHECK-LABEL: wait_i64_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.wait $push2=, 0($pop1), $1, $2{{$}}
+define i32 @wait_i64_with_unfolded_gep_offset(i64* %p, i64 %exp, i64 %timeout) {
+  %s = getelementptr i64, i64* %p, i32 3
+  %t = call i32 @llvm.wasm.atomic.wait.i64(i64* %s, i64 %exp, i64 %timeout)
+  ret i32 %t
+}
+
+;===----------------------------------------------------------------------------
+; Notifies
+;===----------------------------------------------------------------------------
+
+declare i64 @llvm.wasm.atomic.notify(i32*, i64)
+
+; Basic notify.
+
+; CHECK-LABEL: notify_no_offset:
+; CHECK: atomic.notify $push0=, 0($0), $1{{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i64 @notify_no_offset(i32* %p, i64 %notify_count) {
+  %v = call i64 @llvm.wasm.atomic.notify(i32* %p, i64 %notify_count)
+  ret i64 %v
+}
+
+; With an nuw add, we can fold an offset.
+
+; CHECK-LABEL: notify_with_folded_offset:
+; CHECK: atomic.notify $push0=, 24($0), $1{{$}}
+define i64 @notify_with_folded_offset(i32* %p, i64 %notify_count) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; With an inbounds gep, we can fold an offset.
+
+; CHECK-LABEL: notify_with_folded_gep_offset:
+; CHECK: atomic.notify $push0=, 24($0), $1{{$}}
+define i64 @notify_with_folded_gep_offset(i32* %p, i64 %notify_count) {
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; We can't fold a negative offset though, even with an inbounds gep.
+
+; CHECK-LABEL: notify_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: atomic.notify $push2=, 0($pop1), $1{{$}}
+define i64 @notify_with_unfolded_gep_negative_offset(i32* %p, i64 %notify_count) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; Without nuw, and even with nsw, we can't fold an offset.
+
+; CHECK-LABEL: notify_with_unfolded_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: atomic.notify $push2=, 0($pop1), $1{{$}}
+define i64 @notify_with_unfolded_offset(i32* %p, i64 %notify_count) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; Without inbounds, we can't fold a gep offset.
+
+; CHECK-LABEL: notify_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: atomic.notify $push2=, 0($pop1), $1{{$}}
+define i64 @notify_with_unfolded_gep_offset(i32* %p, i64 %notify_count) {
+  %s = getelementptr i32, i32* %p, i32 6
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; When notifying from a fixed address, materialize a zero.
+
+; CHECK-LABEL: notify_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: atomic.notify $push1=, 42($pop0), $0{{$}}
+define i64 @notify_from_numeric_address(i64 %notify_count) {
+  %s = inttoptr i32 42 to i32*
+  %t = call i64 @llvm.wasm.atomic.notify(i32* %s, i64 %notify_count)
+  ret i64 %t
+}
+
+; CHECK-LABEL: notify_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: atomic.notify $push1=, gv($pop0), $0{{$}}
+define i64 @notify_from_global_address(i64 %notify_count) {
+  %t = call i64 @llvm.wasm.atomic.notify(i32* @gv, i64 %notify_count)
+  ret i64 %t
+}