[WebAssembly] Use ComplexPattern on remaining memory instructions

[luke]

Summary

This continues the refactoring work of selecting offset + address operands with the AddrOpsN pattern, previously called LoadOpsN.

This is not an NFC, since constant addresses are now folded into the offset in more places for v128.storeN_lane.

[luke]

Update comments to be more clear

[aheejin]

Nice!

llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

14–33

It looks we don't convert atomic stores in this CL.. Is that going to be a follow-up, or is that not included in the plan? Is converting them different from the normal stores?

llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td

24–35

Nit: 80 col wrapping

[luke]

llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

14–33

I’ve been working on atomic as part of a separate diff, but happy to include them in this one if that’s easier!
The conversion process is the same, and for the atomic instructions you’ll be glad to hear it saves a lot of tablegen.

[luke]

llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td

24–35

I think this is wrapped to 80 columns, unfortunately just a poor choice of long, awkward to wrap words :(

[luke]

Use AddrOps on atomic instructions

[luke]

Include both commits

[luke]

Rebase

[aheejin]

Thank you! This looks certainly easier to read and manage.

llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp

public:
 
   void PreprocessISelDAG() override;
 
   void Select(SDNode *Node) override;
 
   bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
                                     std::vector<SDValue> &OutOps) override;
 
-  bool SelectLoadOperands32(SDValue Op, SDValue &Offset, SDValue &Addr);
-  bool SelectLoadOperands64(SDValue Op, SDValue &Offset, SDValue &Addr);
+  bool SelectAddrOperands32(SDValue Op, SDValue &Offset, SDValue &Addr);
+  bool SelectAddrOperands64(SDValue Op, SDValue &Offset, SDValue &Addr);
 
 // Include the pieces autogenerated from the target description.
 #include "WebAssemblyGenDAGISel.inc"
 
 private:
   // add select functions here...
 
-  bool SelectLoadOperands(MVT AddrType, unsigned ConstOpc, SDValue Op,
+  bool SelectAddrOperands(MVT AddrType, unsigned ConstOpc, SDValue Op,
                           SDValue &Offset, SDValue &Addr);
-  bool SelectLoadAddOperands(MVT OffsetType, SDValue N, SDValue &Offset,
+  bool SelectAddrAddOperands(MVT OffsetType, SDValue N, SDValue &Offset,
                              SDValue &Addr);
 };
 } // end anonymous namespace
 
 void WebAssemblyDAGToDAGISel::PreprocessISelDAG() {
   // Stack objects that should be allocated to locals are hoisted to WebAssembly
   // locals when they are first used.  However for those without uses, we hoist
   // them here.  It would be nice if there were some hook to do this when they
case InlineAsm::Constraint_m:
     return false;
   default:
     break;
   }
 
   return true;
 }
 
-bool WebAssemblyDAGToDAGISel::SelectLoadAddOperands(MVT OffsetType, SDValue N,
+bool WebAssemblyDAGToDAGISel::SelectAddrAddOperands(MVT OffsetType, SDValue N,
                                                     SDValue &Offset,
                                                     SDValue &Addr) {
   assert(N.getNumOperands() == 2 && "Attempting to fold in a non-binary op");
 
   // WebAssembly constant offsets are performed as unsigned with infinite
   // precision, so we need to check for NoUnsignedWrap so that we don't fold an
   // offset for an add that needs wrapping.
   if (N.getOpcode() == ISD::ADD && !N.getNode()->getFlags().hasNoUnsignedWrap())
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
           CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), OffsetType);
       Addr = OtherOp;
       return true;
     }
   }
   return false;
 }
 
-bool WebAssemblyDAGToDAGISel::SelectLoadOperands(MVT AddrType,
+bool WebAssemblyDAGToDAGISel::SelectAddrOperands(MVT AddrType,
                                                  unsigned ConstOpc, SDValue N,
                                                  SDValue &Offset,
                                                  SDValue &Addr) {
   SDLoc DL(N);
 
   // Fold target global addresses into the offset.
   if (!TM.isPositionIndependent()) {
     SDValue Op(N);
     if (Op.getOpcode() == WebAssemblyISD::Wrapper)
       Op = Op.getOperand(0);
 
     if (Op.getOpcode() == ISD::TargetGlobalAddress) {
       Offset = Op;
       Addr = SDValue(
           CurDAG->getMachineNode(ConstOpc, DL, AddrType,
                                  CurDAG->getTargetConstant(0, DL, AddrType)),
           0);
       return true;
     }
   }
 
   // Fold anything inside an add into the offset.
   if (N.getOpcode() == ISD::ADD &&
-      SelectLoadAddOperands(AddrType, N, Offset, Addr))
+      SelectAddrAddOperands(AddrType, N, Offset, Addr))
     return true;
 
   // Likewise, treat an 'or' node as an 'add' if the or'ed bits are known to be
   // zero and fold them into the offset too.
   if (N.getOpcode() == ISD::OR) {
     bool OrIsAdd;
     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
       OrIsAdd =
           CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
     } else {
       KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
       KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
       OrIsAdd = (~Known0.Zero & ~Known1.Zero) == 0;
     }
 
-    if (OrIsAdd && SelectLoadAddOperands(AddrType, N, Offset, Addr))
+    if (OrIsAdd && SelectAddrAddOperands(AddrType, N, Offset, Addr))
       return true;
   }
 
   // Fold constant addresses into the offset.
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) {
     Offset = CurDAG->getTargetConstant(CN->getZExtValue(), DL, AddrType);
     Addr = SDValue(
         CurDAG->getMachineNode(ConstOpc, DL, AddrType,
                                CurDAG->getTargetConstant(0, DL, AddrType)),
         0);
     return true;
   }
 
-  // Else it's a plain old load with no offset.
+  // Else it's a plain old load/store with no offset.
   Offset = CurDAG->getTargetConstant(0, DL, AddrType);
   Addr = N;
   return true;
 }
 
-bool WebAssemblyDAGToDAGISel::SelectLoadOperands32(SDValue Op, SDValue &Offset,
+bool WebAssemblyDAGToDAGISel::SelectAddrOperands32(SDValue Op, SDValue &Offset,
                                                    SDValue &Addr) {
-  return SelectLoadOperands(MVT::i32, WebAssembly::CONST_I32, Op, Offset, Addr);
+  return SelectAddrOperands(MVT::i32, WebAssembly::CONST_I32, Op, Offset, Addr);
 }
 
-bool WebAssemblyDAGToDAGISel::SelectLoadOperands64(SDValue Op, SDValue &Offset,
+bool WebAssemblyDAGToDAGISel::SelectAddrOperands64(SDValue Op, SDValue &Offset,
                                                    SDValue &Addr) {
-  return SelectLoadOperands(MVT::i64, WebAssembly::CONST_I64, Op, Offset, Addr);
+  return SelectAddrOperands(MVT::i64, WebAssembly::CONST_I64, Op, Offset, Addr);
 }
 
 /// This pass converts a legalized DAG into a WebAssembly-specific DAG, ready
 /// for instruction scheduling.
 FunctionPass *llvm::createWebAssemblyISelDag(WebAssemblyTargetMachine &TM,
                                              CodeGenOpt::Level OptLevel) {
   return new WebAssemblyDAGToDAGISel(TM, OptLevel);
 }

llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

 // WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*-
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// WebAssembly Atomic operand code-gen constructs.
 ///
 //===----------------------------------------------------------------------===//
 
 let UseNamedOperandTable = 1 in
 multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
                     list<dag> pattern_r, string asmstr_r,
                     string asmstr_s, bits<32> atomic_op,
                     bit is64 = false> {
   defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
               !or(0xfe00, !and(0xff, atomic_op)), is64>,
             Requires<[HasAtomics]>;
 }
 
 multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "",
                       bits<32> atomic_op = -1> {
   defm "" : NRI<oops, iops, pattern, asmstr,
                 !or(0xfe00, !and(0xff, atomic_op))>,
             Requires<[HasAtomics]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Atomic wait / notify
 //===----------------------------------------------------------------------===//
 
 let hasSideEffects = 1 in {
 defm MEMORY_ATOMIC_NOTIFY_A32 :
   ATOMIC_I<(outs I32:$dst),
            (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count),
            (outs), (ins P2Align:$p2align, offset32_op:$off), [],
            "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count",
            "memory.atomic.notify \t${off}${p2align}", 0x00, false>;
ATOMIC_I<(outs I32:$dst),
            (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp,
                 I64:$timeout),
            (outs), (ins P2Align:$p2align, offset64_op:$off), [],
            "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout",
            "memory.atomic.wait64 \t${off}${p2align}", 0x02, true>;
 } // mayLoad = 1
 } // hasSideEffects = 1
 
-// Select notifys with no constant offset.
-def NotifyPatNoOffset_A32 :
-  Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>,
+def NotifyPat_A32 :
+  Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps32 offset32_op:$offset, I32:$addr), I32:$count)),
+      (MEMORY_ATOMIC_NOTIFY_A32 0, $offset, $addr, $count)>,
   Requires<[HasAddr32, HasAtomics]>;
-def NotifyPatNoOffset_A64 :
-  Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>,
+def NotifyPat_A64 :
+  Pat<(i32 (int_wasm_memory_atomic_notify (AddrOps64 offset64_op:$offset, I64:$addr), I32:$count)),
+      (MEMORY_ATOMIC_NOTIFY_A64 0, $offset, $addr, $count)>,
   Requires<[HasAddr64, HasAtomics]>;
 
-// Select notifys with a constant offset.
 
-// Pattern with address + immediate offset
-multiclass NotifyPatImmOff<PatFrag operand, string inst> {
-  def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off),
-                  I32:$count)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off),
-                  I32:$count)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>,
-        Requires<[HasAddr64, HasAtomics]>;
+multiclass WaitPat<ValueType ty, Intrinsic kind, string inst> {
+  def WaitPat_A32 :
+    Pat<(i32 (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, I64:$timeout)),
+        (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $timeout)>,
+  def WaitPat_A64 :
+    Pat<(i32 (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, I64:$timeout)),
+        (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $timeout)>,
 }
-defm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">;
-defm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">;
 
-// Select notifys with just a constant offset.
-def NotifyPatOffsetOnly_A32 :
-  Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>,
-  Requires<[HasAddr32, HasAtomics]>;
-def NotifyPatOffsetOnly_A64 :
-  Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>,
-  Requires<[HasAddr64, HasAtomics]>;
-
-def NotifyPatGlobalAddrOffOnly_A32 :
-  Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblyWrapper tglobaladdr:$off),
-                                          I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)
-     >,
-  Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
-def NotifyPatGlobalAddrOffOnly_A64 :
-  Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblyWrapper tglobaladdr:$off),
-                                          I32:$count)),
-      (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count)
-     >,
-  Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
-
-// Select waits with no constant offset.
-multiclass WaitPatNoOffset<ValueType ty, Intrinsic kind,
-                      string inst> {
-  def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32,
-                       "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64,
-                       "MEMORY_ATOMIC_WAIT64">;
-defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32,
-                       "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64,
-                       "MEMORY_ATOMIC_WAIT64">;
-
-// Select waits with a constant offset.
-
-// Pattern with address + immediate offset
-multiclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand,
-                         string inst> {
-  def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp,
-              I64:$timeout)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp,
-              I64:$timeout)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm,
-                     "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, or_is_add,
-                     "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, regPlusImm,
-                     "MEMORY_ATOMIC_WAIT64">;
-defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, or_is_add,
-                     "MEMORY_ATOMIC_WAIT64">;
-
-// Select waits with just a constant offset.
-multiclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> {
-  def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp,
-               I64:$timeout)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp,
-               I64:$timeout)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-defm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32,
-                         "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatOffsetOnly<i64, int_wasm_memory_atomic_wait64,
-                         "MEMORY_ATOMIC_WAIT64">;
-
-multiclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> {
-  def : Pat<(i32 (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$exp,
-                  I64:$timeout)),
-            (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp,
-               I64:$timeout)>,
-        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
-  def : Pat<(i32 (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$exp,
-                  I64:$timeout)),
-            (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp,
-               I64:$timeout)>,
-        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
-}
-defm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32,
-                                "MEMORY_ATOMIC_WAIT32">;
-defm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64,
-                                "MEMORY_ATOMIC_WAIT64">;
+defm : WaitPat<i32, int_wasm_memory_atomic_wait32, "MEMORY_ATOMIC_WAIT32">;
+defm : WaitPat<i64, int_wasm_memory_atomic_wait64, "MEMORY_ATOMIC_WAIT64">;
 
 //===----------------------------------------------------------------------===//
 // Atomic fences
 //===----------------------------------------------------------------------===//
 
 // A compiler fence instruction that prevents reordering of instructions.
 let Defs = [ARGUMENTS] in {
 let isPseudo = 1, hasSideEffects = 1 in

 defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>;
 defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>;
 
 // We need an 'atomic' version of store patterns because store and atomic_store
 // nodes have different operand orders:
 // store: (store $val, $ptr)
 // atomic_store: (store $ptr, $val)
 
-
-// Select stores with no constant offset.
-multiclass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(kind I32:$addr, ty:$val),
-            (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(kind I64:$addr, ty:$val),
-            (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-defm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">;
-defm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">;
-
-// Select stores with a constant offset.
-
-// Pattern with address + immediate offset
-multiclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
-                           string inst> {
-  def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val),
-            (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val),
-            (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-defm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">;
-defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">;
-
-// Select stores with just a constant offset.
-multiclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(kind imm:$off, ty:$val),
-            (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>,
+multiclass AStorePat<ValueType ty, PatFrag kind, string inst> {
+  def : Pat<(kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val),
+            (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>,
         Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(kind imm:$off, ty:$val),
-            (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>,
+  def : Pat<(kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val),
+            (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>,
         Requires<[HasAddr64, HasAtomics]>;
 }
-defm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">;
-defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">;
-
-multiclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val),
-            (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
-  def : Pat<(kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val),
-            (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
-}
-defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">;
-defm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">;
-
+defm : AStorePat<i32, atomic_store_32, "ATOMIC_STORE_I32">;
+defm : AStorePat<i64, atomic_store_64, "ATOMIC_STORE_I64">;
 
 // Truncating stores.
 defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>;
 defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>;
 defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>;
 defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>;
 defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>;
 
 // Fragments for truncating stores.
 
 // We don't have single truncating atomic store instructions. For 32-bit
 // instructions, we just need to match bare atomic stores. On the other hand,
 // truncating stores from i64 values are once truncated to i32 first.
 class trunc_astore_64<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$val),
           (kind node:$addr, (i32 (trunc (i64 node:$val))))>;
 def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>;
 def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>;
 def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>;
 
-
 // Truncating stores with no constant offset
-defm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">;
-defm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">;
-defm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">;
-defm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">;
-defm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">;
-
-// Truncating stores with a constant offset
-defm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">;
-defm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">;
-defm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">;
-defm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm,
-                       "ATOMIC_STORE16_I64">;
-defm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm,
-                       "ATOMIC_STORE32_I64">;
-defm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">;
-defm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">;
-defm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">;
-defm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add,
-                       "ATOMIC_STORE16_I64">;
-defm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add,
-                       "ATOMIC_STORE32_I64">;
-
-// Truncating stores with just a constant offset
-defm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">;
-defm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">;
-defm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">;
-defm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">;
-defm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">;
-
-defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">;
-defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">;
-defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">;
-defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">;
-defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">;
-
+defm : AStorePat<i32, atomic_store_8, "ATOMIC_STORE8_I32">;
+defm : AStorePat<i32, atomic_store_16, "ATOMIC_STORE16_I32">;
+defm : AStorePat<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">;
+defm : AStorePat<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">;
+defm : AStorePat<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">;
 
 //===----------------------------------------------------------------------===//
 // Atomic binary read-modify-writes
 //===----------------------------------------------------------------------===//
 
 multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name,
                              int atomic_op> {
   defm "_A32" :
defm ATOMIC_RMW16_U_XCHG_I32 :
   WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>;
 defm ATOMIC_RMW8_U_XCHG_I64 :
   WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>;
 defm ATOMIC_RMW16_U_XCHG_I64 :
   WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>;
 defm ATOMIC_RMW32_U_XCHG_I64 :
   WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>;
 
-// Select binary RMWs with no constant offset.
-multiclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind I32:$addr, ty:$val)),
-            (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(ty (kind I64:$addr, ty:$val)),
-            (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-
-// Select binary RMWs with a constant offset.
-
-// Pattern with address + immediate offset
-multiclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
-                           string inst> {
-  def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-
-// Select binary RMWs with just a constant offset.
-multiclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind imm:$off, ty:$val)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>,
+multiclass BinRMWPat<ValueType ty, PatFrag kind, string inst> {
+  def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val)),
+            (!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>,
         Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(ty (kind imm:$off, ty:$val)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>,
+  def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val)),
+            (!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>,
         Requires<[HasAddr64, HasAtomics]>;
 }
 
-multiclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val)),
-            (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
-  def : Pat<(ty (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val)),
-            (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
-}
-
 // Patterns for various addressing modes.
 multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32,
                          string inst_64> {
-  defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>;
-  defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>;
-
-  defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>;
-  defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>;
-  defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>;
-  defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>;
-
-  defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>;
-  defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>;
-
-  defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>;
+  defm : BinRMWPat<i32, rmw_32, inst_32>;
+  defm : BinRMWPat<i64, rmw_64, inst_64>;
 }
 
 defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64,
                      "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">;
 defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64,
                      "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">;
 defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64,
                      "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">;
PatFrag<(ops node:$addr, node:$val),
           (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>;
 class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>;
 // 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
 
 // Patterns for various addressing modes for truncating-extending binary RMWs.
 multiclass BinRMWTruncExtPattern<
   PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32,
   string inst8_32, string inst16_32, string inst8_64, string inst16_64, string inst32_64> {
-  // Truncating-extending binary RMWs with no constant offset
-  defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>;
-  defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
-
-  // Truncating-extending binary RMWs with a constant offset
-  defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
-  defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm,
-                         inst16_32>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm,
-                         inst16_64>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm,
-                         inst32_64>;
-  defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
-  defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
-  defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>;
-
-  defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
-  defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm,
-                         inst16_32>;
-  defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
-  defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm,
-                         inst16_64>;
-  defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
-  defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
-  defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
-  defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
-
-  // Truncating-extending binary RMWs with just a constant offset
-  defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>;
-  defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
-
-  defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
+  // Truncating-extending binary RMWs
+  defm : BinRMWPat<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
+  defm : BinRMWPat<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
+  defm : BinRMWPat<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>;
+  defm : BinRMWPat<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>;
+  defm : BinRMWPat<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>;
+
+  defm : BinRMWPat<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>;
+  defm : BinRMWPat<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>;
+  defm : BinRMWPat<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>;
+  defm : BinRMWPat<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
 }
 
 defm : BinRMWTruncExtPattern<
   atomic_load_add_8, atomic_load_add_16, atomic_load_add_32,
   "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32",
   "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">;
 defm : BinRMWTruncExtPattern<
   atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32,
defm ATOMIC_RMW16_U_CMPXCHG_I32 :
   WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>;
 defm ATOMIC_RMW8_U_CMPXCHG_I64 :
   WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>;
 defm ATOMIC_RMW16_U_CMPXCHG_I64 :
   WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>;
 defm ATOMIC_RMW32_U_CMPXCHG_I64 :
   WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>;
 
-// Select ternary RMWs with no constant offset.
-multiclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr64, HasAtomics]>;
-}
-
-// Select ternary RMWs with a constant offset.
-
-// Pattern with address + immediate offset
-multiclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
-                           string inst> {
-  def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>,
+multiclass TerRMWPat<ValueType ty, PatFrag kind, string inst> {
+  def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$exp, ty:$new)),
+            (!cast<NI>(inst#_A32) 0, $offset, $addr, $exp, $new)>,
         Requires<[HasAddr32, HasAtomics]>;
-  def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>,
+  def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$exp, ty:$new)),
+            (!cast<NI>(inst#_A64) 0, $offset, $addr, $exp, $new)>,
         Requires<[HasAddr64, HasAtomics]>;
 }
 
-// Select ternary RMWs with just a constant offset.
-multiclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp,
-              ty:$new)>;
-  def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp,
-              ty:$new)>;
-}
-
-multiclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(ty (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp,
-              ty:$new)>,
-        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
-  def : Pat<(ty (kind (WebAssemblyWrapper tglobaladdr:$off), ty:$exp, ty:$new)),
-            (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp,
-              ty:$new)>,
-        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
-}
-
-// Patterns for various addressing modes.
-multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32,
-                         string inst_64> {
-  defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>;
-  defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>;
-
-  defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>;
-  defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>;
-  defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>;
-  defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>;
-
-  defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>;
-  defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>;
-
-  defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>;
-}
-
-defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64,
-                     "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">;
+defm : TerRMWPat<i32, atomic_cmp_swap_32, "ATOMIC_RMW_CMPXCHG_I32">;
+defm : TerRMWPat<i64, atomic_cmp_swap_64, "ATOMIC_RMW_CMPXCHG_I64">;
 
 // Truncating & zero-extending ternary RMW patterns.
 // DAG legalization & optimization before instruction selection may introduce
 // additional nodes such as anyext or assertzext depending on operand types.
 class zext_ter_rmw_8_32<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$exp, node:$new),
           (i32 (kind node:$addr, node:$exp, node:$new))>;
 class zext_ter_rmw_16_32<PatFrag kind> : zext_ter_rmw_8_32<kind>;
class sext_ter_rmw_8_64<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$exp, node:$new),
           (anyext (i32 (assertzext (i32
             (kind node:$addr,
                   (i32 (trunc (i64 node:$exp))),
                   (i32 (trunc (i64 node:$new))))))))>;
 class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>;
 // 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
 
-// Patterns for various addressing modes for truncating-extending ternary RMWs.
-multiclass TerRMWTruncExtPattern<
-  PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32,
-  string inst8_32, string inst16_32, string inst8_64, string inst16_64,
-  string inst32_64> {
-  // Truncating-extending ternary RMWs with no constant offset
-  defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
-  defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
-
-  // Truncating-extending ternary RMWs with a constant offset
-  defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
-  defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm,
-                         inst16_32>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm,
-                         inst16_64>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm,
-                         inst32_64>;
-  defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
-  defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
-  defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>;
-
-  defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>;
-  defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm,
-                         inst16_32>;
-  defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>;
-  defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm,
-                         inst16_64>;
-  defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>;
-  defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>;
-  defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
-  defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
-
-  // Truncating-extending ternary RMWs with just a constant offset
-  defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
-  defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
-
-  defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
-
-  defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
-}
-
-defm : TerRMWTruncExtPattern<
-  atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32,
-  "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32",
-  "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64",
-  "ATOMIC_RMW32_U_CMPXCHG_I64">;
+defm : TerRMWPat<i32, zext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">;
+defm : TerRMWPat<i32, zext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">;
+defm : TerRMWPat<i64, zext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">;
+defm : TerRMWPat<i64, zext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">;
+defm : TerRMWPat<i64, zext_ter_rmw_32_64<atomic_cmp_swap_32>, "ATOMIC_RMW32_U_CMPXCHG_I64">;
+
+defm : TerRMWPat<i32, sext_ter_rmw_8_32<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I32">;
+defm : TerRMWPat<i32, sext_ter_rmw_16_32<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I32">;
+defm : TerRMWPat<i64, sext_ter_rmw_8_64<atomic_cmp_swap_8>, "ATOMIC_RMW8_U_CMPXCHG_I64">;
+defm : TerRMWPat<i64, sext_ter_rmw_16_64<atomic_cmp_swap_16>, "ATOMIC_RMW16_U_CMPXCHG_I64">;

llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td

 // TODO:
 //  - WebAssemblyTargetLowering having to do with atomics
 //  - Each has optional alignment.
 
 // WebAssembly has i8/i16/i32/i64/f32/f64 memory types, but doesn't have i8/i16
 // local types. These memory-only types instead zero- or sign-extend into local
 // types when loading, and truncate when storing.
 
-// WebAssembly constant offsets are performed as unsigned with infinite
-// precision, so we need to check for NoUnsignedWrap so that we don't fold an
-// offset for an add that needs wrapping.
-def regPlusImm : PatFrag<(ops node:$addr, node:$off),
-                         (add node:$addr, node:$off),
-                         [{ return N->getFlags().hasNoUnsignedWrap(); }]>;
-
-// Treat an 'or' node as an 'add' if the or'ed bits are known to be zero.
-def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
-  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
-    return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
-
-  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
-  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
-  return (~Known0.Zero & ~Known1.Zero) == 0;
-}]>;
+// Address Operands
 
-// We don't need a regPlusES because external symbols never have constant
-// offsets folded into them, so we can just use add.
+// These patterns match the static (offset) and dynamic (address stack operand)
+// operands for loads and stores, based on a combination of target global
+// addresses and constants.
+// For example,
+// (load (add tga, x))   -> load offset=tga, addr=x
+// (store v, tga)        -> store v, offset=tga, addr=0
+// (load (add const, x)) -> load offset=const, addr=x
+// (store v, const)      -> store v, offset=const, addr=0
+// (load x)              -> load offset=0, addr=x
+def AddrOps32 : ComplexPattern<i32, 2, "SelectAddrOperands32">;
+def AddrOps64 : ComplexPattern<i64, 2, "SelectAddrOperands64">;
 
 // Defines atomic and non-atomic loads, regular and extending.
 multiclass WebAssemblyLoad<WebAssemblyRegClass rc, string Name, int Opcode,
                            list<Predicate> reqs = []> {
   let mayLoad = 1, UseNamedOperandTable = 1 in {
   defm "_A32": I<(outs rc:$dst),
                  (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
                  (outs), (ins P2Align:$p2align, offset32_op:$off),
                  [], !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}"),

 defm LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.load8_u", 0x31, []>;
 defm LOAD16_S_I64 : WebAssemblyLoad<I64, "i64.load16_s", 0x32, []>;
 defm LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.load16_u", 0x33, []>;
 defm LOAD32_S_I64 : WebAssemblyLoad<I64, "i64.load32_s", 0x34, []>;
 defm LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.load32_u", 0x35, []>;
 
 // Pattern matching
 
-// Patterns that match the static (offset) and dynamic (address stack operand)
-// operand) operands for loads, based on a combination of target global
-// addresses and constants.
-
-// For example,
-// (add tga x)   -> load tga(x)
-// tga           -> load tga(0)
-// (add const x) -> load const(x)
-// const         -> load const(0)
-// x             -> load 0(x)
-def LoadOps32 : ComplexPattern<i32, 2, "SelectLoadOperands32">;
-def LoadOps64 : ComplexPattern<i64, 2, "SelectLoadOperands64">;
-
 multiclass LoadPat<ValueType ty, SDPatternOperator kind, string Name> {
-      def : Pat<(ty (kind (LoadOps32 offset32_op:$offset, I32:$addr))),
+      def : Pat<(ty (kind (AddrOps32 offset32_op:$offset, I32:$addr))),
            (!cast<NI>(Name # "_A32") 0,
                                      offset32_op:$offset,
                                      I32:$addr)>,
                                      Requires<[HasAddr32]>;
 
-      def : Pat<(ty (kind (LoadOps64 offset64_op:$offset, I64:$addr))),
+      def : Pat<(ty (kind (AddrOps64 offset64_op:$offset, I64:$addr))),
            (!cast<NI>(Name # "_A64") 0,
                                      offset64_op:$offset,
                                      I64:$addr)>,
                                      Requires<[HasAddr64]>;
-
 }
 
 defm : LoadPat<i32, load, "LOAD_I32">;
 defm : LoadPat<i64, load, "LOAD_I64">;
 defm : LoadPat<f32, load, "LOAD_F32">;
 defm : LoadPat<f64, load, "LOAD_F64">;
 
 defm : LoadPat<i32, sextloadi8, "LOAD8_S_I32">;

 
 // Basic store.
 // Note: WebAssembly inverts SelectionDAG's usual operand order.
 defm STORE_I32  : WebAssemblyStore<I32, "i32.store", 0x36>;
 defm STORE_I64  : WebAssemblyStore<I64, "i64.store", 0x37>;
 defm STORE_F32  : WebAssemblyStore<F32, "f32.store", 0x38>;
 defm STORE_F64  : WebAssemblyStore<F64, "f64.store", 0x39>;
 
-// Select stores with no constant offset.
-multiclass StorePatNoOffset<ValueType ty, PatFrag node, string inst> {
-  def : Pat<(node ty:$val, I32:$addr),
-            (!cast<NI>(inst # "_A32") 0, 0, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32]>;
-  def : Pat<(node ty:$val, I64:$addr),
-            (!cast<NI>(inst # "_A64") 0, 0, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64]>;
-}
-
-defm : StorePatNoOffset<i32, store, "STORE_I32">;
-defm : StorePatNoOffset<i64, store, "STORE_I64">;
-defm : StorePatNoOffset<f32, store, "STORE_F32">;
-defm : StorePatNoOffset<f64, store, "STORE_F64">;
-
-// Select stores with a constant offset.
-multiclass StorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
-                          string inst> {
-  def : Pat<(kind ty:$val, (operand I32:$addr, imm:$off)),
-            (!cast<NI>(inst # "_A32") 0, imm:$off, I32:$addr, ty:$val)>,
+multiclass StorePat<ValueType ty, SDPatternOperator kind, string Name> {
+  def : Pat<(kind ty:$val, (AddrOps32 offset32_op:$offset, I32:$addr)),
+            (!cast<NI>(Name # "_A32") 0,
+                                      offset32_op:$offset,
+                                      I32:$addr,
+                                      ty:$val)>,
             Requires<[HasAddr32]>;
-  def : Pat<(kind ty:$val, (operand I64:$addr, imm:$off)),
-            (!cast<NI>(inst # "_A64") 0, imm:$off, I64:$addr, ty:$val)>,
+  def : Pat<(kind ty:$val, (AddrOps64 offset64_op:$offset, I64:$addr)),
+            (!cast<NI>(Name # "_A64") 0,
+                                      offset64_op:$offset,
+                                      I64:$addr,
+                                      ty:$val)>,
             Requires<[HasAddr64]>;
 }
 
-defm : StorePatImmOff<i32, store, regPlusImm, "STORE_I32">;
-defm : StorePatImmOff<i64, store, regPlusImm, "STORE_I64">;
-defm : StorePatImmOff<f32, store, regPlusImm, "STORE_F32">;
-defm : StorePatImmOff<f64, store, regPlusImm, "STORE_F64">;
-defm : StorePatImmOff<i32, store, or_is_add, "STORE_I32">;
-defm : StorePatImmOff<i64, store, or_is_add, "STORE_I64">;
-defm : StorePatImmOff<f32, store, or_is_add, "STORE_F32">;
-defm : StorePatImmOff<f64, store, or_is_add, "STORE_F64">;
-
-// Select stores with just a constant offset.
-multiclass StorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(kind ty:$val, imm:$off),
-            (!cast<NI>(inst # "_A32") 0, imm:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32]>;
-  def : Pat<(kind ty:$val, imm:$off),
-            (!cast<NI>(inst # "_A64") 0, imm:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64]>;
-}
-defm : StorePatOffsetOnly<i32, store, "STORE_I32">;
-defm : StorePatOffsetOnly<i64, store, "STORE_I64">;
-defm : StorePatOffsetOnly<f32, store, "STORE_F32">;
-defm : StorePatOffsetOnly<f64, store, "STORE_F64">;
-
-multiclass StorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
-  def : Pat<(kind ty:$val, (WebAssemblyWrapper tglobaladdr:$off)),
-            (!cast<NI>(inst # "_A32") 0, tglobaladdr:$off, (CONST_I32 0),
-             ty:$val)>,
-        Requires<[IsNotPIC, HasAddr32]>;
-  def : Pat<(kind ty:$val, (WebAssemblyWrapper tglobaladdr:$off)),
-            (!cast<NI>(inst # "_A64") 0, tglobaladdr:$off, (CONST_I64 0),
-             ty:$val)>,
-        Requires<[IsNotPIC, HasAddr64]>;
-}
-defm : StorePatGlobalAddrOffOnly<i32, store, "STORE_I32">;
-defm : StorePatGlobalAddrOffOnly<i64, store, "STORE_I64">;
-defm : StorePatGlobalAddrOffOnly<f32, store, "STORE_F32">;
-defm : StorePatGlobalAddrOffOnly<f64, store, "STORE_F64">;
+defm : StorePat<i32, store, "STORE_I32">;
+defm : StorePat<i64, store, "STORE_I64">;
+defm : StorePat<f32, store, "STORE_F32">;
+defm : StorePat<f64, store, "STORE_F64">;
 
 // Truncating store.
 defm STORE8_I32 : WebAssemblyStore<I32, "i32.store8", 0x3a>;
 defm STORE16_I32 : WebAssemblyStore<I32, "i32.store16", 0x3b>;
 defm STORE8_I64 : WebAssemblyStore<I64, "i64.store8", 0x3c>;
 defm STORE16_I64 : WebAssemblyStore<I64, "i64.store16", 0x3d>;
 defm STORE32_I64 : WebAssemblyStore<I64, "i64.store32", 0x3e>;
 
-// Select truncating stores with no constant offset.
-defm : StorePatNoOffset<i32, truncstorei8, "STORE8_I32">;
-defm : StorePatNoOffset<i32, truncstorei16, "STORE16_I32">;
-defm : StorePatNoOffset<i64, truncstorei8, "STORE8_I64">;
-defm : StorePatNoOffset<i64, truncstorei16, "STORE16_I64">;
-defm : StorePatNoOffset<i64, truncstorei32, "STORE32_I64">;
-
-// Select truncating stores with a constant offset.
-defm : StorePatImmOff<i32, truncstorei8, regPlusImm, "STORE8_I32">;
-defm : StorePatImmOff<i32, truncstorei16, regPlusImm, "STORE16_I32">;
-defm : StorePatImmOff<i64, truncstorei8, regPlusImm, "STORE8_I64">;
-defm : StorePatImmOff<i64, truncstorei16, regPlusImm, "STORE16_I64">;
-defm : StorePatImmOff<i64, truncstorei32, regPlusImm, "STORE32_I64">;
-defm : StorePatImmOff<i32, truncstorei8, or_is_add, "STORE8_I32">;
-defm : StorePatImmOff<i32, truncstorei16, or_is_add, "STORE16_I32">;
-defm : StorePatImmOff<i64, truncstorei8, or_is_add, "STORE8_I64">;
-defm : StorePatImmOff<i64, truncstorei16, or_is_add, "STORE16_I64">;
-defm : StorePatImmOff<i64, truncstorei32, or_is_add, "STORE32_I64">;
-
-// Select truncating stores with just a constant offset.
-defm : StorePatOffsetOnly<i32, truncstorei8, "STORE8_I32">;
-defm : StorePatOffsetOnly<i32, truncstorei16, "STORE16_I32">;
-defm : StorePatOffsetOnly<i64, truncstorei8, "STORE8_I64">;
-defm : StorePatOffsetOnly<i64, truncstorei16, "STORE16_I64">;
-defm : StorePatOffsetOnly<i64, truncstorei32, "STORE32_I64">;
-defm : StorePatGlobalAddrOffOnly<i32, truncstorei8, "STORE8_I32">;
-defm : StorePatGlobalAddrOffOnly<i32, truncstorei16, "STORE16_I32">;
-defm : StorePatGlobalAddrOffOnly<i64, truncstorei8, "STORE8_I64">;
-defm : StorePatGlobalAddrOffOnly<i64, truncstorei16, "STORE16_I64">;
-defm : StorePatGlobalAddrOffOnly<i64, truncstorei32, "STORE32_I64">;
+defm : StorePat<i32, truncstorei8, "STORE8_I32">;
+defm : StorePat<i32, truncstorei16, "STORE16_I32">;
+defm : StorePat<i64, truncstorei8, "STORE8_I64">;
+defm : StorePat<i64, truncstorei16, "STORE16_I64">;
+defm : StorePat<i64, truncstorei32, "STORE32_I64">;
 
 multiclass MemoryOps<WebAssemblyRegClass rc, string B> {
 // Current memory size.
 defm MEMORY_SIZE_A#B : I<(outs rc:$dst), (ins i32imm:$flags),
                          (outs), (ins i32imm:$flags),
                          [(set rc:$dst,
                            (int_wasm_memory_size (i32 imm:$flags)))],
                          "memory.size\t$dst, $flags", "memory.size\t$flags",

llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

defm STORE_V128_A64 :
   SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr, V128:$vec),
          (outs), (ins P2Align:$p2align, offset64_op:$off), [],
          "v128.store\t${off}(${addr})$p2align, $vec",
          "v128.store\t$off$p2align", 11>;
 }
 
 // Def store patterns from WebAssemblyInstrMemory.td for vector types
 foreach vec = AllVecs in {
-defm : StorePatNoOffset<vec.vt, store, "STORE_V128">;
-defm : StorePatImmOff<vec.vt, store, regPlusImm, "STORE_V128">;
-defm : StorePatImmOff<vec.vt, store, or_is_add, "STORE_V128">;
-defm : StorePatOffsetOnly<vec.vt, store, "STORE_V128">;
-defm : StorePatGlobalAddrOffOnly<vec.vt, store, "STORE_V128">;
+defm : StorePat<vec.vt, store, "STORE_V128">;
 }
 
 // Store lane
 multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
   defvar name = "v128.store"#vec.lane_bits#"_lane";
   let mayStore = 1, UseNamedOperandTable = 1 in {
   defm STORE_LANE_#vec#_A32 :
     SIMD_I<(outs),
multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
   } // mayStore = 1, UseNamedOperandTable = 1
 }
 
 defm "" : SIMDStoreLane<I8x16, 0x58>;
 defm "" : SIMDStoreLane<I16x8, 0x59>;
 defm "" : SIMDStoreLane<I32x4, 0x5a>;
 defm "" : SIMDStoreLane<I64x2, 0x5b>;
 
-// Select stores with no constant offset.
-multiclass StoreLanePatNoOffset<Vec vec, SDPatternOperator kind> {
-  def : Pat<(kind (i32 I32:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
-            (!cast<NI>("STORE_LANE_"#vec#"_A32") 0, 0, imm:$idx, $addr, $vec)>,
+multiclass StoreLanePat<Vec vec, SDPatternOperator kind> {
+  def : Pat<(kind (AddrOps32 offset32_op:$offset, I32:$addr),
+                  (vec.vt V128:$vec),
+                  (i32 vec.lane_idx:$idx)),
+            (!cast<NI>("STORE_LANE_"#vec#"_A32") 0, $offset, imm:$idx, $addr, $vec)>,
         Requires<[HasAddr32]>;
-  def : Pat<(kind (i64 I64:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
-            (!cast<NI>("STORE_LANE_"#vec#"_A64") 0, 0, imm:$idx, $addr, $vec)>,
+  def : Pat<(kind (AddrOps64 offset64_op:$offset, I64:$addr),
+                  (vec.vt V128:$vec),
+                  (i32 vec.lane_idx:$idx)),
+            (!cast<NI>("STORE_LANE_"#vec#"_A64") 0, $offset, imm:$idx, $addr, $vec)>,
         Requires<[HasAddr64]>;
 }
 
 def store8_lane :
   PatFrag<(ops node:$ptr, node:$vec, node:$idx),
           (truncstorei8 (i32 (vector_extract $vec, $idx)), $ptr)>;
 def store16_lane :
   PatFrag<(ops node:$ptr, node:$vec, node:$idx),
           (truncstorei16 (i32 (vector_extract $vec, $idx)), $ptr)>;
 def store32_lane :
   PatFrag<(ops node:$ptr, node:$vec, node:$idx),
           (store (i32 (vector_extract $vec, $idx)), $ptr)>;
 def store64_lane :
   PatFrag<(ops node:$ptr, node:$vec, node:$idx),
           (store (i64 (vector_extract $vec, $idx)), $ptr)>;
 // TODO: floating point lanes as well
 
 let AddedComplexity = 1 in {
-defm : StoreLanePatNoOffset<I8x16, store8_lane>;
-defm : StoreLanePatNoOffset<I16x8, store16_lane>;
-defm : StoreLanePatNoOffset<I32x4, store32_lane>;
-defm : StoreLanePatNoOffset<I64x2, store64_lane>;
+defm : StoreLanePat<I8x16, store8_lane>;
+defm : StoreLanePat<I16x8, store16_lane>;
+defm : StoreLanePat<I32x4, store32_lane>;
+defm : StoreLanePat<I64x2, store64_lane>;
 }
 
 //===----------------------------------------------------------------------===//
 // Constructing SIMD values
 //===----------------------------------------------------------------------===//
 
 // Constant: v128.const
 multiclass ConstVec<Vec vec, dag ops, dag pat, string args> {

llvm/test/CodeGen/WebAssembly/simd-load-lane-offset.ll

; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_lane_i8_with_folded_offset(<16 x i8> %v, i8* %p) {
 ; CHECK-LABEL: store_lane_i8_with_folded_offset:
 ; CHECK:         .functype store_lane_i8_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store8_lane 0, 0
+; CHECK-NEXT:    v128.store8_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, i8* %s
   ret void
 }
 
 define void @store_lane_i8_with_folded_gep_offset(<16 x i8> %v, i8* %p) {
 ; CHECK-LABEL: store_lane_i8_with_folded_gep_offset:
 ; CHECK:         .functype store_lane_i8_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store8_lane 0, 0
+; CHECK-NEXT:    v128.store8_lane 6, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, i8* %p, i32 6
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, i8* %s
   ret void
 }
 
 define void @store_lane_i8_with_unfolded_gep_negative_offset(<16 x i8> %v, i8* %p) {
; CHECK-NEXT:    # fallthrough-return
   store i8 %x, i8* %s
   ret void
 }
 
 define void @store_lane_i8_to_numeric_address(<16 x i8> %v) {
 ; CHECK-LABEL: store_lane_i8_to_numeric_address:
 ; CHECK:         .functype store_lane_i8_to_numeric_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store8_lane 0, 0
+; CHECK-NEXT:    v128.store8_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to i8*
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, i8* %s
   ret void
 }
 
 define void @store_lane_i8_from_global_address(<16 x i8> %v) {
 ; CHECK-LABEL: store_lane_i8_from_global_address:
 ; CHECK:         .functype store_lane_i8_from_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_i8
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store8_lane 0, 0
+; CHECK-NEXT:    v128.store8_lane gv_i8, 0
 ; CHECK-NEXT:    # fallthrough-return
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, i8* @gv_i8
   ret void
 }
 
 ;===----------------------------------------------------------------------------
 ; v128.load16_lane / v128.store16_lane
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_lane_i16_with_folded_offset(<8 x i16> %v, i16* %p) {
 ; CHECK-LABEL: store_lane_i16_with_folded_offset:
 ; CHECK:         .functype store_lane_i16_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store16_lane 0, 0
+; CHECK-NEXT:    v128.store16_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i16* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i16*
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, i16* %s
   ret void
 }
 
 define void @store_lane_i16_with_folded_gep_offset(<8 x i16> %v, i16* %p) {
 ; CHECK-LABEL: store_lane_i16_with_folded_gep_offset:
 ; CHECK:         .functype store_lane_i16_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 12
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store16_lane 0, 0
+; CHECK-NEXT:    v128.store16_lane 12, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, i16* %p, i32 6
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, i16* %s
   ret void
 }
 
 define void @store_lane_i16_with_unfolded_gep_negative_offset(<8 x i16> %v, i16* %p) {
; CHECK-NEXT:    # fallthrough-return
   store i16 %x, i16* %s
   ret void
 }
 
 define void @store_lane_i16_to_numeric_address(<8 x i16> %v) {
 ; CHECK-LABEL: store_lane_i16_to_numeric_address:
 ; CHECK:         .functype store_lane_i16_to_numeric_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store16_lane 0, 0
+; CHECK-NEXT:    v128.store16_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to i16*
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, i16* %s
   ret void
 }
 
 define void @store_lane_i16_from_global_address(<8 x i16> %v) {
 ; CHECK-LABEL: store_lane_i16_from_global_address:
 ; CHECK:         .functype store_lane_i16_from_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_i16
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store16_lane 0, 0
+; CHECK-NEXT:    v128.store16_lane gv_i16, 0
 ; CHECK-NEXT:    # fallthrough-return
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, i16* @gv_i16
   ret void
 }
 
 ;===----------------------------------------------------------------------------
 ; v128.load32_lane / v128.store32_lane
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_lane_i32_with_folded_offset(<4 x i32> %v, i32* %p) {
 ; CHECK-LABEL: store_lane_i32_with_folded_offset:
 ; CHECK:         .functype store_lane_i32_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store32_lane 0, 0
+; CHECK-NEXT:    v128.store32_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, i32* %s
   ret void
 }
 
 define void @store_lane_i32_with_folded_gep_offset(<4 x i32> %v, i32* %p) {
 ; CHECK-LABEL: store_lane_i32_with_folded_gep_offset:
 ; CHECK:         .functype store_lane_i32_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store32_lane 0, 0
+; CHECK-NEXT:    v128.store32_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, i32* %p, i32 6
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, i32* %s
   ret void
 }
 
 define void @store_lane_i32_with_unfolded_gep_negative_offset(<4 x i32> %v, i32* %p) {
; CHECK-NEXT:    # fallthrough-return
   store i32 %x, i32* %s
   ret void
 }
 
 define void @store_lane_i32_to_numeric_address(<4 x i32> %v) {
 ; CHECK-LABEL: store_lane_i32_to_numeric_address:
 ; CHECK:         .functype store_lane_i32_to_numeric_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store32_lane 0, 0
+; CHECK-NEXT:    v128.store32_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to i32*
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, i32* %s
   ret void
 }
 
 define void @store_lane_i32_from_global_address(<4 x i32> %v) {
 ; CHECK-LABEL: store_lane_i32_from_global_address:
 ; CHECK:         .functype store_lane_i32_from_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_i32
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store32_lane 0, 0
+; CHECK-NEXT:    v128.store32_lane gv_i32, 0
 ; CHECK-NEXT:    # fallthrough-return
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, i32* @gv_i32
   ret void
 }
 
 ;===----------------------------------------------------------------------------
 ; v128.load64_lane / v128.store64_lane
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_lane_i64_with_folded_offset(<2 x i64> %v, i64* %p) {
 ; CHECK-LABEL: store_lane_i64_with_folded_offset:
 ; CHECK:         .functype store_lane_i64_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, i64* %s
   ret void
 }
 
 define void @store_lane_i64_with_folded_gep_offset(<2 x i64> %v, i64* %p) {
 ; CHECK-LABEL: store_lane_i64_with_folded_gep_offset:
 ; CHECK:         .functype store_lane_i64_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 48
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 48, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, i64* %p, i32 6
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, i64* %s
   ret void
 }
 
 define void @store_lane_i64_with_unfolded_gep_negative_offset(<2 x i64> %v, i64* %p) {
; CHECK-NEXT:    # fallthrough-return
   store i64 %x, i64* %s
   ret void
 }
 
 define void @store_lane_i64_to_numeric_address(<2 x i64> %v) {
 ; CHECK-LABEL: store_lane_i64_to_numeric_address:
 ; CHECK:         .functype store_lane_i64_to_numeric_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to i64*
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, i64* %s
   ret void
 }
 
 define void @store_lane_i64_from_global_address(<2 x i64> %v) {
 ; CHECK-LABEL: store_lane_i64_from_global_address:
 ; CHECK:         .functype store_lane_i64_from_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_i64
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane gv_i64, 0
 ; CHECK-NEXT:    # fallthrough-return
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, i64* @gv_i64
   ret void
 }

llvm/test/CodeGen/WebAssembly/simd-offset.ll

; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_narrowing_v8i16_with_folded_offset(<8 x i8> %v, <8 x i8>* %p) {
 ; CHECK-LABEL: store_narrowing_v8i16_with_folded_offset:
 ; CHECK:         .functype store_narrowing_v8i16_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 16, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
   store <8 x i8> %v , <8 x i8>* %s
   ret void
 }
 
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_narrowing_v8i16_with_folded_gep_offset(<8 x i8> %v, <8 x i8>* %p) {
 ; CHECK-LABEL: store_narrowing_v8i16_with_folded_gep_offset:
 ; CHECK:         .functype store_narrowing_v8i16_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 8, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
   store <8 x i8> %v , <8 x i8>* %s
   ret void
 }
 
 define void @store_v8i16_with_unfolded_gep_negative_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_unfolded_gep_negative_offset:
; CHECK-NEXT:    # fallthrough-return
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
 define void @store_narrowing_v8i16_to_numeric_address(<8 x i8> %v, <8 x i8>* %p) {
 ; CHECK-LABEL: store_narrowing_v8i16_to_numeric_address:
 ; CHECK:         .functype store_narrowing_v8i16_to_numeric_address (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 32, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i8>*
   store <8 x i8> %v , <8 x i8>* %s
   ret void
 }
 
 define void @store_v8i16_to_global_address(<8 x i16> %v) {
 ; CHECK-LABEL: store_v8i16_to_global_address:
 ; CHECK:         .functype store_v8i16_to_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store gv_v8i16
 ; CHECK-NEXT:    # fallthrough-return
   store <8 x i16> %v , <8 x i16>* @gv_v8i16
   ret void
 }
 
 define void @store_narrowing_v8i16_to_global_address(<8 x i8> %v) {
 ; CHECK-LABEL: store_narrowing_v8i16_to_global_address:
 ; CHECK:         .functype store_narrowing_v8i16_to_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_v8i8
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane gv_v8i8, 0
 ; CHECK-NEXT:    # fallthrough-return
   store <8 x i8> %v , <8 x i8>* @gv_v8i8
   ret void
 }
 
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
; CHECK-NEXT:    # fallthrough-return
   %v2 = sext <4 x i8> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
 define <4 x i32> @load_zext_v4i8_to_v4i32_with_folded_gep_offset(<4 x i8>* %p) {
 ; CHECK-LABEL: load_zext_v4i8_to_v4i32_with_folded_gep_offset:
 ; CHECK:         .functype load_zext_v4i8_to_v4i32_with_folded_gep_offset (i32) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:  v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT:  local.get 0
-; CHECK-NEXT:  v128.load32_zero 4
-; CHECK-NEXT:  i8x16.shuffle 16, 1, 2, 3, 17, 5, 6, 7, 18, 9, 10, 11, 19, 13, 14, 15
-; CHECK-NEXT:  # fallthrough-return
+; CHECK-NEXT:    v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load32_zero 4
+; CHECK-NEXT:    i8x16.shuffle 16, 1, 2, 3, 17, 5, 6, 7, 18, 9, 10, 11, 19, 13, 14, 15
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i8>, <4 x i8>* %p, i32 1
   %v = load <4 x i8>, <4 x i8>* %s
   %v2 = zext <4 x i8> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
 define <4 x i16> @load_ext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
 ; CHECK-LABEL: load_ext_v4i32_with_folded_gep_offset:
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_narrowing_v4i32_with_folded_offset(<4 x i16> %v, <4 x i16>* %p) {
 ; CHECK-LABEL: store_narrowing_v4i32_with_folded_offset:
 ; CHECK:         .functype store_narrowing_v4i32_with_folded_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 16, 0
 ; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
   store <4 x i16> %v , <4 x i16>* %s
   ret void
 }
 
; CHECK-NEXT:    # fallthrough-return
   ret void
 }
 
 define void @store_narrowing_v4i32_with_folded_gep_offset(<4 x i16> %v, <4 x i16>* %p) {
 ; CHECK-LABEL: store_narrowing_v4i32_with_folded_gep_offset:
 ; CHECK:         .functype store_narrowing_v4i32_with_folded_gep_offset (v128, i32) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 1
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 8, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
   store <4 x i16> %v , <4 x i16>* %s
   ret void
 }
 
 define void @store_v4i32_with_unfolded_gep_negative_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_unfolded_gep_negative_offset:
; CHECK-NEXT:    # fallthrough-return
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
 define void @store_narrowing_v4i32_to_numeric_address(<4 x i16> %v) {
 ; CHECK-LABEL: store_narrowing_v4i32_to_numeric_address:
 ; CHECK:         .functype store_narrowing_v4i32_to_numeric_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 32
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane 32, 0
 ; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i16>*
   store <4 x i16> %v , <4 x i16>* %s
   ret void
 }
 
 define void @store_v4i32_to_global_address(<4 x i32> %v) {
 ; CHECK-LABEL: store_v4i32_to_global_address:
 ; CHECK:         .functype store_v4i32_to_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store gv_v4i32
 ; CHECK-NEXT:    # fallthrough-return
   store <4 x i32> %v , <4 x i32>* @gv_v4i32
   ret void
 }
 
 define void @store_narrowing_v4i32_to_global_address(<4 x i16> %v) {
 ; CHECK-LABEL: store_narrowing_v4i32_to_global_address:
 ; CHECK:         .functype store_narrowing_v4i32_to_global_address (v128) -> ()
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv_v4i16
+; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    v128.store64_lane 0, 0
+; CHECK-NEXT:    v128.store64_lane gv_v4i16, 0
 ; CHECK-NEXT:    # fallthrough-return
   store <4 x i16> %v , <4 x i16>* @gv_v4i16
   ret void
 }
 
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================