[WebAssembly] Improve readability of load/stores and tests. NFC.

[aheejin]

Summary

• Changed variable/function names to be more consistent
• Improved comments in test files
• Added more tests
• Fixed a few typos
• Misc. cosmetic changes

[aheejin]

• Pattern name change: anyext_aload -> sext_aload

This pattern doesn't cover all the sign extension selection (we need a
sext_inreg after that), but it looks less confusing and more consistent with
zext_aload patterns. And we gave enough explanations in comments anyway.

lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

 def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>;
 
 def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
 def : LoadPatExternalSym<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatExternalSym<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
-
 // Select loads with just a constant offset.
 def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
 def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
 def : LoadPatExternSymOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>;
 
 } // Predicates = [HasAtomics]
 
 // Extending loads. Note that there are only zero-extending atomic loads, no
 // sign-extending loads.
 let Defs = [ARGUMENTS] in {
 defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>;
 defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>;
 defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>;
 defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>;
 defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>;
 } // Defs = [ARGUMENTS]
 
-// Fragments for exending loads. These are different from regular loads because
+// Fragments for extending loads. These are different from regular loads because
 // the SDNodes are derived from AtomicSDNode rather than LoadSDNode and
 // therefore don't have the extension type field. So instead of matching that,
 // we match the patterns that the type legalizer expands them to.
 
 // We directly match zext patterns and select the zext atomic loads.
 // i32 (zext (i8 (atomic_load_8))) gets legalized to
 // i32 (and (i32 (atomic_load_8)), 255)
 // These can be selected to a single zero-extending atomic load instruction.
-def zext_aload_8 : PatFrag<(ops node:$addr),
-                           (and (i32 (atomic_load_8 node:$addr)), 255)>;
-def zext_aload_16 : PatFrag<(ops node:$addr),
-                            (and (i32 (atomic_load_16 node:$addr)), 65535)>;
+def zext_aload_8_32 :
+  PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>;
+def zext_aload_16_32 :
+  PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>;
 // Unlike regular loads, extension to i64 is handled differently than i32.
 // i64 (zext (i8 (atomic_load_8))) gets legalized to
 // i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255)
 def zext_aload_8_64 :
   PatFrag<(ops node:$addr),
           (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>;
 def zext_aload_16_64 :
   PatFrag<(ops node:$addr),
           (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>;
 def zext_aload_32_64 :
   PatFrag<(ops node:$addr),
           (zext (i32 (atomic_load node:$addr)))>;
 
 // We don't have single sext atomic load instructions. So for sext loads, we
 // match bare subword loads (for 32-bit results) and anyext loads (for 64-bit
 // results) and select a zext load; the next instruction will be sext_inreg
 // which is selected by itself.
-def anyext_aload_8_64 :
+def sext_aload_8_64 :
   PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>;
-def anyext_aload_16_64 :
+def sext_aload_16_64 :
   PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>;
 
 let Predicates = [HasAtomics] in {
 // Select zero-extending loads with no constant offset.
-def : LoadPatNoOffset<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatNoOffset<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 
 // Select sign-extending loads with no constant offset
 def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatNoOffset<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatNoOffset<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i64
-
+def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
+// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32
 
 // Zero-extending loads with constant offset
-def : LoadPatImmOff<i32, zext_aload_8, regPlusImm, ATOMIC_LOAD8_U_I32>;
-def : LoadPatImmOff<i32, zext_aload_16, regPlusImm, ATOMIC_LOAD16_U_I32>;
-def : LoadPatImmOff<i32, zext_aload_8, or_is_add, ATOMIC_LOAD8_U_I32>;
-def : LoadPatImmOff<i32, zext_aload_16, or_is_add, ATOMIC_LOAD16_U_I32>;
+def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>;
+def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>;
+def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>;
+def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>;
 def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>;
 def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>;
 def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>;
 def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>;
 def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>;
 def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>;
 
 // Sign-extending loads with constant offset
 def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>;
 def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>;
 def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>;
 def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>;
-def : LoadPatImmOff<i64, anyext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>;
-def : LoadPatImmOff<i64, anyext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>;
-def : LoadPatImmOff<i64, anyext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>;
-def : LoadPatImmOff<i64, anyext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>;
+def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>;
+def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>;
+def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>;
+def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>;
 // No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64
 
-def : LoadPatGlobalAddr<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatGlobalAddr<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatGlobalAddr<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatGlobalAddr<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
+def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 
-def : LoadPatExternalSym<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatExternalSym<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatExternalSym<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatExternalSym<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatExternalSym<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatExternalSym<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatExternalSym<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 def : LoadPatExternalSym<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatExternalSym<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatExternalSym<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatExternalSym<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-
+def : LoadPatExternalSym<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatExternalSym<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 
 // Extending loads with just a constant offset
-def : LoadPatOffsetOnly<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatOffsetOnly<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatOffsetOnly<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatOffsetOnly<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
+def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 
-def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatGlobalAddrOffOnly<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatGlobalAddrOffOnly<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
+def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 
-def : LoadPatExternSymOffOnly<i32, zext_aload_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatExternSymOffOnly<i32, zext_aload_16, ATOMIC_LOAD16_U_I32>;
+def : LoadPatExternSymOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
+def : LoadPatExternSymOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
 def : LoadPatExternSymOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatExternSymOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 def : LoadPatExternSymOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
 def : LoadPatExternSymOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatExternSymOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatExternSymOffOnly<i64, anyext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatExternSymOffOnly<i64, anyext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-
+def : LoadPatExternSymOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
+def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
 
 } // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic stores
 //===----------------------------------------------------------------------===//
 
 let Defs = [ARGUMENTS] in {
 defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>;
 defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>;
 } // Defs = [ARGUMENTS]
 
 // We need an 'atomic' version of store patterns because store and atomic_store
 // nodes have different operand orders:
 // store: (store $val, $ptr)
 // atomic_store: (store $ptr, $val)
 
 let Predicates = [HasAtomics] in {
 
 // Select stores with no constant offset.
-class AStorePatNoOffset<ValueType ty, PatFrag node, NI inst> :
-  Pat<(node I32:$addr, ty:$val), (inst 0, 0, $addr, $val)>;
+class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>;
 def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
 // Select stores with a constant offset.
 
 // Pattern with address + immediate offset
-class AStorePatImmOff<ValueType ty, PatFrag storekind, PatFrag operand,
-                      NI inst> :
-  Pat<(storekind (operand I32:$addr, imm:$off), ty:$val),
-      (inst 0, imm:$off, $addr, ty:$val)>;
+class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
+  Pat<(kind (operand I32:$addr, imm:$off), ty:$val),
+      (inst 0, imm:$off, I32:$addr, ty:$val)>;
 def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>;
 def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>;
 def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>;
 def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>;
 
-class AStorePatGlobalAddr<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
-                 ty:$val),
+class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
       (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>;
 def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
-class AStorePatExternalSym<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
-                 ty:$val),
+class AStorePatExternalSym<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), ty:$val),
       (inst 0, texternalsym:$off, I32:$addr, ty:$val)>;
 def : AStorePatExternalSym<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatExternalSym<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
 // Select stores with just a constant offset.
-class AStorePatOffsetOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind imm:$off, ty:$val),
-      (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
+class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
 def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
-class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind (WebAssemblywrapper tglobaladdr:$off), ty:$val),
+class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val),
       (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>;
 def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
-class AStorePatExternSymOffOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind (WebAssemblywrapper texternalsym:$off), ty:$val),
+class AStorePatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind (WebAssemblywrapper texternalsym:$off), ty:$val),
       (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>;
 def : AStorePatExternSymOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
 def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
 
 } // Predicates = [HasAtomics]
 
 // Truncating stores.
 let Defs = [ARGUMENTS] in {
 defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>;
 defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>;
 defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>;
 defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>;
 defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>;
 } // Defs = [ARGUMENTS]
 
 // Fragments for truncating stores.
 
 // We don't have single truncating atomic store instructions. For 32-bit
 // instructions, we just need to match bare atomic stores. On the other hand,
 // truncating stores from i64 values are once truncated to i32 first.
-class trunc_astore_64<PatFrag storekind> :
+class trunc_astore_64<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$val),
-          (storekind node:$addr, (i32 (trunc (i64 node:$val))))>;
+          (kind node:$addr, (i32 (trunc (i64 node:$val))))>;
 def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>;
 def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>;
 def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>;
 
 let Predicates = [HasAtomics] in {
 
 // Truncating stores with no constant offset
 def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>;

lib/Target/WebAssembly/WebAssemblyInstrMemory.td

 defm LOAD_I32 : WebAssemblyLoad<I32, "i32.load", 0x28>;
 defm LOAD_I64 : WebAssemblyLoad<I64, "i64.load", 0x29>;
 defm LOAD_F32 : WebAssemblyLoad<F32, "f32.load", 0x2a>;
 defm LOAD_F64 : WebAssemblyLoad<F64, "f64.load", 0x2b>;
 
 } // Defs = [ARGUMENTS]
 
 // Select loads with no constant offset.
-class LoadPatNoOffset<ValueType ty, PatFrag node, NI inst> :
-  Pat<(ty (node I32:$addr)), (inst 0, 0, $addr)>;
+class LoadPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind I32:$addr)), (inst 0, 0, I32:$addr)>;
 
 def : LoadPatNoOffset<i32, load, LOAD_I32>;
 def : LoadPatNoOffset<i64, load, LOAD_I64>;
 def : LoadPatNoOffset<f32, load, LOAD_F32>;
 def : LoadPatNoOffset<f64, load, LOAD_F64>;
 
 
 // Select loads with a constant offset.
 
 // Pattern with address + immediate offset
-class LoadPatImmOff<ValueType ty, PatFrag loadkind, PatFrag operand, NI inst> :
-  Pat<(ty (loadkind (operand I32:$addr, imm:$off))),
-      (inst 0, imm:$off, $addr)>;
+class LoadPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
+  Pat<(ty (kind (operand I32:$addr, imm:$off))), (inst 0, imm:$off, I32:$addr)>;
 
 def : LoadPatImmOff<i32, load, regPlusImm, LOAD_I32>;
 def : LoadPatImmOff<i64, load, regPlusImm, LOAD_I64>;
 def : LoadPatImmOff<f32, load, regPlusImm, LOAD_F32>;
 def : LoadPatImmOff<f64, load, regPlusImm, LOAD_F64>;
 def : LoadPatImmOff<i32, load, or_is_add, LOAD_I32>;
 def : LoadPatImmOff<i64, load, or_is_add, LOAD_I64>;
 def : LoadPatImmOff<f32, load, or_is_add, LOAD_F32>;
 def : LoadPatImmOff<f64, load, or_is_add, LOAD_F64>;
 
-class LoadPatGlobalAddr<ValueType ty, PatFrag loadkind, NI inst> :
-  Pat<(ty (loadkind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)))),
-      (inst 0, tglobaladdr:$off, $addr)>;
+class LoadPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)))),
+      (inst 0, tglobaladdr:$off, I32:$addr)>;
 
 def : LoadPatGlobalAddr<i32, load, LOAD_I32>;
 def : LoadPatGlobalAddr<i64, load, LOAD_I64>;
 def : LoadPatGlobalAddr<f32, load, LOAD_F32>;
 def : LoadPatGlobalAddr<f64, load, LOAD_F64>;
 
-class LoadPatExternalSym<ValueType ty, PatFrag loadkind, NI inst> :
-  Pat<(ty (loadkind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)))),
-      (inst 0, texternalsym:$off, $addr)>;
+class LoadPatExternalSym<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)))),
+      (inst 0, texternalsym:$off, I32:$addr)>;
 def : LoadPatExternalSym<i32, load, LOAD_I32>;
 def : LoadPatExternalSym<i64, load, LOAD_I64>;
 def : LoadPatExternalSym<f32, load, LOAD_F32>;
 def : LoadPatExternalSym<f64, load, LOAD_F64>;
 
 
 // Select loads with just a constant offset.
-class LoadPatOffsetOnly<ValueType ty, PatFrag loadkind, NI inst> :
-  Pat<(ty (loadkind imm:$off)), (inst 0, imm:$off, (CONST_I32 0))>;
+class LoadPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind imm:$off)), (inst 0, imm:$off, (CONST_I32 0))>;
 
 def : LoadPatOffsetOnly<i32, load, LOAD_I32>;
 def : LoadPatOffsetOnly<i64, load, LOAD_I64>;
 def : LoadPatOffsetOnly<f32, load, LOAD_F32>;
 def : LoadPatOffsetOnly<f64, load, LOAD_F64>;
 
-class LoadPatGlobalAddrOffOnly<ValueType ty, PatFrag loadkind, NI inst> :
-  Pat<(ty (loadkind (WebAssemblywrapper tglobaladdr:$off))),
+class LoadPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off))),
       (inst 0, tglobaladdr:$off, (CONST_I32 0))>;
 
 def : LoadPatGlobalAddrOffOnly<i32, load, LOAD_I32>;
 def : LoadPatGlobalAddrOffOnly<i64, load, LOAD_I64>;
 def : LoadPatGlobalAddrOffOnly<f32, load, LOAD_F32>;
 def : LoadPatGlobalAddrOffOnly<f64, load, LOAD_F64>;
 
-class LoadPatExternSymOffOnly<ValueType ty, PatFrag loadkind, NI inst> :
-  Pat<(ty (loadkind (WebAssemblywrapper texternalsym:$off))),
+class LoadPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(ty (kind (WebAssemblywrapper texternalsym:$off))),
       (inst 0, texternalsym:$off, (CONST_I32 0))>;
 def : LoadPatExternSymOffOnly<i32, load, LOAD_I32>;
 def : LoadPatExternSymOffOnly<i64, load, LOAD_I64>;
 def : LoadPatExternSymOffOnly<f32, load, LOAD_F32>;
 def : LoadPatExternSymOffOnly<f64, load, LOAD_F64>;
 
 let Defs = [ARGUMENTS] in {
 

 defm STORE_I64  : WebAssemblyStore<I64, "i64.store", 0x37>;
 defm STORE_F32  : WebAssemblyStore<F32, "f32.store", 0x38>;
 defm STORE_F64  : WebAssemblyStore<F64, "f64.store", 0x39>;
 
 } // Defs = [ARGUMENTS]
 
 // Select stores with no constant offset.
 class StorePatNoOffset<ValueType ty, PatFrag node, NI inst> :
-  Pat<(node ty:$val, I32:$addr), (inst 0, 0, $addr, $val)>;
+  Pat<(node ty:$val, I32:$addr), (inst 0, 0, I32:$addr, ty:$val)>;
 
 def : StorePatNoOffset<i32, store, STORE_I32>;
 def : StorePatNoOffset<i64, store, STORE_I64>;
 def : StorePatNoOffset<f32, store, STORE_F32>;
 def : StorePatNoOffset<f64, store, STORE_F64>;
 
 // Select stores with a constant offset.
-class StorePatImmOff<ValueType ty, PatFrag storekind, PatFrag operand, NI inst> :
-  Pat<(storekind ty:$val, (operand I32:$addr, imm:$off)),
-      (inst 0, imm:$off, $addr, ty:$val)>;
+class StorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
+  Pat<(kind ty:$val, (operand I32:$addr, imm:$off)),
+      (inst 0, imm:$off, I32:$addr, ty:$val)>;
 
 def : StorePatImmOff<i32, store, regPlusImm, STORE_I32>;
 def : StorePatImmOff<i64, store, regPlusImm, STORE_I64>;
 def : StorePatImmOff<f32, store, regPlusImm, STORE_F32>;
 def : StorePatImmOff<f64, store, regPlusImm, STORE_F64>;
 def : StorePatImmOff<i32, store, or_is_add, STORE_I32>;
 def : StorePatImmOff<i64, store, or_is_add, STORE_I64>;
 def : StorePatImmOff<f32, store, or_is_add, STORE_F32>;
 def : StorePatImmOff<f64, store, or_is_add, STORE_F64>;
 
-class StorePatGlobalAddr<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind ty:$val, (regPlusGA I32:$addr,
-                                     (WebAssemblywrapper tglobaladdr:$off))),
+class StorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind ty:$val,
+            (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off))),
       (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>;
 def : StorePatGlobalAddr<i32, store, STORE_I32>;
 def : StorePatGlobalAddr<i64, store, STORE_I64>;
 def : StorePatGlobalAddr<f32, store, STORE_F32>;
 def : StorePatGlobalAddr<f64, store, STORE_F64>;
 
-class StorePatExternalSym<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind ty:$val, (add I32:$addr,
-                               (WebAssemblywrapper texternalsym:$off))),
+class StorePatExternalSym<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind ty:$val, (add I32:$addr, (WebAssemblywrapper texternalsym:$off))),
       (inst 0, texternalsym:$off, I32:$addr, ty:$val)>;
 def : StorePatExternalSym<i32, store, STORE_I32>;
 def : StorePatExternalSym<i64, store, STORE_I64>;
 def : StorePatExternalSym<f32, store, STORE_F32>;
 def : StorePatExternalSym<f64, store, STORE_F64>;
 
 // Select stores with just a constant offset.
-class StorePatOffsetOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind ty:$val, imm:$off),
-      (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
+class StorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind ty:$val, imm:$off), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
 def : StorePatOffsetOnly<i32, store, STORE_I32>;
 def : StorePatOffsetOnly<i64, store, STORE_I64>;
 def : StorePatOffsetOnly<f32, store, STORE_F32>;
 def : StorePatOffsetOnly<f64, store, STORE_F64>;
 
-class StorePatGlobalAddrOffOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind ty:$val, (WebAssemblywrapper tglobaladdr:$off)),
+class StorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind ty:$val, (WebAssemblywrapper tglobaladdr:$off)),
       (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>;
 def : StorePatGlobalAddrOffOnly<i32, store, STORE_I32>;
 def : StorePatGlobalAddrOffOnly<i64, store, STORE_I64>;
 def : StorePatGlobalAddrOffOnly<f32, store, STORE_F32>;
 def : StorePatGlobalAddrOffOnly<f64, store, STORE_F64>;
 
-class StorePatExternSymOffOnly<ValueType ty, PatFrag storekind, NI inst> :
-  Pat<(storekind ty:$val, (WebAssemblywrapper texternalsym:$off)),
+class StorePatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> :
+  Pat<(kind ty:$val, (WebAssemblywrapper texternalsym:$off)),
       (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>;
 def : StorePatExternSymOffOnly<i32, store, STORE_I32>;
 def : StorePatExternSymOffOnly<i64, store, STORE_I64>;
 def : StorePatExternSymOffOnly<f32, store, STORE_F32>;
 def : StorePatExternSymOffOnly<f64, store, STORE_F64>;
 
 
 let Defs = [ARGUMENTS] in {

test/CodeGen/WebAssembly/i32-load-store-alignment.ll

 ; RUN: llc < %s -mattr=+atomics -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; Loads.
+;===----------------------------------------------------------------------------
+; Loads
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: ldi32_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.load $push[[NUM:[0-9]+]]=, 0($0):p2align=0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_a1(i32 *%p) {
   %v = load i32, i32* %p, align 1

 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_a8(i32 *%p) {
   %v = load i32, i32* %p, align 8
   ret i32 %v
 }
 
-; Extending loads.
+;===----------------------------------------------------------------------------
+; Extending loads
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: ldi8_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.load8_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i8 @ldi8_a1(i8 *%p) {
   %v = load i8, i8* %p, align 1

 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.load16_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i16 @ldi16_a4(i16 *%p) {
   %v = load i16, i16* %p, align 4
   ret i16 %v
 }
 
-; Stores.
+;===----------------------------------------------------------------------------
+; Stores
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: sti32_a1:
 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.store 0($0):p2align=0, $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32_a1(i32 *%p, i32 %v) {
   store i32 %v, i32* %p, align 1
   ret void

 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32_a8(i32 *%p, i32 %v) {
   store i32 %v, i32* %p, align 8
   ret void
 }
 
-; Truncating stores.
+;===----------------------------------------------------------------------------
+; Truncating stores
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: sti8_a1:
 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.store8 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti8_a1(i8 *%p, i8 %v) {
   store i8 %v, i8* %p, align 1
   ret void

 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.store16 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti16_a4(i16 *%p, i16 %v) {
   store i16 %v, i16* %p, align 4
   ret void
 }
 
-; Atomics.
-; Wasm atomics have the alignment field, but it must always have the
-; type's natural alignment.
+;===----------------------------------------------------------------------------
+; Atomic loads
+;===----------------------------------------------------------------------------
+
+; Wasm atomics have the alignment field, but it must always have the type's
+; natural alignment.
 
 ; CHECK-LABEL: ldi32_atomic_a4:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_atomic_a4(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 4
   ret i32 %v
 }
 
-; 8 is greater than the default alignment so it is rounded down to 4
+; 8 is greater than the default alignment so it is ignored.
 
 ; CHECK-LABEL: ldi32_atomic_a8:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_atomic_a8(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 8
   ret i32 %v
 }
 
+;===----------------------------------------------------------------------------
+; Atomic stores
+;===----------------------------------------------------------------------------
+
 ; CHECK-LABEL: sti32_atomic_a4:
 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32_atomic_a4(i32 *%p, i32 %v) {
  store atomic i32 %v, i32* %p seq_cst, align 4
  ret void
 }
 
+; 8 is greater than the default alignment so it is ignored.
+
 ; CHECK-LABEL: sti32_atomic_a8:
 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32_atomic_a8(i32 *%p, i32 %v) {
  store atomic i32 %v, i32* %p seq_cst, align 8
  ret void
 }

test/CodeGen/WebAssembly/i64-load-store-alignment.ll

 ; RUN: llc < %s -mattr=+atomics -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; Loads.
+;===----------------------------------------------------------------------------
+; Loads
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: ldi64_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.load $push[[NUM:[0-9]+]]=, 0($0):p2align=0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_a1(i64 *%p) {
   %v = load i64, i64* %p, align 1

 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_a16(i64 *%p) {
   %v = load i64, i64* %p, align 16
   ret i64 %v
 }
 
-; Extending loads.
+;===----------------------------------------------------------------------------
+; Extending loads
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: ldi8_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.load8_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi8_a1(i8 *%p) {
   %v = load i8, i8* %p, align 1

 ; CHECK-NEXT: i64.load32_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi32_a8(i32 *%p) {
   %v = load i32, i32* %p, align 8
   %w = zext i32 %v to i64
   ret i64 %w
 }
 
-; Stores.
+;===----------------------------------------------------------------------------
+; Stores
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: sti64_a1:
 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.store 0($0):p2align=0, $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti64_a1(i64 *%p, i64 %v) {
   store i64 %v, i64* %p, align 1
   ret void

 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti64_a16(i64 *%p, i64 %v) {
   store i64 %v, i64* %p, align 16
   ret void
 }
 
-; Truncating stores.
+;===----------------------------------------------------------------------------
+; Truncating stores
+;===----------------------------------------------------------------------------
 
 ; CHECK-LABEL: sti8_a1:
 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.store8 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti8_a1(i8 *%p, i64 %w) {
   %v = trunc i64 %w to i8
   store i8 %v, i8* %p, align 1

 ; CHECK-NEXT: i64.store32 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32_a8(i32 *%p, i64 %w) {
   %v = trunc i64 %w to i32
   store i32 %v, i32* %p, align 8
   ret void
 }
 
-; Atomics.
+;===----------------------------------------------------------------------------
+; Atomic loads
+;===----------------------------------------------------------------------------
+
 ; Wasm atomics have the alignment field, but it must always have the type's
 ; natural alignment.
 
 ; CHECK-LABEL: ldi64_atomic_a8:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_atomic_a8(i64 *%p) {
   %v = load atomic i64, i64* %p seq_cst, align 8
   ret i64 %v
 }
 
 ; 16 is greater than the default alignment so it is ignored.
+
 ; CHECK-LABEL: ldi64_atomic_a16:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_atomic_a16(i64 *%p) {
   %v = load atomic i64, i64* %p seq_cst, align 16
   ret i64 %v
 }
 
+;===----------------------------------------------------------------------------
+; Atomic stores
+;===----------------------------------------------------------------------------
+
 ; CHECK-LABEL: sti64_atomic_a4:
 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti64_atomic_a4(i64 *%p, i64 %v) {
  store atomic i64 %v, i64* %p seq_cst, align 8
  ret void
 }
 
 ; 16 is greater than the default alignment so it is ignored.
+
 ; CHECK-LABEL: sti64_atomic_a8:
 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti64_atomic_a8(i64 *%p, i64 %v) {
  store atomic i64 %v, i64* %p seq_cst, align 16
  ret void
 }

test/CodeGen/WebAssembly/offset-atomics.ll

 ; RUN: not llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals -mattr=+atomics,+sign-ext | FileCheck %s
 
 ; Test that atomic loads are assembled properly.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
+;===----------------------------------------------------------------------------
+; Atomic loads: 32-bit
+;===----------------------------------------------------------------------------
+
 ; Basic load.
 
 ; CHECK-LABEL: load_i32_no_offset:
-; CHECK: i32.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.atomic.load $push0=, 0($0){{$}}
+; CHECK-NEXT: return $pop0{{$}}
 define i32 @load_i32_no_offset(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 4
   ret i32 %v
 }
 
 ; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: load_i32_with_folded_offset:
-; CHECK: i32.atomic.load  $push0=, 24($0){{$}}
+; CHECK: i32.atomic.load $push0=, 24($0){{$}}
 define i32 @load_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
 ; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: load_i32_with_folded_gep_offset:
-; CHECK: i32.atomic.load  $push0=, 24($0){{$}}
+; CHECK: i32.atomic.load $push0=, 24($0){{$}}
 define i32 @load_i32_with_folded_gep_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 6
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
 ; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: load_i32_with_unfolded_gep_negative_offset:
 ; CHECK: i32.const $push0=, -24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i32.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.load $push2=, 0($pop1){{$}}
 define i32 @load_i32_with_unfolded_gep_negative_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 -6
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
 ; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: load_i32_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i32.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.load $push2=, 0($pop1){{$}}
 define i32 @load_i32_with_unfolded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
 ; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: load_i32_with_unfolded_gep_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i32.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.load $push2=, 0($pop1){{$}}
 define i32 @load_i32_with_unfolded_gep_offset(i32* %p) {
   %s = getelementptr i32, i32* %p, i32 6
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
+; When loading from a fixed address, materialize a zero.
+
+; CHECK-LABEL: load_i32_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.load $push1=, 42($pop0){{$}}
+define i32 @load_i32_from_numeric_address() {
+  %s = inttoptr i32 42 to i32*
+  %t = load atomic i32, i32* %s seq_cst, align 4
+  ret i32 %t
+}
+
+; CHECK-LABEL: load_i32_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.load $push1=, gv($pop0){{$}}
+@gv = global i32 0
+define i32 @load_i32_from_global_address() {
+  %t = load atomic i32, i32* @gv seq_cst, align 4
+  ret i32 %t
+}
+
+;===----------------------------------------------------------------------------
+; Atomic loads: 64-bit
+;===----------------------------------------------------------------------------
+
+; Basic load.
+
 ; CHECK-LABEL: load_i64_no_offset:
-; CHECK: i64.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.atomic.load $push0=, 0($0){{$}}
+; CHECK-NEXT: return $pop0{{$}}
 define i64 @load_i64_no_offset(i64 *%p) {
   %v = load atomic i64, i64* %p seq_cst, align 8
   ret i64 %v
 }
 
-; Same as above but with i64.
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: load_i64_with_folded_offset:
-; CHECK: i64.atomic.load  $push0=, 24($0){{$}}
+; CHECK: i64.atomic.load $push0=, 24($0){{$}}
 define i64 @load_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load atomic i64, i64* %s seq_cst, align 8
   ret i64 %t
 }
 
-; Same as above but with i64.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: load_i64_with_folded_gep_offset:
-; CHECK: i64.atomic.load  $push0=, 24($0){{$}}
+; CHECK: i64.atomic.load $push0=, 24($0){{$}}
 define i64 @load_i64_with_folded_gep_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 3
   %t = load atomic i64, i64* %s seq_cst, align 8
   ret i64 %t
 }
 
-; Same as above but with i64.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: load_i64_with_unfolded_gep_negative_offset:
 ; CHECK: i32.const $push0=, -24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i64.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.load $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_gep_negative_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 -3
   %t = load atomic i64, i64* %s seq_cst, align 8
   ret i64 %t
 }
 
-; Same as above but with i64.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: load_i64_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i64.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.load $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load atomic i64, i64* %s seq_cst, align 8
   ret i64 %t
 }
 
-; Same as above but with i64.
+; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: load_i64_with_unfolded_gep_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
-; CHECK: i32.add   $push1=, $0, $pop0{{$}}
-; CHECK: i64.atomic.load  $push2=, 0($pop1){{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.load $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_gep_offset(i64* %p) {
   %s = getelementptr i64, i64* %p, i32 3
   %t = load atomic i64, i64* %s seq_cst, align 8
   ret i64 %t
 }
 
-; CHECK-LABEL: load_i32_with_folded_or_offset:
-; CHECK: i32.atomic.load8_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}){{$}}
-; CHECK-NEXT: i32.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
-define i32 @load_i32_with_folded_or_offset(i32 %x) {
-  %and = and i32 %x, -4
-  %t0 = inttoptr i32 %and to i8*
-  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
-  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 8
-  %conv = sext i8 %t1 to i32
-  ret i32 %conv
-}
+;===----------------------------------------------------------------------------
+; Atomic stores: 32-bit
+;===----------------------------------------------------------------------------
 
-; Same as above but with store.
+; Basic store.
 
 ; CHECK-LABEL: store_i32_no_offset:
 ; CHECK-NEXT: .param i32, i32{{$}}
 ; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @store_i32_no_offset(i32 *%p, i32 %v) {
   store atomic i32 %v, i32* %p seq_cst, align 4
   ret void
 }
 
-; Same as above but with store.
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: store_i32_with_folded_offset:
 ; CHECK: i32.atomic.store 24($0), $pop0{{$}}
 define void @store_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store atomic i32 0, i32* %s seq_cst, align 4
   ret void
 }
 
-; Same as above but with store.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: store_i32_with_folded_gep_offset:
 ; CHECK: i32.atomic.store 24($0), $pop0{{$}}
 define void @store_i32_with_folded_gep_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 6
   store atomic i32 0, i32* %s seq_cst, align 4
   ret void
 }
 
-; Same as above but with store.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: store_i32_with_unfolded_gep_negative_offset:
-; CHECK: i32.const        $push0=, -24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_gep_negative_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 -6
   store atomic i32 0, i32* %s seq_cst, align 4
   ret void
 }
 
-; Same as above but with store.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: store_i32_with_unfolded_offset:
-; CHECK: i32.const        $push0=, 24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store atomic i32 0, i32* %s seq_cst, align 4
   ret void
 }
 
-; Same as above but with store.
+; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: store_i32_with_unfolded_gep_offset:
-; CHECK: i32.const        $push0=, 24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_gep_offset(i32* %p) {
   %s = getelementptr i32, i32* %p, i32 6
   store atomic i32 0, i32* %s seq_cst, align 4
   ret void
 }
 
-; Same as above but with store with i64.
+; When storing from a fixed address, materialize a zero.
+
+; CHECK-LABEL: store_i32_to_numeric_address:
+; CHECK-NEXT: i32.const $push0=, 0{{$}}
+; CHECK-NEXT: i32.const $push1=, 0{{$}}
+; CHECK-NEXT: i32.atomic.store 42($pop0), $pop1{{$}}
+define void @store_i32_to_numeric_address() {
+  %s = inttoptr i32 42 to i32*
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_i32_to_global_address:
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.const $push1=, 0{{$}}
+; CHECK: i32.atomic.store gv($pop0), $pop1{{$}}
+define void @store_i32_to_global_address() {
+  store atomic i32 0, i32* @gv seq_cst, align 4
+  ret void
+}
+
+;===----------------------------------------------------------------------------
+; Atomic stores: 64-bit
+;===----------------------------------------------------------------------------
+
+; Basic store.
 
 ; CHECK-LABEL: store_i64_no_offset:
 ; CHECK-NEXT: .param i32, i64{{$}}
 ; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @store_i64_no_offset(i64 *%p, i64 %v) {
   store atomic i64 %v, i64* %p seq_cst, align 8
   ret void
 }
 
-; Same as above but with store with i64.
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: store_i64_with_folded_offset:
 ; CHECK: i64.atomic.store 24($0), $pop0{{$}}
 define void @store_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store atomic i64 0, i64* %s seq_cst, align 8
   ret void
 }
 
-; Same as above but with store with i64.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: store_i64_with_folded_gep_offset:
 ; CHECK: i64.atomic.store 24($0), $pop0{{$}}
 define void @store_i64_with_folded_gep_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 3
   store atomic i64 0, i64* %s seq_cst, align 8
   ret void
 }
 
-; Same as above but with store with i64.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: store_i64_with_unfolded_gep_negative_offset:
-; CHECK: i32.const        $push0=, -24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_gep_negative_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 -3
   store atomic i64 0, i64* %s seq_cst, align 8
   ret void
 }
 
-; Same as above but with store with i64.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: store_i64_with_unfolded_offset:
-; CHECK: i32.const        $push0=, 24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store atomic i64 0, i64* %s seq_cst, align 8
   ret void
 }
 
-; Same as above but with store with i64.
+; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: store_i64_with_unfolded_gep_offset:
-; CHECK: i32.const        $push0=, 24{{$}}
-; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add $push1=, $0, $pop0{{$}}
 ; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_gep_offset(i64* %p) {
   %s = getelementptr i64, i64* %p, i32 3
   store atomic i64 0, i64* %s seq_cst, align 8
   ret void
 }
 
-; When loading from a fixed address, materialize a zero.
-
-; CHECK-LABEL: load_i32_from_numeric_address
-; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.load  $push1=, 42($pop0){{$}}
-define i32 @load_i32_from_numeric_address() {
-  %s = inttoptr i32 42 to i32*
-  %t = load atomic i32, i32* %s seq_cst, align 4
-  ret i32 %t
-}
-
-
-; CHECK-LABEL: load_i32_from_global_address
-; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.load  $push1=, gv($pop0){{$}}
-@gv = global i32 0
-define i32 @load_i32_from_global_address() {
-  %t = load atomic i32, i32* @gv seq_cst, align 4
-  ret i32 %t
-}
-
-; CHECK-LABEL: store_i32_to_numeric_address:
-; CHECK-NEXT: i32.const        $push0=, 0{{$}}
-; CHECK-NEXT: i32.const        $push1=, 0{{$}}
-; CHECK-NEXT: i32.atomic.store 42($pop0), $pop1{{$}}
-define void @store_i32_to_numeric_address() {
-  %s = inttoptr i32 42 to i32*
-  store atomic i32 0, i32* %s seq_cst, align 4
-  ret void
-}
-
-; CHECK-LABEL: store_i32_to_global_address:
-; CHECK: i32.const        $push0=, 0{{$}}
-; CHECK: i32.const        $push1=, 0{{$}}
-; CHECK: i32.atomic.store gv($pop0), $pop1{{$}}
-define void @store_i32_to_global_address() {
-  store atomic i32 0, i32* @gv seq_cst, align 4
-  ret void
-}
+;===----------------------------------------------------------------------------
+; Atomic sign-extending loads
+;===----------------------------------------------------------------------------
 
 ; Fold an offset into a sign-extending load.
 
-; CHECK-LABEL: load_i8_s_with_folded_offset:
+; CHECK-LABEL: load_i8_i32_s_with_folded_offset:
 ; CHECK: i32.atomic.load8_u $push0=, 24($0){{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
-define i32 @load_i8_s_with_folded_offset(i8* %p) {
+define i32 @load_i8_i32_s_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %t = load atomic i8, i8* %s seq_cst, align 1
   %u = sext i8 %t to i32
   ret i32 %u
 }
 
+; 32->64 sext load gets selected as i32.atomic.load, i64_extend_s/i32
+; CHECK-LABEL: load_i32_i64_s_with_folded_offset:
+; CHECK: i32.atomic.load $push0=, 24($0){{$}}
+; CHECK-NEXT: i64.extend_s/i32 $push1=, $pop0{{$}}
+define i64 @load_i32_i64_s_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = load atomic i32, i32* %s seq_cst, align 4
+  %u = sext i32 %t to i64
+  ret i64 %u
+}
+
 ; Fold a gep offset into a sign-extending load.
 
-; CHECK-LABEL: load_i8_s_with_folded_gep_offset:
+; CHECK-LABEL: load_i8_i32_s_with_folded_gep_offset:
 ; CHECK: i32.atomic.load8_u $push0=, 24($0){{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
-define i32 @load_i8_s_with_folded_gep_offset(i8* %p) {
+define i32 @load_i8_i32_s_with_folded_gep_offset(i8* %p) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %t = load atomic i8, i8* %s seq_cst, align 1
   %u = sext i8 %t to i32
   ret i32 %u
 }
 
-; CHECK-LABEL: load_i16_s_i64_with_folded_gep_offset:
-; CHECK: i64.atomic.load16_u  $push0=, 6($0){{$}}
-define i64 @load_i16_s_i64_with_folded_gep_offset(i16* %p) {
-  %s = getelementptr inbounds i16, i16* %p, i32 3
+; CHECK-LABEL: load_i16_i32_s_with_folded_gep_offset:
+; CHECK: i32.atomic.load16_u $push0=, 48($0){{$}}
+; CHECK-NEXT: i32.extend16_s $push1=, $pop0
+define i32 @load_i16_i32_s_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
   %t = load atomic i16, i16* %s seq_cst, align 2
-  %u = zext i16 %t to i64
+  %u = sext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i16_i64_s_with_folded_gep_offset:
+; CHECK: i64.atomic.load16_u $push0=, 48($0){{$}}
+; CHECK-NEXT: i64.extend16_s $push1=, $pop0
+define i64 @load_i16_i64_s_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = load atomic i16, i16* %s seq_cst, align 2
+  %u = sext i16 %t to i64
   ret i64 %u
 }
 
-; CHECK-LABEL: load_i64_with_folded_or_offset:
+; 'add' in this code becomes 'or' after DAG optimization. Treat an 'or' node as
+; an 'add' if the or'ed bits are known to be zero.
+
+; CHECK-LABEL: load_i8_i32_s_with_folded_or_offset:
+; CHECK: i32.atomic.load8_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}){{$}}
+; CHECK-NEXT: i32.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
+define i32 @load_i8_i32_s_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %t0 = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
+  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 1
+  %conv = sext i8 %t1 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: load_i8_i64_s_with_folded_or_offset:
 ; CHECK: i64.atomic.load8_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}){{$}}
 ; CHECK-NEXT: i64.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
-define i64 @load_i64_with_folded_or_offset(i32 %x) {
+define i64 @load_i8_i64_s_with_folded_or_offset(i32 %x) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
   %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
-  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 8
+  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 1
   %conv = sext i8 %t1 to i64
   ret i64 %conv
 }
 
+; When loading from a fixed address, materialize a zero.
+
+; CHECK-LABEL: load_i16_i32_s_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.load16_u $push1=, 42($pop0){{$}}
+; CHECK-NEXT: i32.extend16_s $push2=, $pop1
+define i32 @load_i16_i32_s_from_numeric_address() {
+  %s = inttoptr i32 42 to i16*
+  %t = load atomic i16, i16* %s seq_cst, align 2
+  %u = sext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i8_i32_s_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.atomic.load8_u $push1=, gv8($pop0){{$}}
+; CHECK-NEXT: i32.extend8_s $push2=, $pop1{{$}}
+@gv8 = global i8 0
+define i32 @load_i8_i32_s_from_global_address() {
+  %t = load atomic i8, i8* @gv8 seq_cst, align 1
+  %u = sext i8 %t to i32
+  ret i32 %u
+}
+
+;===----------------------------------------------------------------------------
+; Atomic zero-extending loads
+;===----------------------------------------------------------------------------
 
 ; Fold an offset into a zero-extending load.
 
-; CHECK-LABEL: load_i16_u_with_folded_offset:
-; CHECK: i32.atomic.load16_u $push0=, 24($0){{$}}
-define i32 @load_i16_u_with_folded_offset(i8* %p) {
+; CHECK-LABEL: load_i8_i32_z_with_folded_offset:
+; CHECK: i32.atomic.load8_u $push0=, 24($0){{$}}
+define i32 @load_i8_i32_z_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i16*
-  %t = load atomic i16, i16* %s seq_cst, align 2
-  %u = zext i16 %t to i32
+  %s = inttoptr i32 %r to i8*
+  %t = load atomic i8, i8* %s seq_cst, align 1
+  %u = zext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i32_i64_z_with_folded_offset:
+; CHECK: i64.atomic.load32_u $push0=, 24($0){{$}}
+define i64 @load_i32_i64_z_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = load atomic i32, i32* %s seq_cst, align 4
+  %u = zext i32 %t to i64
+  ret i64 %u
+}
+
 ; Fold a gep offset into a zero-extending load.
 
-; CHECK-LABEL: load_i8_u_with_folded_gep_offset:
+; CHECK-LABEL: load_i8_i32_z_with_folded_gep_offset:
 ; CHECK: i32.atomic.load8_u $push0=, 24($0){{$}}
-define i32 @load_i8_u_with_folded_gep_offset(i8* %p) {
+define i32 @load_i8_i32_z_with_folded_gep_offset(i8* %p) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %t = load atomic i8, i8* %s seq_cst, align 1
   %u = zext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i16_i32_z_with_folded_gep_offset:
+; CHECK: i32.atomic.load16_u $push0=, 48($0){{$}}
+define i32 @load_i16_i32_z_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = load atomic i16, i16* %s seq_cst, align 2
+  %u = zext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i16_i64_z_with_folded_gep_offset:
+; CHECK: i64.atomic.load16_u $push0=, 48($0){{$}}
+define i64 @load_i16_i64_z_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i64 24
+  %t = load atomic i16, i16* %s seq_cst, align 2
+  %u = zext i16 %t to i64
+  ret i64 %u
+}
+
+; 'add' in this code becomes 'or' after DAG optimization. Treat an 'or' node as
+; an 'add' if the or'ed bits are known to be zero.
+
+; CHECK-LABEL: load_i8_i32_z_with_folded_or_offset:
+; CHECK: i32.atomic.load8_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}){{$}}
+define i32 @load_i8_i32_z_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %t0 = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
+  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 1
+  %conv = zext i8 %t1 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: load_i8_i64_z_with_folded_or_offset:
+; CHECK: i64.atomic.load8_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}){{$}}
+define i64 @load_i8_i64_z_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %t0 = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
+  %t1 = load atomic i8, i8* %arrayidx seq_cst, align 1
+  %conv = zext i8 %t1 to i64
+  ret i64 %conv
+}
 
 ; When loading from a fixed address, materialize a zero.
-; As above but with extending load.
 
-; CHECK-LABEL: load_zext_i32_from_numeric_address
+; CHECK-LABEL: load_i16_i32_z_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.load16_u  $push1=, 42($pop0){{$}}
-define i32 @load_zext_i32_from_numeric_address() {
+; CHECK: i32.atomic.load16_u $push1=, 42($pop0){{$}}
+define i32 @load_i16_i32_z_from_numeric_address() {
   %s = inttoptr i32 42 to i16*
   %t = load atomic i16, i16* %s seq_cst, align 2
   %u = zext i16 %t to i32
   ret i32 %u
 }
 
-; CHECK-LABEL: load_sext_i32_from_global_address
+; CHECK-LABEL: load_i8_i32_z_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.load8_u  $push1=, gv8($pop0){{$}}
-; CHECK-NEXT: i32.extend8_s $push2=, $pop1{{$}}
-@gv8 = global i8 0
-define i32 @load_sext_i32_from_global_address() {
+; CHECK: i32.atomic.load8_u $push1=, gv8($pop0){{$}}
+define i32 @load_i8_i32_z_from_global_address() {
   %t = load atomic i8, i8* @gv8 seq_cst, align 1
-  %u = sext i8 %t to i32
+  %u = zext i8 %t to i32
   ret i32 %u
 }
 
-; Fold an offset into a sign-extending load.
-; As above but 32 extended to 64 bit.
-; CHECK-LABEL: load_i32_i64_s_with_folded_offset:
-; CHECK: i32.atomic.load $push0=, 24($0){{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push1=, $pop0{{$}}
-define i64 @load_i32_i64_s_with_folded_offset(i32* %p) {
-  %q = ptrtoint i32* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i32*
-  %t = load atomic i32, i32* %s seq_cst, align 4
-  %u = sext i32 %t to i64
-  ret i64 %u
-}
-
-; Fold a gep offset into a zero-extending load.
-; As above but 32 extended to 64 bit.
-; CHECK-LABEL: load_i32_i64_u_with_folded_gep_offset:
-; CHECK: i64.atomic.load32_u $push0=, 96($0){{$}}
-define i64 @load_i32_i64_u_with_folded_gep_offset(i32* %p) {
-  %s = getelementptr inbounds i32, i32* %p, i32 24
-  %t = load atomic i32, i32* %s seq_cst, align 4
-  %u = zext i32 %t to i64
-  ret i64 %u
-}
-
 ; i8 return value should test anyext loads
-; CHECK-LABEL: ldi8_a1:
-; CHECK: i32.atomic.load8_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
-define i8 @ldi8_a1(i8 *%p) {
+
+; CHECK-LABEL: load_i8_i32_retvalue:
+; CHECK: i32.atomic.load8_u $push0=, 0($0){{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i8 @load_i8_i32_retvalue(i8 *%p) {
   %v = load atomic i8, i8* %p seq_cst, align 1
   ret i8 %v
 }
 
-; Fold an offset into a truncating store.
-
-; CHECK-LABEL: store_i8_with_folded_offset:
-; CHECK: i32.atomic.store8 24($0), $pop0{{$}}
-define void @store_i8_with_folded_offset(i8* %p) {
-  %q = ptrtoint i8* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i8*
-  store atomic i8 0, i8* %s seq_cst, align 1
-  ret void
-}
+;===----------------------------------------------------------------------------
+; Atomic truncating stores
+;===----------------------------------------------------------------------------
 
-; CHECK-LABEL: store_i16_with_folded_offset:
-; CHECK: i32.atomic.store16 24($0), $pop0{{$}}
-define void @store_i16_with_folded_offset(i16* %p) {
-  %q = ptrtoint i16* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i16*
-  store atomic i16 0, i16* %s seq_cst, align 2
-  ret void
-}
+; Fold an offset into a truncating store.
 
-; CHECK-LABEL: store_i8_i64_with_folded_offset:
-; CHECK: i64.atomic.store8 24($0), $1{{$}}
-define void @store_i8_i64_with_folded_offset(i8* %p, i64 %v) {
+; CHECK-LABEL: store_i8_i32_with_folded_offset:
+; CHECK: i32.atomic.store8 24($0), $1{{$}}
+define void @store_i8_i32_with_folded_offset(i8* %p, i32 %v) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
-  %t = trunc i64 %v to i8
+  %t = trunc i32 %v to i8
   store atomic i8 %t, i8* %s seq_cst, align 1
   ret void
 }
 
-; CHECK-LABEL: store_i16_i64_with_folded_offset:
-; CHECK: i64.atomic.store16 24($0), $1{{$}}
-define void @store_i16_i64_with_folded_offset(i16* %p, i64 %v) {
-  %q = ptrtoint i16* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i16*
-  %t = trunc i64 %v to i16
-  store atomic i16 %t, i16* %s seq_cst, align 2
-  ret void
-}
-
 ; CHECK-LABEL: store_i32_i64_with_folded_offset:
 ; CHECK: i64.atomic.store32 24($0), $1{{$}}
 define void @store_i32_i64_with_folded_offset(i32* %p, i64 %v) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %t = trunc i64 %v to i32
   store atomic i32 %t, i32* %s seq_cst, align 4
   ret void
 }
 
 ; Fold a gep offset into a truncating store.
 
-; CHECK-LABEL: store_i8_with_folded_gep_offset:
-; CHECK: i32.atomic.store8 24($0), $pop0{{$}}
-define void @store_i8_with_folded_gep_offset(i8* %p) {
+; CHECK-LABEL: store_i8_i32_with_folded_gep_offset:
+; CHECK: i32.atomic.store8 24($0), $1{{$}}
+define void @store_i8_i32_with_folded_gep_offset(i8* %p, i32 %v) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
-  store atomic i8 0, i8* %s seq_cst, align 1
+  %t = trunc i32 %v to i8
+  store atomic i8 %t, i8* %s seq_cst, align 1
   ret void
 }
 
-; CHECK-LABEL: store_i16_with_folded_gep_offset:
-; CHECK: i32.atomic.store16 48($0), $pop0{{$}}
-define void @store_i16_with_folded_gep_offset(i16* %p) {
+; CHECK-LABEL: store_i16_i32_with_folded_gep_offset:
+; CHECK: i32.atomic.store16 48($0), $1{{$}}
+define void @store_i16_i32_with_folded_gep_offset(i16* %p, i32 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
-  store atomic i16 0, i16* %s seq_cst, align 2
-  ret void
-}
-
-; CHECK-LABEL: store_i8_i64_with_folded_gep_offset:
-; CHECK: i64.atomic.store8 24($0), $1{{$}}
-define void @store_i8_i64_with_folded_gep_offset(i8* %p, i64 %v) {
-  %s = getelementptr inbounds i8, i8* %p, i32 24
-  %t = trunc i64 %v to i8
-  store atomic i8 %t, i8* %s seq_cst, align 2
+  %t = trunc i32 %v to i16
+  store atomic i16 %t, i16* %s seq_cst, align 2
   ret void
 }
 
 ; CHECK-LABEL: store_i16_i64_with_folded_gep_offset:
 ; CHECK: i64.atomic.store16 48($0), $1{{$}}
 define void @store_i16_i64_with_folded_gep_offset(i16* %p, i64 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
   %t = trunc i64 %v to i16
   store atomic i16 %t, i16* %s seq_cst, align 2
   ret void
 }
 
-; CHECK-LABEL: store_i32_i64_with_folded_gep_offset:
-; CHECK: i64.atomic.store32 96($0), $1{{$}}
-define void @store_i32_i64_with_folded_gep_offset(i32* %p, i64 %v) {
-  %s = getelementptr inbounds i32, i32* %p, i32 24
-  %t = trunc i64 %v to i32
-  store atomic i32 %t, i32* %s seq_cst, align 4
-  ret void
-}
-
-; Fold an or_is_add pattern based offset into a truncating store.
+; 'add' in this code becomes 'or' after DAG optimization. Treat an 'or' node as
+; an 'add' if the or'ed bits are known to be zero.
 
-; CHECK-LABEL: store_i8_with_folded_or_offset:
-; CHECK: i32.atomic.store8 2($pop{{[0-9]+}}), $pop{{[0-9]+}}{{$}}
-define void @store_i8_with_folded_or_offset(i32 %x) {
+; CHECK-LABEL: store_i8_i32_with_folded_or_offset:
+; CHECK: i32.atomic.store8 2($pop{{[0-9]+}}), $1{{$}}
+define void @store_i8_i32_with_folded_or_offset(i32 %x, i32 %v) {
   %and = and i32 %x, -4
   %p = inttoptr i32 %and to i8*
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
-  store atomic i8 0, i8* %arrayidx seq_cst, align 1
-  ret void
-}
-
-; CHECK-LABEL: store_i16_with_folded_or_offset:
-; CHECK: i32.atomic.store16 4($pop{{[0-9]+}}), $pop{{[0-9]+}}{{$}}
-define void @store_i16_with_folded_or_offset(i32 %x) {
-  %and = and i32 %x, -4
-  %p = inttoptr i32 %and to i16*
-  %arrayidx = getelementptr inbounds i16, i16* %p, i32 2
-  store atomic i16 0, i16* %arrayidx seq_cst, align 2
+  %t = trunc i32 %v to i8
+  store atomic i8 %t, i8* %arrayidx seq_cst, align 1
   ret void
 }
 
 ; CHECK-LABEL: store_i8_i64_with_folded_or_offset:
 ; CHECK: i64.atomic.store8 2($pop{{[0-9]+}}), $1{{$}}
 define void @store_i8_i64_with_folded_or_offset(i32 %x, i64 %v) {
   %and = and i32 %x, -4
   %p = inttoptr i32 %and to i8*
   %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
   %t = trunc i64 %v to i8
   store atomic i8 %t, i8* %arrayidx seq_cst, align 1
   ret void
 }
-
-; CHECK-LABEL: store_i16_i64_with_folded_or_offset:
-; CHECK: i64.atomic.store16 4($pop{{[0-9]+}}), $1{{$}}
-define void @store_i16_i64_with_folded_or_offset(i32 %x, i64 %v) {
-  %and = and i32 %x, -4
-  %p = inttoptr i32 %and to i16*
-  %arrayidx = getelementptr inbounds i16, i16* %p, i32 2
-  %t = trunc i64 %v to i16
-  store atomic i16 %t, i16* %arrayidx seq_cst, align 2
-  ret void
-}
-
-; CHECK-LABEL: store_i32_i64_with_folded_or_offset:
-; CHECK: i64.atomic.store32 8($pop{{[0-9]+}}), $1{{$}}
-define void @store_i32_i64_with_folded_or_offset(i32 %x, i64 %v) {
-  %and = and i32 %x, -4
-  %p = inttoptr i32 %and to i32*
-  %arrayidx = getelementptr inbounds i32, i32* %p, i32 2
-  %t = trunc i64 %v to i32
-  store atomic i32 %t, i32* %arrayidx seq_cst, align 4
-  ret void
-}

test/CodeGen/WebAssembly/offset.ll

-; RUN: llc < %s -asm-verbose=false -disable-wasm-explicit-locals | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-explicit-locals -disable-wasm-fallthrough-return-opt | FileCheck %s
 
 ; Test constant load and store address offsets.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
+;===----------------------------------------------------------------------------
+; Loads: 32-bit
+;===----------------------------------------------------------------------------
+
+; Basic load.
+
+; CHECK-LABEL: load_i32_no_offset:
+; CHECK: i32.load $push0=, 0($0){{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i32 @load_i32_no_offset(i32 *%p) {
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
 ; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: load_i32_with_folded_offset:
 ; CHECK: i32.load  $push0=, 24($0){{$}}
 define i32 @load_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*

 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.load  $push2=, 0($pop1){{$}}
 define i32 @load_i32_with_unfolded_gep_offset(i32* %p) {
   %s = getelementptr i32, i32* %p, i32 6
   %t = load i32, i32* %s
   ret i32 %t
 }
 
-; Same as above but with i64.
+; When loading from a fixed address, materialize a zero.
+
+; CHECK-LABEL: load_i32_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load  $push1=, 42($pop0){{$}}
+define i32 @load_i32_from_numeric_address() {
+  %s = inttoptr i32 42 to i32*
+  %t = load i32, i32* %s
+  ret i32 %t
+}
+
+; CHECK-LABEL: load_i32_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load  $push1=, gv($pop0){{$}}
+@gv = global i32 0
+define i32 @load_i32_from_global_address() {
+  %t = load i32, i32* @gv
+  ret i32 %t
+}
+
+;===----------------------------------------------------------------------------
+; Loads: 64-bit
+;===----------------------------------------------------------------------------
+
+; Basic load.
+
+; CHECK-LABEL: load_i64_no_offset:
+; CHECK: i64.load $push0=, 0($0){{$}}
+; CHECK-NEXT: return $pop0{{$}}
+define i64 @load_i64_no_offset(i64 *%p) {
+  %v = load i64, i64* %p
+  ret i64 %v
+}
+
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: load_i64_with_folded_offset:
 ; CHECK: i64.load  $push0=, 24($0){{$}}
 define i64 @load_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load i64, i64* %s
   ret i64 %t
 }
 
-; Same as above but with i64.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: load_i64_with_folded_gep_offset:
 ; CHECK: i64.load  $push0=, 24($0){{$}}
 define i64 @load_i64_with_folded_gep_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 3
   %t = load i64, i64* %s
   ret i64 %t
 }
 
-; Same as above but with i64.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: load_i64_with_unfolded_gep_negative_offset:
 ; CHECK: i32.const $push0=, -24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.load  $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_gep_negative_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 -3
   %t = load i64, i64* %s
   ret i64 %t
 }
 
-; Same as above but with i64.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: load_i64_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.load  $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load i64, i64* %s
   ret i64 %t
 }
 
-; Same as above but with i64.
+; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: load_i64_with_unfolded_gep_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.load  $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_gep_offset(i64* %p) {
   %s = getelementptr i64, i64* %p, i32 3
   %t = load i64, i64* %s
   ret i64 %t
 }
 
-; CHECK-LABEL: load_i32_with_folded_or_offset:
-; CHECK: i32.load8_s $push{{[0-9]+}}=, 2($pop{{[0-9]+}}){{$}}
-define i32 @load_i32_with_folded_or_offset(i32 %x) {
-  %and = and i32 %x, -4
-  %t0 = inttoptr i32 %and to i8*
-  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
-  %t1 = load i8, i8* %arrayidx, align 1
-  %conv = sext i8 %t1 to i32
-  ret i32 %conv
+;===----------------------------------------------------------------------------
+; Stores: 32-bit
+;===----------------------------------------------------------------------------
+
+; Basic store.
+
+; CHECK-LABEL: store_i32_no_offset:
+; CHECK-NEXT: .param i32, i32{{$}}
+; CHECK-NEXT: i32.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @store_i32_no_offset(i32 *%p, i32 %v) {
+  store i32 %v, i32* %p
+  ret void
 }
 
-; Same as above but with store.
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: store_i32_with_folded_offset:
 ; CHECK: i32.store 24($0), $pop0{{$}}
 define void @store_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store i32 0, i32* %s
   ret void
 }
 
-; Same as above but with store.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: store_i32_with_folded_gep_offset:
 ; CHECK: i32.store 24($0), $pop0{{$}}
 define void @store_i32_with_folded_gep_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 6
   store i32 0, i32* %s
   ret void
 }
 
-; Same as above but with store.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: store_i32_with_unfolded_gep_negative_offset:
 ; CHECK: i32.const $push0=, -24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_gep_negative_offset(i32* %p) {
   %s = getelementptr inbounds i32, i32* %p, i32 -6
   store i32 0, i32* %s
   ret void
 }
 
-; Same as above but with store.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: store_i32_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store i32 0, i32* %s
   ret void
 }
 
-; Same as above but with store.
+; Without inbounds, we can't fold a gep offset.
 
 ; CHECK-LABEL: store_i32_with_unfolded_gep_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.store 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_gep_offset(i32* %p) {
   %s = getelementptr i32, i32* %p, i32 6
   store i32 0, i32* %s
   ret void
 }
 
-; Same as above but with store with i64.
+; When storing from a fixed address, materialize a zero.
+
+; CHECK-LABEL: store_i32_to_numeric_address:
+; CHECK-NEXT: i32.const $push0=, 0{{$}}
+; CHECK-NEXT: i32.const $push1=, 0{{$}}
+; CHECK-NEXT: i32.store 42($pop0), $pop1{{$}}
+define void @store_i32_to_numeric_address() {
+  %s = inttoptr i32 42 to i32*
+  store i32 0, i32* %s
+  ret void
+}
+
+; CHECK-LABEL: store_i32_to_global_address:
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.const $push1=, 0{{$}}
+; CHECK: i32.store gv($pop0), $pop1{{$}}
+define void @store_i32_to_global_address() {
+  store i32 0, i32* @gv
+  ret void
+}
+
+;===----------------------------------------------------------------------------
+; Stores: 64-bit
+;===----------------------------------------------------------------------------
+
+; Basic store.
 
 ; CHECK-LABEL: store_i64_with_folded_offset:
 ; CHECK: i64.store 24($0), $pop0{{$}}
 define void @store_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store i64 0, i64* %s
   ret void
 }
 
-; Same as above but with store with i64.
+; With an nuw add, we can fold an offset.
 
 ; CHECK-LABEL: store_i64_with_folded_gep_offset:
 ; CHECK: i64.store 24($0), $pop0{{$}}
 define void @store_i64_with_folded_gep_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 3
   store i64 0, i64* %s
   ret void
 }
 
-; Same as above but with store with i64.
+; With an inbounds gep, we can fold an offset.
 
 ; CHECK-LABEL: store_i64_with_unfolded_gep_negative_offset:
 ; CHECK: i32.const $push0=, -24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_gep_negative_offset(i64* %p) {
   %s = getelementptr inbounds i64, i64* %p, i32 -3
   store i64 0, i64* %s
   ret void
 }
 
-; Same as above but with store with i64.
+; We can't fold a negative offset though, even with an inbounds gep.
 
 ; CHECK-LABEL: store_i64_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store i64 0, i64* %s
   ret void
 }
 
-; Same as above but with store with i64.
+; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: store_i64_with_unfolded_gep_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.store 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_gep_offset(i64* %p) {
   %s = getelementptr i64, i64* %p, i32 3
   store i64 0, i64* %s
   ret void
 }
 
+; Without inbounds, we can't fold a gep offset.
+
 ; CHECK-LABEL: store_i32_with_folded_or_offset:
 ; CHECK: i32.store8 2($pop{{[0-9]+}}), $pop{{[0-9]+}}{{$}}
 define void @store_i32_with_folded_or_offset(i32 %x) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
   %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
   store i8 0, i8* %arrayidx, align 1
   ret void
 }
 
-; When loading from a fixed address, materialize a zero.
-
-; CHECK-LABEL: load_i32_from_numeric_address
-; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.load  $push1=, 42($pop0){{$}}
-define i32 @load_i32_from_numeric_address() {
-  %s = inttoptr i32 42 to i32*
-  %t = load i32, i32* %s
-  ret i32 %t
-}
-
-; CHECK-LABEL: load_i32_from_global_address
-; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.load  $push1=, gv($pop0){{$}}
-@gv = global i32 0
-define i32 @load_i32_from_global_address() {
-  %t = load i32, i32* @gv
-  ret i32 %t
-}
-
-; CHECK-LABEL: store_i32_to_numeric_address:
-; CHECK-NEXT: i32.const $push0=, 0{{$}}
-; CHECK-NEXT: i32.const $push1=, 0{{$}}
-; CHECK-NEXT: i32.store 42($pop0), $pop1{{$}}
-define void @store_i32_to_numeric_address() {
-  %s = inttoptr i32 42 to i32*
-  store i32 0, i32* %s
-  ret void
-}
-
-; CHECK-LABEL: store_i32_to_global_address:
-; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.const $push1=, 0{{$}}
-; CHECK: i32.store gv($pop0), $pop1{{$}}
-define void @store_i32_to_global_address() {
-  store i32 0, i32* @gv
-  ret void
-}
+;===----------------------------------------------------------------------------
+; Sign-extending loads
+;===----------------------------------------------------------------------------
 
 ; Fold an offset into a sign-extending load.
 
-; CHECK-LABEL: load_i8_s_with_folded_offset:
+; CHECK-LABEL: load_i8_i32_s_with_folded_offset:
 ; CHECK: i32.load8_s $push0=, 24($0){{$}}
-define i32 @load_i8_s_with_folded_offset(i8* %p) {
+define i32 @load_i8_i32_s_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %t = load i8, i8* %s
   %u = sext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i32_i64_s_with_folded_offset:
+; CHECK: i64.load32_s $push0=, 24($0){{$}}
+define i64 @load_i32_i64_s_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = load i32, i32* %s
+  %u = sext i32 %t to i64
+  ret i64 %u
+}
+
 ; Fold a gep offset into a sign-extending load.
 
-; CHECK-LABEL: load_i8_s_with_folded_gep_offset:
+; CHECK-LABEL: load_i8_i32_s_with_folded_gep_offset:
 ; CHECK: i32.load8_s $push0=, 24($0){{$}}
-define i32 @load_i8_s_with_folded_gep_offset(i8* %p) {
+define i32 @load_i8_i32_s_with_folded_gep_offset(i8* %p) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %t = load i8, i8* %s
   %u = sext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i16_i32_s_with_folded_gep_offset:
+; CHECK: i32.load16_s $push0=, 48($0){{$}}
+define i32 @load_i16_i32_s_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = load i16, i16* %s
+  %u = sext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i16_i64_s_with_folded_gep_offset:
+; CHECK: i64.load16_s $push0=, 48($0){{$}}
+define i64 @load_i16_i64_s_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = load i16, i16* %s
+  %u = sext i16 %t to i64
+  ret i64 %u
+}
+
+; 'add' in this code becomes 'or' after DAG optimization. Treat an 'or' node as
+; an 'add' if the or'ed bits are known to be zero.
+
+; CHECK-LABEL: load_i8_i32_s_with_folded_or_offset:
+; CHECK: i32.load8_s $push{{[0-9]+}}=, 2($pop{{[0-9]+}}){{$}}
+define i32 @load_i8_i32_s_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %t0 = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
+  %t1 = load i8, i8* %arrayidx
+  %conv = sext i8 %t1 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: load_i8_i64_s_with_folded_or_offset:
+; CHECK: i64.load8_s $push{{[0-9]+}}=, 2($pop{{[0-9]+}}){{$}}
+define i64 @load_i8_i64_s_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %t0 = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
+  %t1 = load i8, i8* %arrayidx
+  %conv = sext i8 %t1 to i64
+  ret i64 %conv
+}
+
+; When loading from a fixed address, materialize a zero.
+
+; CHECK-LABEL: load_i16_i32_s_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load16_s  $push1=, 42($pop0){{$}}
+define i32 @load_i16_i32_s_from_numeric_address() {
+  %s = inttoptr i32 42 to i16*
+  %t = load i16, i16* %s
+  %u = sext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i8_i32_s_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load8_s  $push1=, gv8($pop0){{$}}
+@gv8 = global i8 0
+define i32 @load_i8_i32_s_from_global_address() {
+  %t = load i8, i8* @gv8
+  %u = sext i8 %t to i32
+  ret i32 %u
+}
+
+;===----------------------------------------------------------------------------
+; Zero-extending loads
+;===----------------------------------------------------------------------------
+
 ; Fold an offset into a zero-extending load.
 
-; CHECK-LABEL: load_i8_u_with_folded_offset:
+; CHECK-LABEL: load_i8_i32_z_with_folded_offset:
 ; CHECK: i32.load8_u $push0=, 24($0){{$}}
-define i32 @load_i8_u_with_folded_offset(i8* %p) {
+define i32 @load_i8_i32_z_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %t = load i8, i8* %s
   %u = zext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i32_i64_z_with_folded_offset:
+; CHECK: i64.load32_u $push0=, 24($0){{$}}
+define i64 @load_i32_i64_z_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = load i32, i32* %s
+  %u = zext i32 %t to i64
+  ret i64 %u
+}
+
 ; Fold a gep offset into a zero-extending load.
 
-; CHECK-LABEL: load_i8_u_with_folded_gep_offset:
+; CHECK-LABEL: load_i8_i32_z_with_folded_gep_offset:
 ; CHECK: i32.load8_u $push0=, 24($0){{$}}
-define i32 @load_i8_u_with_folded_gep_offset(i8* %p) {
+define i32 @load_i8_i32_z_with_folded_gep_offset(i8* %p) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %t = load i8, i8* %s
   %u = zext i8 %t to i32
   ret i32 %u
 }
 
+; CHECK-LABEL: load_i16_i32_z_with_folded_gep_offset:
+; CHECK: i32.load16_u $push0=, 48($0){{$}}
+define i32 @load_i16_i32_z_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = load i16, i16* %s
+  %u = zext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i16_i64_z_with_folded_gep_offset:
+; CHECK: i64.load16_u $push0=, 48($0){{$}}
+define i64 @load_i16_i64_z_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i64 24
+  %t = load i16, i16* %s
+  %u = zext i16 %t to i64
+  ret i64 %u
+}
+
+; When loading from a fixed address, materialize a zero.
+
+; CHECK-LABEL: load_i16_i32_z_from_numeric_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load16_u  $push1=, 42($pop0){{$}}
+define i32 @load_i16_i32_z_from_numeric_address() {
+  %s = inttoptr i32 42 to i16*
+  %t = load i16, i16* %s
+  %u = zext i16 %t to i32
+  ret i32 %u
+}
+
+; CHECK-LABEL: load_i8_i32_z_from_global_address
+; CHECK: i32.const $push0=, 0{{$}}
+; CHECK: i32.load8_u  $push1=, gv8($pop0){{$}}
+define i32 @load_i8_i32_z_from_global_address() {
+  %t = load i8, i8* @gv8
+  %u = zext i8 %t to i32
+  ret i32 %u
+}
+
+; i8 return value should test anyext loads
+; CHECK-LABEL: load_i8_i32_retvalue:
+; CHECK: i32.load8_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
+; CHECK-NEXT: return $pop[[NUM]]{{$}}
+define i8 @load_i8_i32_retvalue(i8 *%p) {
+  %v = load i8, i8* %p
+  ret i8 %v
+}
+
+;===----------------------------------------------------------------------------
+; Truncating stores
+;===----------------------------------------------------------------------------
+
 ; Fold an offset into a truncating store.
 
-; CHECK-LABEL: store_i8_with_folded_offset:
-; CHECK: i32.store8 24($0), $pop0{{$}}
-define void @store_i8_with_folded_offset(i8* %p) {
+; CHECK-LABEL: store_i8_i32_with_folded_offset:
+; CHECK: i32.store8 24($0), $1{{$}}
+define void @store_i8_i32_with_folded_offset(i8* %p, i32 %v) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
-  store i8 0, i8* %s
+  %t = trunc i32 %v to i8
+  store i8 %t, i8* %s
+  ret void
+}
+
+; CHECK-LABEL: store_i32_i64_with_folded_offset:
+; CHECK: i64.store32 24($0), $1{{$}}
+define void @store_i32_i64_with_folded_offset(i32* %p, i64 %v) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = trunc i64 %v to i32
+  store i32 %t, i32* %s
   ret void
 }
 
 ; Fold a gep offset into a truncating store.
 
-; CHECK-LABEL: store_i8_with_folded_gep_offset:
-; CHECK: i32.store8 24($0), $pop0{{$}}
-define void @store_i8_with_folded_gep_offset(i8* %p) {
+; CHECK-LABEL: store_i8_i32_with_folded_gep_offset:
+; CHECK: i32.store8 24($0), $1{{$}}
+define void @store_i8_i32_with_folded_gep_offset(i8* %p, i32 %v) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
-  store i8 0, i8* %s
+  %t = trunc i32 %v to i8
+  store i8 %t, i8* %s
+  ret void
+}
+
+; CHECK-LABEL: store_i16_i32_with_folded_gep_offset:
+; CHECK: i32.store16 48($0), $1{{$}}
+define void @store_i16_i32_with_folded_gep_offset(i16* %p, i32 %v) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = trunc i32 %v to i16
+  store i16 %t, i16* %s
   ret void
 }
 
+; CHECK-LABEL: store_i16_i64_with_folded_gep_offset:
+; CHECK: i64.store16 48($0), $1{{$}}
+define void @store_i16_i64_with_folded_gep_offset(i16* %p, i64 %v) {
+  %s = getelementptr inbounds i16, i16* %p, i64 24
+  %t = trunc i64 %v to i16
+  store i16 %t, i16* %s
+  ret void
+}
+
+; 'add' in this code becomes 'or' after DAG optimization. Treat an 'or' node as
+; an 'add' if the or'ed bits are known to be zero.
+
+; CHECK-LABEL: store_i8_i32_with_folded_or_offset:
+; CHECK: i32.store8 2($pop{{[0-9]+}}), $1{{$}}
+define void @store_i8_i32_with_folded_or_offset(i32 %x, i32 %v) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
+  %t = trunc i32 %v to i8
+  store i8 %t, i8* %arrayidx
+  ret void
+}
+
+; CHECK-LABEL: store_i8_i64_with_folded_or_offset:
+; CHECK: i64.store8 2($pop{{[0-9]+}}), $1{{$}}
+define void @store_i8_i64_with_folded_or_offset(i32 %x, i64 %v) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
+  %t = trunc i64 %v to i8
+  store i8 %t, i8* %arrayidx
+  ret void
+}
+
+;===----------------------------------------------------------------------------
+; Aggregate values
+;===----------------------------------------------------------------------------
+
 ; Fold the offsets when lowering aggregate loads and stores.
 
 ; CHECK-LABEL: aggregate_load_store:
 ; CHECK: i32.load  $2=, 0($0){{$}}
 ; CHECK: i32.load  $3=, 4($0){{$}}
 ; CHECK: i32.load  $4=, 8($0){{$}}
 ; CHECK: i32.load  $push0=, 12($0){{$}}
 ; CHECK: i32.store 12($1), $pop0{{$}}