[WebAssembly] Support for atomic stores

[aheejin]

Summary

Add support for atomic store instructions.

[aheejin]

• blank lines

[sbc100]

Nice!

[dschuff]

lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

207

😭😭😭😭😭

lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h

inline unsigned GetDefaultP2Align(unsigned Opcode) {
   case WebAssembly::ATOMIC_LOAD8_U_I32:
   case WebAssembly::ATOMIC_LOAD8_U_I32_S:
   case WebAssembly::ATOMIC_LOAD8_U_I64:
   case WebAssembly::ATOMIC_LOAD8_U_I64_S:
   case WebAssembly::STORE8_I32:
   case WebAssembly::STORE8_I32_S:
   case WebAssembly::STORE8_I64:
   case WebAssembly::STORE8_I64_S:
+  case WebAssembly::ATOMIC_STORE8_I32:
+  case WebAssembly::ATOMIC_STORE8_I64:
     return 0;
   case WebAssembly::LOAD16_S_I32:
   case WebAssembly::LOAD16_S_I32_S:
   case WebAssembly::LOAD16_U_I32:
   case WebAssembly::LOAD16_U_I32_S:
   case WebAssembly::LOAD16_S_I64:
   case WebAssembly::LOAD16_S_I64_S:
   case WebAssembly::LOAD16_U_I64:
   case WebAssembly::LOAD16_U_I64_S:
   case WebAssembly::ATOMIC_LOAD16_U_I32:
   case WebAssembly::ATOMIC_LOAD16_U_I32_S:
   case WebAssembly::ATOMIC_LOAD16_U_I64:
   case WebAssembly::ATOMIC_LOAD16_U_I64_S:
   case WebAssembly::STORE16_I32:
   case WebAssembly::STORE16_I32_S:
   case WebAssembly::STORE16_I64:
   case WebAssembly::STORE16_I64_S:
+  case WebAssembly::ATOMIC_STORE16_I32:
+  case WebAssembly::ATOMIC_STORE16_I64:
     return 1;
   case WebAssembly::LOAD_I32:
   case WebAssembly::LOAD_I32_S:
   case WebAssembly::LOAD_F32:
   case WebAssembly::LOAD_F32_S:
   case WebAssembly::STORE_I32:
   case WebAssembly::STORE_I32_S:
   case WebAssembly::STORE_F32:
   case WebAssembly::STORE_F32_S:
   case WebAssembly::LOAD32_S_I64:
   case WebAssembly::LOAD32_S_I64_S:
   case WebAssembly::LOAD32_U_I64:
   case WebAssembly::LOAD32_U_I64_S:
   case WebAssembly::STORE32_I64:
   case WebAssembly::STORE32_I64_S:
   case WebAssembly::ATOMIC_LOAD_I32:
   case WebAssembly::ATOMIC_LOAD_I32_S:
   case WebAssembly::ATOMIC_LOAD32_U_I64:
   case WebAssembly::ATOMIC_LOAD32_U_I64_S:
+  case WebAssembly::ATOMIC_STORE_I32:
+  case WebAssembly::ATOMIC_STORE32_I64:
     return 2;
   case WebAssembly::LOAD_I64:
   case WebAssembly::LOAD_I64_S:
   case WebAssembly::LOAD_F64:
   case WebAssembly::LOAD_F64_S:
   case WebAssembly::STORE_I64:
   case WebAssembly::STORE_I64_S:
   case WebAssembly::STORE_F64:
   case WebAssembly::STORE_F64_S:
   case WebAssembly::ATOMIC_LOAD_I64:
   case WebAssembly::ATOMIC_LOAD_I64_S:
+  case WebAssembly::ATOMIC_STORE_I64:
     return 3;
   default:
     llvm_unreachable("Only loads and stores have p2align values");
   }
 }
 
 /// The operand number of the load or store address in load/store instructions.
 static const unsigned LoadAddressOperandNo = 3;

lib/Target/WebAssembly/WebAssemblyInstrAtomics.td

 
 
 } // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic stores
 //===----------------------------------------------------------------------===//
 
-// TODO: add atomic stores here...
+let Defs = [ARGUMENTS] in {
+defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>;
+defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>;
+} // Defs = [ARGUMENTS]
+
+// We need an 'atomic' version of store patterns because store and atomic_store
+// nodes have different operand orders:
+// store: (store $val, $ptr)
+// atomic_store: (store $ptr, $val)
+
+let Predicates = [HasAtomics] in {
+
+// Select stores with no constant offset.
+class AStorePatNoOffset<ValueType ty, PatFrag node, NI inst> :
+  Pat<(node I32:$addr, ty:$val), (inst 0, 0, $addr, $val)>;
+def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+// Select stores with a constant offset.
+
+// Pattern with address + immediate offset
+class AStorePatImmOff<ValueType ty, PatFrag storekind, PatFrag operand,
+                      NI inst> :
+  Pat<(storekind (operand I32:$addr, imm:$off), ty:$val),
+      (inst 0, imm:$off, $addr, ty:$val)>;
+def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>;
+def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>;
+def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>;
+def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>;
+
+class AStorePatGlobalAddr<ValueType ty, PatFrag storekind, NI inst> :
+  Pat<(storekind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
+                 ty:$val),
+      (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>;
+def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+class AStorePatExternalSym<ValueType ty, PatFrag storekind, NI inst> :
+  Pat<(storekind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)),
+                 ty:$val),
+      (inst 0, texternalsym:$off, I32:$addr, ty:$val)>;
+def : AStorePatExternalSym<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatExternalSym<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+// Select stores with just a constant offset.
+class AStorePatOffsetOnly<ValueType ty, PatFrag storekind, NI inst> :
+  Pat<(storekind imm:$off, ty:$val),
+      (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
+def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag storekind, NI inst> :
+  Pat<(storekind (WebAssemblywrapper tglobaladdr:$off), ty:$val),
+      (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>;
+def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+class AStorePatExternSymOffOnly<ValueType ty, PatFrag storekind, NI inst> :
+  Pat<(storekind (WebAssemblywrapper texternalsym:$off), ty:$val),
+      (inst 0, texternalsym:$off, (CONST_I32 0), ty:$val)>;
+def : AStorePatExternSymOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>;
+def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>;
+
+} // Predicates = [HasAtomics]
+
+// Truncating stores.
+let Defs = [ARGUMENTS] in {
+defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>;
+defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>;
+defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>;
+defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>;
+defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>;
+} // Defs = [ARGUMENTS]
+
+// Fragments for truncating stores.
+
+// We don't have single truncating atomic store instructions. For 32-bit
+// instructions, we just need to match bare atomic stores. On the other hand,
+// truncating stores from i64 values are once truncated to i32 first.
+class trunc_astore_64<PatFrag storekind> :
+  PatFrag<(ops node:$addr, node:$val),
+          (storekind node:$addr, (i32 (trunc (i64 node:$val))))>;
+def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>;
+def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>;
+def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>;
+
+let Predicates = [HasAtomics] in {
+
+// Truncating stores with no constant offset
+def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+// Truncating stores with a constant offset
+def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>;
+def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>;
+def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>;
+def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>;
+def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>;
+def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>;
+def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>;
+def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>;
+def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>;
+def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>;
+
+def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+def : AStorePatExternalSym<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatExternalSym<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatExternalSym<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatExternalSym<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatExternalSym<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+// Truncating stores with just a constant offset
+def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+def : AStorePatExternSymOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>;
+def : AStorePatExternSymOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>;
+def : AStorePatExternSymOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
+def : AStorePatExternSymOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
+def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
+
+} // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Low-level exclusive operations
 //===----------------------------------------------------------------------===//
 
 // TODO: add exclusive operations here...
 
 // Load-exclusives.
 
 // Store-exclusives.
 
 // Store-release-exclusives.
 
 // And clear exclusive.
 

lib/Target/WebAssembly/WebAssemblyInstrMemory.td

 // types when loading, and truncate when storing.
 
 // WebAssembly constant offsets are performed as unsigned with infinite
 // precision, so we need to check for NoUnsignedWrap so that we don't fold an
 // offset for an add that needs wrapping.
 def regPlusImm : PatFrag<(ops node:$addr, node:$off),
                          (add node:$addr, node:$off),
                          [{ return N->getFlags().hasNoUnsignedWrap(); }]>;
-
 // Treat an 'or' node as an 'add' if the or'ed bits are known to be zero.
 def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
     return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
 
   KnownBits Known0;
   CurDAG->computeKnownBits(N->getOperand(0), Known0, 0);
   KnownBits Known1;

 def : LoadPatNoOffset<i64, sextloadi8, LOAD8_S_I64>;
 def : LoadPatNoOffset<i64, zextloadi8, LOAD8_U_I64>;
 def : LoadPatNoOffset<i64, sextloadi16, LOAD16_S_I64>;
 def : LoadPatNoOffset<i64, zextloadi16, LOAD16_U_I64>;
 def : LoadPatNoOffset<i64, sextloadi32, LOAD32_S_I64>;
 def : LoadPatNoOffset<i64, zextloadi32, LOAD32_U_I64>;
 
 // Select extending loads with a constant offset.
+
 def : LoadPatImmOff<i32, sextloadi8, regPlusImm, LOAD8_S_I32>;
 def : LoadPatImmOff<i32, zextloadi8, regPlusImm, LOAD8_U_I32>;
 def : LoadPatImmOff<i32, sextloadi16, regPlusImm, LOAD16_S_I32>;
 def : LoadPatImmOff<i32, zextloadi16, regPlusImm, LOAD16_U_I32>;
 def : LoadPatImmOff<i64, sextloadi8, regPlusImm, LOAD8_S_I64>;
 def : LoadPatImmOff<i64, zextloadi8, regPlusImm, LOAD8_U_I64>;
 def : LoadPatImmOff<i64, sextloadi16, regPlusImm, LOAD16_S_I64>;
 def : LoadPatImmOff<i64, zextloadi16, regPlusImm, LOAD16_U_I64>;

lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp

for (auto &MI : MBB) {
       case WebAssembly::STORE_I64:
       case WebAssembly::STORE_F32:
       case WebAssembly::STORE_F64:
       case WebAssembly::STORE8_I32:
       case WebAssembly::STORE16_I32:
       case WebAssembly::STORE8_I64:
       case WebAssembly::STORE16_I64:
       case WebAssembly::STORE32_I64:
+      case WebAssembly::ATOMIC_STORE_I32:
+      case WebAssembly::ATOMIC_STORE8_I32:
+      case WebAssembly::ATOMIC_STORE16_I32:
+      case WebAssembly::ATOMIC_STORE_I64:
+      case WebAssembly::ATOMIC_STORE8_I64:
+      case WebAssembly::ATOMIC_STORE16_I64:
+      case WebAssembly::ATOMIC_STORE32_I64:
         RewriteP2Align(MI, WebAssembly::StoreP2AlignOperandNo);
         break;
       default:
         break;
       }
     }
   }
 
   return Changed;
 }

test/CodeGen/WebAssembly/i32-load-store-alignment.ll

 ; RUN: llc < %s -mattr=+atomics -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
+; Loads.
+
 ; CHECK-LABEL: ldi32_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.load $push[[NUM:[0-9]+]]=, 0($0):p2align=0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_a1(i32 *%p) {
   %v = load i32, i32* %p, align 1
   ret i32 %v

 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i32{{$}}
 ; CHECK-NEXT: i32.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32_atomic_a8(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 8
   ret i32 %v
 }
+
+; CHECK-LABEL: sti32_atomic_a4:
+; CHECK-NEXT: .param i32, i32{{$}}
+; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @sti32_atomic_a4(i32 *%p, i32 %v) {
+ store atomic i32 %v, i32* %p seq_cst, align 4
+ ret void
+}
+
+; CHECK-LABEL: sti32_atomic_a8:
+; CHECK-NEXT: .param i32, i32{{$}}
+; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @sti32_atomic_a8(i32 *%p, i32 %v) {
+ store atomic i32 %v, i32* %p seq_cst, align 8
+ ret void
+}

test/CodeGen/WebAssembly/i64-load-store-alignment.ll

 ; RUN: llc < %s -mattr=+atomics -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
+; Loads.
+
 ; CHECK-LABEL: ldi64_a1:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.load $push[[NUM:[0-9]+]]=, 0($0):p2align=0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_a1(i64 *%p) {
   %v = load i64, i64* %p, align 1
   ret i64 %v

 ; CHECK-NEXT: return{{$}}
 define void @sti32_a8(i32 *%p, i64 %w) {
   %v = trunc i64 %w to i32
   store i32 %v, i32* %p, align 8
   ret void
 }
 
 ; Atomics.
+; Wasm atomics have the alignment field, but it must always have the type's
+; natural alignment.
+
 ; CHECK-LABEL: ldi64_atomic_a8:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_atomic_a8(i64 *%p) {
   %v = load atomic i64, i64* %p seq_cst, align 8
   ret i64 %v
 }
 
 ; 16 is greater than the default alignment so it is ignored.
-
 ; CHECK-LABEL: ldi64_atomic_a16:
 ; CHECK-NEXT: .param i32{{$}}
 ; CHECK-NEXT: .result i64{{$}}
 ; CHECK-NEXT: i64.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64_atomic_a16(i64 *%p) {
   %v = load atomic i64, i64* %p seq_cst, align 16
   ret i64 %v
 }
+
+; CHECK-LABEL: sti64_atomic_a4:
+; CHECK-NEXT: .param i32, i64{{$}}
+; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @sti64_atomic_a4(i64 *%p, i64 %v) {
+ store atomic i64 %v, i64* %p seq_cst, align 8
+ ret void
+}
+
+; 16 is greater than the default alignment so it is ignored.
+; CHECK-LABEL: sti64_atomic_a8:
+; CHECK-NEXT: .param i32, i64{{$}}
+; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @sti64_atomic_a8(i64 *%p, i64 %v) {
+ store atomic i64 %v, i64* %p seq_cst, align 16
+ ret void
+}

test/CodeGen/WebAssembly/offset-atomics.ll

 ; RUN: not llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals -mattr=+atomics,+sign-ext | FileCheck %s
 
 ; Test that atomic loads are assembled properly.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
+; Basic load.
+
 ; CHECK-LABEL: load_i32_no_offset:
 ; CHECK: i32.atomic.load $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @load_i32_no_offset(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 4
   ret i32 %v
 }
 
define i32 @load_i32_with_folded_or_offset(i32 %x) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
   %arrayidx = getelementptr inbounds i8, i8* %t0, i32 2
   %t1 = load atomic i8, i8* %arrayidx seq_cst, align 8
   %conv = sext i8 %t1 to i32
   ret i32 %conv
 }
 
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_no_offset:
+; CHECK-NEXT: .param i32, i32{{$}}
+; CHECK-NEXT: i32.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @store_i32_no_offset(i32 *%p, i32 %v) {
+  store atomic i32 %v, i32* %p seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_folded_offset:
+; CHECK: i32.atomic.store 24($0), $pop0{{$}}
+define void @store_i32_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_folded_gep_offset:
+; CHECK: i32.atomic.store 24($0), $pop0{{$}}
+define void @store_i32_with_folded_gep_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_unfolded_gep_negative_offset:
+; CHECK: i32.const        $push0=, -24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i32_with_unfolded_gep_negative_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_unfolded_offset:
+; CHECK: i32.const        $push0=, 24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i32_with_unfolded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_unfolded_gep_offset:
+; CHECK: i32.const        $push0=, 24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i32.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i32_with_unfolded_gep_offset(i32* %p) {
+  %s = getelementptr i32, i32* %p, i32 6
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_no_offset:
+; CHECK-NEXT: .param i32, i64{{$}}
+; CHECK-NEXT: i64.atomic.store 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @store_i64_no_offset(i64 *%p, i64 %v) {
+  store atomic i64 %v, i64* %p seq_cst, align 8
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_folded_offset:
+; CHECK: i64.atomic.store 24($0), $pop0{{$}}
+define void @store_i64_with_folded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  store atomic i64 0, i64* %s seq_cst, align 8
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_folded_gep_offset:
+; CHECK: i64.atomic.store 24($0), $pop0{{$}}
+define void @store_i64_with_folded_gep_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 3
+  store atomic i64 0, i64* %s seq_cst, align 8
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_unfolded_gep_negative_offset:
+; CHECK: i32.const        $push0=, -24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i64_with_unfolded_gep_negative_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -3
+  store atomic i64 0, i64* %s seq_cst, align 8
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_unfolded_offset:
+; CHECK: i32.const        $push0=, 24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i64_with_unfolded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  store atomic i64 0, i64* %s seq_cst, align 8
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_unfolded_gep_offset:
+; CHECK: i32.const        $push0=, 24{{$}}
+; CHECK: i32.add          $push1=, $0, $pop0{{$}}
+; CHECK: i64.atomic.store 0($pop1), $pop2{{$}}
+define void @store_i64_with_unfolded_gep_offset(i64* %p) {
+  %s = getelementptr i64, i64* %p, i32 3
+  store atomic i64 0, i64* %s seq_cst, align 8
+  ret void
+}
+
 ; When loading from a fixed address, materialize a zero.
 
 ; CHECK-LABEL: load_i32_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
 ; CHECK: i32.atomic.load  $push1=, 42($pop0){{$}}
 define i32 @load_i32_from_numeric_address() {
   %s = inttoptr i32 42 to i32*
   %t = load atomic i32, i32* %s seq_cst, align 4
   ret i32 %t
 }
 
 
 ; CHECK-LABEL: load_i32_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
 ; CHECK: i32.atomic.load  $push1=, gv($pop0){{$}}
 @gv = global i32 0
 define i32 @load_i32_from_global_address() {
   %t = load atomic i32, i32* @gv seq_cst, align 4
   ret i32 %t
 }
 
+; CHECK-LABEL: store_i32_to_numeric_address:
+; CHECK-NEXT: i32.const        $push0=, 0{{$}}
+; CHECK-NEXT: i32.const        $push1=, 0{{$}}
+; CHECK-NEXT: i32.atomic.store 42($pop0), $pop1{{$}}
+define void @store_i32_to_numeric_address() {
+  %s = inttoptr i32 42 to i32*
+  store atomic i32 0, i32* %s seq_cst, align 4
+  ret void
+}
+
+; CHECK-LABEL: store_i32_to_global_address:
+; CHECK: i32.const        $push0=, 0{{$}}
+; CHECK: i32.const        $push1=, 0{{$}}
+; CHECK: i32.atomic.store gv($pop0), $pop1{{$}}
+define void @store_i32_to_global_address() {
+  store atomic i32 0, i32* @gv seq_cst, align 4
+  ret void
+}
+
 ; Fold an offset into a sign-extending load.
 
 ; CHECK-LABEL: load_i8_s_with_folded_offset:
 ; CHECK: i32.atomic.load8_u $push0=, 24($0){{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
 define i32 @load_i8_s_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24

 ; i8 return value should test anyext loads
 ; CHECK-LABEL: ldi8_a1:
 ; CHECK: i32.atomic.load8_u $push[[NUM:[0-9]+]]=, 0($0){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i8 @ldi8_a1(i8 *%p) {
   %v = load atomic i8, i8* %p seq_cst, align 1
   ret i8 %v
 }
+
+; Fold an offset into a truncating store.
+
+; CHECK-LABEL: store_i8_with_folded_offset:
+; CHECK: i32.atomic.store8 24($0), $pop0{{$}}
+define void @store_i8_with_folded_offset(i8* %p) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  store atomic i8 0, i8* %s seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: store_i16_with_folded_offset:
+; CHECK: i32.atomic.store16 24($0), $pop0{{$}}
+define void @store_i16_with_folded_offset(i16* %p) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i16*
+  store atomic i16 0, i16* %s seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i8_i64_with_folded_offset:
+; CHECK: i64.atomic.store8 24($0), $1{{$}}
+define void @store_i8_i64_with_folded_offset(i8* %p, i64 %v) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i8*
+  %t = trunc i64 %v to i8
+  store atomic i8 %t, i8* %s seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: store_i16_i64_with_folded_offset:
+; CHECK: i64.atomic.store16 24($0), $1{{$}}
+define void @store_i16_i64_with_folded_offset(i16* %p, i64 %v) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i16*
+  %t = trunc i64 %v to i16
+  store atomic i16 %t, i16* %s seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i32_i64_with_folded_offset:
+; CHECK: i64.atomic.store32 24($0), $1{{$}}
+define void @store_i32_i64_with_folded_offset(i32* %p, i64 %v) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = trunc i64 %v to i32
+  store atomic i32 %t, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Fold a gep offset into a truncating store.
+
+; CHECK-LABEL: store_i8_with_folded_gep_offset:
+; CHECK: i32.atomic.store8 24($0), $pop0{{$}}
+define void @store_i8_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 24
+  store atomic i8 0, i8* %s seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: store_i16_with_folded_gep_offset:
+; CHECK: i32.atomic.store16 48($0), $pop0{{$}}
+define void @store_i16_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  store atomic i16 0, i16* %s seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i8_i64_with_folded_gep_offset:
+; CHECK: i64.atomic.store8 24($0), $1{{$}}
+define void @store_i8_i64_with_folded_gep_offset(i8* %p, i64 %v) {
+  %s = getelementptr inbounds i8, i8* %p, i32 24
+  %t = trunc i64 %v to i8
+  store atomic i8 %t, i8* %s seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i16_i64_with_folded_gep_offset:
+; CHECK: i64.atomic.store16 48($0), $1{{$}}
+define void @store_i16_i64_with_folded_gep_offset(i16* %p, i64 %v) {
+  %s = getelementptr inbounds i16, i16* %p, i32 24
+  %t = trunc i64 %v to i16
+  store atomic i16 %t, i16* %s seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i32_i64_with_folded_gep_offset:
+; CHECK: i64.atomic.store32 96($0), $1{{$}}
+define void @store_i32_i64_with_folded_gep_offset(i32* %p, i64 %v) {
+  %s = getelementptr inbounds i32, i32* %p, i32 24
+  %t = trunc i64 %v to i32
+  store atomic i32 %t, i32* %s seq_cst, align 4
+  ret void
+}
+
+; Fold an or_is_add pattern based offset into a truncating store.
+
+; CHECK-LABEL: store_i8_with_folded_or_offset:
+; CHECK: i32.atomic.store8 2($pop{{[0-9]+}}), $pop{{[0-9]+}}{{$}}
+define void @store_i8_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
+  store atomic i8 0, i8* %arrayidx seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: store_i16_with_folded_or_offset:
+; CHECK: i32.atomic.store16 4($pop{{[0-9]+}}), $pop{{[0-9]+}}{{$}}
+define void @store_i16_with_folded_or_offset(i32 %x) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i16*
+  %arrayidx = getelementptr inbounds i16, i16* %p, i32 2
+  store atomic i16 0, i16* %arrayidx seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i8_i64_with_folded_or_offset:
+; CHECK: i64.atomic.store8 2($pop{{[0-9]+}}), $1{{$}}
+define void @store_i8_i64_with_folded_or_offset(i32 %x, i64 %v) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i8*
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 2
+  %t = trunc i64 %v to i8
+  store atomic i8 %t, i8* %arrayidx seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: store_i16_i64_with_folded_or_offset:
+; CHECK: i64.atomic.store16 4($pop{{[0-9]+}}), $1{{$}}
+define void @store_i16_i64_with_folded_or_offset(i32 %x, i64 %v) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i16*
+  %arrayidx = getelementptr inbounds i16, i16* %p, i32 2
+  %t = trunc i64 %v to i16
+  store atomic i16 %t, i16* %arrayidx seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: store_i32_i64_with_folded_or_offset:
+; CHECK: i64.atomic.store32 8($pop{{[0-9]+}}), $1{{$}}
+define void @store_i32_i64_with_folded_or_offset(i32 %x, i64 %v) {
+  %and = and i32 %x, -4
+  %p = inttoptr i32 %and to i32*
+  %arrayidx = getelementptr inbounds i32, i32* %p, i32 2
+  %t = trunc i64 %v to i32
+  store atomic i32 %t, i32* %arrayidx seq_cst, align 4
+  ret void
+}

test/CodeGen/WebAssembly/store-trunc-atomic.ll

+; RUN: llc < %s -mattr=+atomics,+sign-ext -asm-verbose=false -disable-wasm-explicit-locals | FileCheck %s
+
+; Test that truncating stores are assembled properly.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: trunc_i8_i32:
+; CHECK: i32.atomic.store8 0($0), $1{{$}}
+define void @trunc_i8_i32(i8 *%p, i32 %v) {
+  %t = trunc i32 %v to i8
+  store atomic i8 %t, i8* %p seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: trunc_i16_i32:
+; CHECK: i32.atomic.store16 0($0), $1{{$}}
+define void @trunc_i16_i32(i16 *%p, i32 %v) {
+  %t = trunc i32 %v to i16
+  store atomic i16 %t, i16* %p seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: trunc_i8_i64:
+; CHECK: i64.atomic.store8 0($0), $1{{$}}
+define void @trunc_i8_i64(i8 *%p, i64 %v) {
+  %t = trunc i64 %v to i8
+  store atomic i8 %t, i8* %p seq_cst, align 1
+  ret void
+}
+
+; CHECK-LABEL: trunc_i16_i64:
+; CHECK: i64.atomic.store16 0($0), $1{{$}}
+define void @trunc_i16_i64(i16 *%p, i64 %v) {
+  %t = trunc i64 %v to i16
+  store atomic i16 %t, i16* %p seq_cst, align 2
+  ret void
+}
+
+; CHECK-LABEL: trunc_i32_i64:
+; CHECK: i64.atomic.store32 0($0), $1{{$}}
+define void @trunc_i32_i64(i32 *%p, i64 %v) {
+  %t = trunc i64 %v to i32
+  store atomic i32 %t, i32* %p seq_cst, align 4
+  ret void
+}