[DAG] Elide stores which are overwritten without being observed.

[niravd]

Summary

In SelectionDAG, when a store is immediately chained to another store
to the same address, elide the first store as it has no observable
effects. This is causes small improvements dealing with intrinsics
lowered to stores.

Test notes:

• Many testcases overwrite store addresses multiple times and needed minor changes, mainly making stores volatile to prevent the optimization from optimizing the test away.

• Many X86 test cases optimized out instructions associated with associated with va_start.

• Note that test_splat in CodeGen/AArch64/misched-stp.ll no longer has dependencies to check and can probably be removed and potentially replaced with another test.

[rnk]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

13132 ↗

(On Diff #99027)

What do you think of moving this logic in here so that we don't need two ST1 dyn_casts? It'd look like:

if (ST1->getBasePtr() == Ptr && ST->getMemoryVT() == ST1->getMemoryVT() &&
    ST->isUnindexed() && !ST->isVolatile() &&
    ST1->isUnindexed() && !ST1->isVolatile()) {
  if (ST1->getValue() == Value) { ... }
  else if (OptLevel != CodeGenOpt::None && ST1->hasOneUse() && !ST1->getPasePtr().isUndef()) { ... }
}

[niravd]

Rebasing past visitTokenFactor cleanup and do Reid's suggested cleanup

[rnk]

lgtm, thanks!

llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() &&
         // There can't be any side effects between the load and store, such as
         // a call or store.
         Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) {
       // The store is dead, remove it.
       return Chain;
     }
   }
 
+  if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
+    if (ST->isUnindexed() && !ST->isVolatile() && ST1->isUnindexed() &&
+        !ST1->isVolatile() && ST1->getBasePtr() == Ptr &&
+        ST->getMemoryVT() == ST1->getMemoryVT()) {
       // If this is a store followed by a store with the same value to the same
       // location, then the store is dead/noop.
-  if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
-    if (ST1->getBasePtr() == Ptr && ST->getMemoryVT() == ST1->getMemoryVT() &&
-        ST1->getValue() == Value && ST->isUnindexed() && !ST->isVolatile() &&
-        ST1->isUnindexed() && !ST1->isVolatile()) {
+      if (ST1->getValue() == Value) {
         // The store is dead, remove it.
         return Chain;
       }
+
+      // If this is a store who's preceeding store to the same location
+      // and no one other node is chained to that store we can effectively
+      // drop the store. Do not remove stores to undef as they may be used as
+      // data sinks.
+      if (OptLevel != CodeGenOpt::None && ST1->hasOneUse() &&
+          !ST1->getBasePtr().isUndef()) {
+        // ST1 is fully overwritten and can be elided. Combine with it's chain
+        // value.
+        CombineTo(ST1, ST1->getChain());
+        return SDValue();
+      }
+    }
   }
 
   // If this is an FP_ROUND or TRUNC followed by a store, fold this into a
   // truncating store.  We can do this even if this is already a truncstore.
   if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE)
       && Value.getNode()->hasOneUse() && ST->isUnindexed() &&
       TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
                             ST->getMemoryVT())) {

llvm/trunk/test/CodeGen/AArch64/ldst-zero.ll

 ; RUN: llc -mtriple=aarch64 -mcpu=cortex-a53 < %s | FileCheck %s
 
 ; Tests to check that zero stores which are generated as STP xzr, xzr aren't
 ; scheduled incorrectly due to incorrect alias information
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
 %struct.tree_common = type { i8*, i8*, i32 }
 
 ; Original test case which exhibited the bug
 define void @test1(%struct.tree_common* %t, i32 %code, i8* %type) {
 ; CHECK-LABEL: test1:
-; CHECK: stp xzr, xzr, [x0, #8]
-; CHECK: stp xzr, x2, [x0]
-; CHECK: str w1, [x0, #16]
+; CHECK-DAG: stp x2, xzr, [x0, #8]
+; CHECK-DAG: str w1, [x0, #16]
+; CHECK-DAG: str xzr, [x0]
 entry:
   %0 = bitcast %struct.tree_common* %t to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 24, i32 8, i1 false)
   %code1 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 2
   store i32 %code, i32* %code1, align 8
   %type2 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 1
   store i8* %type, i8** %type2, align 8
   ret void
 }
 
 ; Store to each struct element instead of using memset
 define void @test2(%struct.tree_common* %t, i32 %code, i8* %type) {
 ; CHECK-LABEL: test2:
-; CHECK: stp xzr, xzr, [x0]
-; CHECK: str wzr, [x0, #16]
-; CHECK: str w1, [x0, #16]
-; CHECK: str x2, [x0, #8]
+; CHECK-DAG: str w1, [x0, #16]
+; CHECK-DAG: stp xzr, x2, [x0]
 entry:
   %0 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 0
   %1 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 1
   %2 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 2
   store i8* zeroinitializer, i8** %0, align 8
   store i8* zeroinitializer, i8** %1, align 8
   store i32 zeroinitializer, i32* %2, align 8
   store i32 %code, i32* %2, align 8
   store i8* %type, i8** %1, align 8
   ret void
 }
 
 ; Vector store instead of memset
 define void @test3(%struct.tree_common* %t, i32 %code, i8* %type) {
 ; CHECK-LABEL: test3:
-; CHECK: stp xzr, xzr, [x0, #8]
-; CHECK: stp xzr, x2, [x0]
-; CHECK: str w1, [x0, #16]
+; CHECK-DAG: stp x2, xzr, [x0, #8]
+; CHECK-DAG: str w1, [x0, #16]
+; CHECK-DAG: str xzr, [x0]
 entry:
   %0 = bitcast %struct.tree_common* %t to <3 x i64>*
   store <3 x i64> zeroinitializer, <3 x i64>* %0, align 8
   %code1 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 2
   store i32 %code, i32* %code1, align 8
   %type2 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 1
   store i8* %type, i8** %type2, align 8
   ret void
 }
 
 ; Vector store, then store to vector elements
 define void @test4(<3 x i64>* %p, i64 %x, i64 %y) {
 ; CHECK-LABEL: test4:
-; CHECK: stp xzr, xzr, [x0, #8]
-; CHECK: stp xzr, x2, [x0]
-; CHECK: str x1, [x0, #16]
+; CHECK-DAG: stp x2, x1, [x0, #8]
+; CHECK-DAG: str xzr, [x0]
 entry:
   store <3 x i64> zeroinitializer, <3 x i64>* %p, align 8
   %0 = bitcast <3 x i64>* %p to i64*
   %1 = getelementptr inbounds i64, i64* %0, i64 2
   store i64 %x, i64* %1, align 8
   %2 = getelementptr inbounds i64, i64* %0, i64 1
   store i64 %y, i64* %2, align 8
   ret void
 }

llvm/trunk/test/CodeGen/AArch64/misched-stp.ll

 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=aarch64 -mcpu=cyclone -mattr=+use-aa -enable-misched -verify-misched -debug-only=misched -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64 -mcpu=cyclone -mattr=+use-aa -enable-misched -verify-misched -o - | FileCheck %s
 
 ; Tests to check that the scheduler dependencies derived from alias analysis are
 ; correct when we have loads that have been split up so that they can later be
 ; merged into STP.
 
-; CHECK: ********** MI Scheduling **********
-; CHECK: test_splat:BB#0 entry
-; CHECK: SU({{[0-9]+}}):   STRWui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 3; mem:ST4[%3+8]
-; CHECK: Successors:
-; CHECK-NEXT: ord  [[SU1:SU\([0-9]+\)]]
-; CHECK: SU({{[0-9]+}}):   STRWui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 2; mem:ST4[%3+4]
-; CHECK: Successors:
-; CHECK-NEXT: ord  [[SU2:SU\([0-9]+\)]]
-; CHECK: [[SU1]]:   STRWui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 3; mem:ST4[%2]
-; CHECK: [[SU2]]:   STRWui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 2; mem:ST4[%1]
+; Now that overwritten stores are elided in SelectionDAG, dependencies
+; are resolved and removed before MISCHED. Check that we have
+; equivalent pair of stp calls as a baseline.
+
+; CHECK-LABEL: test_splat
+; CHECK:     ldr [[REG:w[0-9]+]], [x2]
+; CHECK-DAG: stp w0, [[REG]], [x2, #12]
+; CHECK-DAG: stp [[REG]], w1, [x2, #4]
 define void @test_splat(i32 %x, i32 %y, i32* %p) {
 entry:
   %val = load i32, i32* %p, align 4
   %0 = getelementptr inbounds i32, i32* %p, i64 1
   %1 = getelementptr inbounds i32, i32* %p, i64 2
   %2 = getelementptr inbounds i32, i32* %p, i64 3
   %vec0 = insertelement <4 x i32> undef, i32 %val, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %val, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %val, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %val, i32 3
   %3 = bitcast i32* %0 to <4 x i32>*
   store <4 x i32> %vec3, <4 x i32>* %3, align 4
   store i32 %x, i32* %2, align 4
   store i32 %y, i32* %1, align 4
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
 %struct.tree_common = type { i8*, i8*, i32 }
 
-; CHECK: ********** MI Scheduling **********
-; CHECK: test_zero:BB#0 entry
-; CHECK: SU({{[0-9]+}}):   STRXui %XZR, %vreg{{[0-9]+}}, 2; mem:ST8[%0+16]
-; CHECK: Successors:
-; CHECK-NEXT: ord  [[SU3:SU\([0-9]+\)]]
-; CHECK: SU({{[0-9]+}}):   STRXui %XZR, %vreg{{[0-9]+}}, 1; mem:ST8[%0+8]
-; CHECK: Successors:
-; CHECK-NEXT: ord  [[SU4:SU\([0-9]+\)]]
-; CHECK: [[SU3]]:   STRWui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 4; mem:ST4[%code1]
-; CHECK: [[SU4]]:   STRXui %vreg{{[0-9]+}}, %vreg{{[0-9]+}}, 1; mem:ST8[%type2]
+; CHECK-LABEL: test_zero
+; CHECK-DAG: stp x2, xzr, [x0, #8]
+; CHECK-DAG: str w1, [x0, #16]
+; CHECK-DAG: str xzr, [x0]
+
 define void @test_zero(%struct.tree_common* %t, i32 %code, i8* %type) {
 entry:
   %0 = bitcast %struct.tree_common* %t to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 24, i32 8, i1 false)
   %code1 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 2
   store i32 %code, i32* %code1, align 8
   %type2 = getelementptr inbounds %struct.tree_common, %struct.tree_common* %t, i64 0, i32 1
   store i8* %type, i8** %type2, align 8
   ret void
 }

llvm/trunk/test/CodeGen/AMDGPU/global-constant.ll

 
 ; HSA OSes use relocations.
 ; HSA: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2@rel32@lo+4
 ; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], private2@rel32@hi+4
 
 define amdgpu_kernel void @private_test(i32 %index, float addrspace(1)* %out) {
   %ptr = getelementptr [4 x float], [4 x float] addrspace(2) * @private1, i32 0, i32 %index
   %val = load float, float addrspace(2)* %ptr
-  store float %val, float addrspace(1)* %out
+  store volatile float %val, float addrspace(1)* %out
   %ptr2 = getelementptr [4 x float], [4 x float] addrspace(2) * @private2, i32 0, i32 %index
   %val2 = load float, float addrspace(2)* %ptr2
-  store float %val2, float addrspace(1)* %out
+  store volatile float %val2, float addrspace(1)* %out
   ret void
 }
 
 ; HSA-LABEL: {{^}}available_externally_test:
 ; HSA: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}}
 ; HSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], available_externally@gotpcrel32@lo+4
 ; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], available_externally@gotpcrel32@hi+4
 define amdgpu_kernel void @available_externally_test(i32 addrspace(1)* %out) {

llvm/trunk/test/CodeGen/ARM/2011-02-04-AntidepMultidef.ll

bb:                                               ; preds = %entry
   br label %bb3
 
 bb3:                                              ; preds = %bb, %entry
   %0 = call noalias i8* @malloc() nounwind
   br i1 %tst, label %bb46, label %bb8
 
 bb8:                                              ; preds = %bb3
   %1 = getelementptr inbounds i8, i8* %0, i32 0
-  store i8 0, i8* %1, align 1
+  store volatile i8 0, i8* %1, align 1
   %2 = call i32 @ptou() nounwind
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %3 = udiv i32 %2, 10
   %4 = urem i32 %3, 10
   %5 = icmp ult i32 %4, 10
   %6 = trunc i32 %4 to i8
   %7 = or i8 %6, 48
   %8 = add i8 %6, 87
   %iftmp.5.0.1 = select i1 %5, i8 %7, i8 %8
-  store i8 %iftmp.5.0.1, i8* %p8, align 1
+  store volatile i8 %iftmp.5.0.1, i8* %p8, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %9 = udiv i32 %2, 100
   %10 = urem i32 %9, 10
   %11 = icmp ult i32 %10, 10
   %12 = trunc i32 %10 to i8
   %13 = or i8 %12, 48
   %14 = add i8 %12, 87
   %iftmp.5.0.2 = select i1 %11, i8 %13, i8 %14
-  store i8 %iftmp.5.0.2, i8* %p8, align 1
+  store volatile i8 %iftmp.5.0.2, i8* %p8, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %15 = udiv i32 %2, 10000
   %16 = urem i32 %15, 10
   %17 = icmp ult i32 %16, 10
   %18 = trunc i32 %16 to i8
   %19 = or i8 %18, 48
   %20 = add i8 %18, 87
   %iftmp.5.0.4 = select i1 %17, i8 %19, i8 %20
-  store i8 %iftmp.5.0.4, i8* null, align 1
+  store volatile i8 %iftmp.5.0.4, i8* null, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %21 = udiv i32 %2, 100000
   %22 = urem i32 %21, 10
   %23 = icmp ult i32 %22, 10
   %iftmp.5.0.5 = select i1 %23, i8 0, i8 %val8
-  store i8 %iftmp.5.0.5, i8* %p8, align 1
+  store volatile i8 %iftmp.5.0.5, i8* %p8, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %24 = udiv i32 %2, 1000000
   %25 = urem i32 %24, 10
   %26 = icmp ult i32 %25, 10
   %27 = trunc i32 %25 to i8
   %28 = or i8 %27, 48
   %29 = add i8 %27, 87
   %iftmp.5.0.6 = select i1 %26, i8 %28, i8 %29
-  store i8 %iftmp.5.0.6, i8* %p8, align 1
+  store volatile i8 %iftmp.5.0.6, i8* %p8, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %30 = udiv i32 %2, 10000000
   %31 = urem i32 %30, 10
   %32 = icmp ult i32 %31, 10
   %33 = trunc i32 %31 to i8
   %34 = or i8 %33, 48
   %35 = add i8 %33, 87
   %iftmp.5.0.7 = select i1 %32, i8 %34, i8 %35
-  store i8 %iftmp.5.0.7, i8* %p8, align 1
+  store volatile i8 %iftmp.5.0.7, i8* %p8, align 1
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   ; CHECK: umull [[REGISTER:lr|r[0-9]+]],
   ; CHECK-NOT: [[REGISTER]],
   ; CHECK: {{lr|r[0-9]+}}, {{lr|r[0-9]+$}}
   %36 = udiv i32 %2, 100000000
   %37 = urem i32 %36, 10
   %38 = icmp ult i32 %37, 10
   %39 = trunc i32 %37 to i8
   %40 = or i8 %39, 48
   %41 = add i8 %39, 87
   %iftmp.5.0.8 = select i1 %38, i8 %40, i8 %41
-  store i8 %iftmp.5.0.8, i8* null, align 1
+  store volatile i8 %iftmp.5.0.8, i8* null, align 1
   br label %bb46
 
 bb46:                                             ; preds = %bb3
   ret void
 }
 
 declare noalias i8* @malloc() nounwind
 
 declare i32 @ptou()

llvm/trunk/test/CodeGen/ARM/2012-10-04-AAPCS-byval-align8.ll

 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi | FileCheck %s
 ; Test that we correctly use registers and align elements when using va_arg
 
 %struct_t = type { double, double, double }
 @static_val = constant %struct_t { double 1.0, double 2.0, double 3.0 }
 
 declare void @llvm.va_start(i8*) nounwind
 declare void @llvm.va_end(i8*) nounwind
 
 ; CHECK-LABEL: test_byval_8_bytes_alignment:
 define void @test_byval_8_bytes_alignment(i32 %i, ...) {
 entry:
 ; CHECK: sub       sp, sp, #12
 ; CHECK: sub       sp, sp, #4
 ; CHECK: add       r0, sp, #4
-; CHECK: stm       sp, {r0, r1, r2, r3}
+; CHECK: stmib     sp, {r1, r2, r3}
   %g = alloca i8*
   %g1 = bitcast i8** %g to i8*
   call void @llvm.va_start(i8* %g1)
 
 ; CHECK: add  [[REG:(r[0-9]+)|(lr)]], {{(r[0-9]+)|(lr)}}, #7
 ; CHECK: bic  [[REG]], [[REG]], #7
   %0 = va_arg i8** %g, double
   call void @llvm.va_end(i8* %g1)

llvm/trunk/test/CodeGen/ARM/dag-combine-ldst.ll

 ; RUN: llc < %s -mtriple=arm-eabi -mattr=+v4t -O0 | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK_O0
 ; RUN: llc < %s -mtriple=arm-eabi -mattr=+v4t -O1 | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK_O1
 
 ; In /O0, the addition must not be eliminated. This happens when the load
 ; and store are folded by the DAGCombiner. In /O1 and above, the optimization
 ; must be executed.
 
 ; CHECK-LABEL:   {{^}}main
 ; CHECK:         mov [[TMP:r[0-9]+]], #0
 ; CHECK-NEXT:    str [[TMP]], [sp, #4]
-; CHECK-NEXT:    str [[TMP]], [sp]
+; CHECK_O0:      str [[TMP]], [sp]
 ; CHECK_O0:      ldr [[TMP:r[0-9]+]], [sp]
 ; CHECK_O0-NEXT: add [[TMP]], [[TMP]], #2
 ; CHECK_O1-NOT:  ldr [[TMP:r[0-9]+]], [sp]
 ; CHECK_O1-NOT:  add [[TMP]], [[TMP]], #2
 
 define i32 @main() {
 bb:
   %tmp = alloca i32, align 4

llvm/trunk/test/CodeGen/MSP430/vararg.ll

; CHECK-NEXT: mov.w [[REG]], 0(r1)
   call void @llvm.va_end(i8* %vl1)
   ret void
 }
 
 define i16 @va_arg(i8* %vl) nounwind {
 entry:
 ; CHECK-LABEL: va_arg:
   %vl.addr = alloca i8*, align 2
-; CHECK: mov.w r12, 0(r1)
   store i8* %vl, i8** %vl.addr, align 2
 ; CHECK: mov.w r12, [[REG:r[0-9]+]]
 ; CHECK-NEXT: add.w #2, [[REG]]
 ; CHECK-NEXT: mov.w [[REG]], 0(r1)
   %0 = va_arg i8** %vl.addr, i16
 ; CHECK-NEXT: mov.w 0(r12), r12
   ret i16 %0
 }

llvm/trunk/test/CodeGen/Mips/msa/bmzi_bmnzi.ll

 ; RUN: llc -march=mipsel -mattr=+msa,+fp64 -relocation-model=pic < %s | FileCheck %s
 
 @llvm_mips_bmnzi_b_ARG1 = global <16 x i8> <i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>, align 16
 @llvm_mips_bmnzi_b_ARG2 = global <16 x i8> zeroinitializer, align 16
 @llvm_mips_bmnzi_b_RES = global <16 x i8> zeroinitializer, align 16
 
 define void @llvm_mips_bmnzi_b_test() nounwind {
 entry:
   %0 = load <16 x i8>, <16 x i8>* @llvm_mips_bmnzi_b_ARG1
   %1 = load <16 x i8>, <16 x i8>* @llvm_mips_bmnzi_b_ARG2
   %2 = tail call <16 x i8> @llvm.mips.bmnzi.b(<16 x i8> %0, <16 x i8> %1, i32 240)
-  store <16 x i8> %2, <16 x i8>* @llvm_mips_bmnzi_b_RES
+  store volatile <16 x i8> %2, <16 x i8>* @llvm_mips_bmnzi_b_RES
   %3 = tail call <16 x i8> @llvm.mips.bmnzi.b(<16 x i8> %0, <16 x i8> %1, i32 15)
-  store <16 x i8> %3, <16 x i8>* @llvm_mips_bmnzi_b_RES
+  store volatile <16 x i8> %3, <16 x i8>* @llvm_mips_bmnzi_b_RES
   %4 = tail call <16 x i8> @llvm.mips.bmnzi.b(<16 x i8> %0, <16 x i8> %1, i32 170)
   store <16 x i8> %4, <16 x i8>* @llvm_mips_bmnzi_b_RES
   ret void
 }
 ; CHECK-LABEL: llvm_mips_bmnzi_b_test:
 ; CHECK: lw [[R0:\$[0-9]+]], %got(llvm_mips_bmnzi_b_RES)(
 ; CHECK: lw [[R1:\$[0-9]+]], %got(llvm_mips_bmnzi_b_ARG1)(
 ; CHECK: lw [[R2:\$[0-9]+]], %got(llvm_mips_bmnzi_b_ARG2)(
 ; CHECK: ld.b [[R3:\$w[0-9]+]], 0([[R2]])
 ; CHECK: ld.b [[R4:\$w[0-9]+]], 0([[R1]])
 ; CHECK: move.v [[R5:\$w[0-9]+]], [[R4]]
 ; CHECK: binsli.b [[R5]], [[R3]], 3
 ; CHECK: binsri.b [[R5]], [[R3]], 3
 ; CHECK: bmnzi.b [[R4]], [[R3]], 170
 
 define void @llvm_mips_bmzi_b_test() nounwind {
 entry:
   %0 = load <16 x i8>, <16 x i8>* @llvm_mips_bmnzi_b_ARG1
   %1 = load <16 x i8>, <16 x i8>* @llvm_mips_bmnzi_b_ARG2
   %2 = tail call <16 x i8> @llvm.mips.bmzi.b(<16 x i8> %0, <16 x i8> %1, i32 240)
-  store <16 x i8> %2, <16 x i8>* @llvm_mips_bmnzi_b_RES
+  store volatile <16 x i8> %2, <16 x i8>* @llvm_mips_bmnzi_b_RES
   %3 = tail call <16 x i8> @llvm.mips.bmzi.b(<16 x i8> %0, <16 x i8> %1, i32 15)
-  store <16 x i8> %3, <16 x i8>* @llvm_mips_bmnzi_b_RES
+  store volatile <16 x i8> %3, <16 x i8>* @llvm_mips_bmnzi_b_RES
   %4 = tail call <16 x i8> @llvm.mips.bmzi.b(<16 x i8> %0, <16 x i8> %1, i32 170)
   store <16 x i8> %4, <16 x i8>* @llvm_mips_bmnzi_b_RES
   ret void
 }
 ; CHECK-LABEL: llvm_mips_bmzi_b_test:
 ; CHECK: lw [[R0:\$[0-9]+]], %got(llvm_mips_bmnzi_b_RES)(
 ; CHECK: lw [[R1:\$[0-9]+]], %got(llvm_mips_bmnzi_b_ARG2)(
 ; CHECK: lw [[R2:\$[0-9]+]], %got(llvm_mips_bmnzi_b_ARG1)(

llvm/trunk/test/CodeGen/PowerPC/ppcf128sf.ll

 ; RUN: llc -verify-machineinstrs  -mtriple=powerpc-unknown-linux-gnu -O0 < %s | FileCheck %s
 
 @ld = common global ppc_fp128 0xM00000000000000000000000000000000, align 16
 @ld2 = common global ppc_fp128 0xM00000000000000000000000000000000, align 16
 @d = common global double 0.000000e+00, align 8
 @f = common global float 0.000000e+00, align 4
 @i = common global i32 0, align 4
 @ui = common global i32 0, align 4
 @var = common global i8 0, align 1
 
 define void @foo() #0 {
 entry:
   %c = alloca ppc_fp128, align 16
   %0 = load ppc_fp128, ppc_fp128* @ld, align 16
   %1 = load ppc_fp128, ppc_fp128* @ld2, align 16
   %add = fadd ppc_fp128 %0, %1
-  store ppc_fp128 %add, ppc_fp128* %c, align 16
+  store volatile ppc_fp128 %add, ppc_fp128* %c, align 16
   %2 = load ppc_fp128, ppc_fp128* @ld, align 16
   %3 = load ppc_fp128, ppc_fp128* @ld2, align 16
   %sub = fsub ppc_fp128 %2, %3
-  store ppc_fp128 %sub, ppc_fp128* %c, align 16
+  store volatile ppc_fp128 %sub, ppc_fp128* %c, align 16
   %4 = load ppc_fp128, ppc_fp128* @ld, align 16
   %5 = load ppc_fp128, ppc_fp128* @ld2, align 16
   %mul = fmul ppc_fp128 %4, %5
-  store ppc_fp128 %mul, ppc_fp128* %c, align 16
+  store volatile ppc_fp128 %mul, ppc_fp128* %c, align 16
   %6 = load ppc_fp128, ppc_fp128* @ld, align 16
   %7 = load ppc_fp128, ppc_fp128* @ld2, align 16
   %div = fdiv ppc_fp128 %6, %7
-  store ppc_fp128 %div, ppc_fp128* %c, align 16
+  store volatile ppc_fp128 %div, ppc_fp128* %c, align 16
   ret void
 
   ; CHECK-LABEL:    __gcc_qadd
   ; CHECK-LABEL:    __gcc_qsub
   ; CHECK-LABEL:    __gcc_qmul
   ; CHECK-LABEL:    __gcc_qdiv
 }
 

llvm/trunk/test/CodeGen/SPARC/32abi.ll

 define void @intarg(i8  %a0,   ; %i0
                     i8  %a1,   ; %i1
                     i16 %a2,   ; %i2
                     i32 %a3,   ; %i3
                     i8* %a4,   ; %i4
                     i32 %a5,   ; %i5
                     i32 signext %a6,   ; [%fp+92]
                     i8* %a7) { ; [%fp+96]
-  store i8 %a0, i8* %a4
-  store i8 %a1, i8* %a4
+  store volatile i8 %a0, i8* %a4
+  store volatile i8 %a1, i8* %a4
   %p16 = bitcast i8* %a4 to i16*
-  store i16 %a2, i16* %p16
+  store volatile i16 %a2, i16* %p16
   %p32 = bitcast i8* %a4 to i32*
-  store i32 %a3, i32* %p32
+  store volatile i32 %a3, i32* %p32
   %pp = bitcast i8* %a4 to i8**
-  store i8* %a4, i8** %pp
-  store i32 %a5, i32* %p32
-  store i32 %a6, i32* %p32
-  store i8* %a7, i8** %pp
+  store volatile i8* %a4, i8** %pp
+  store volatile i32 %a5, i32* %p32
+  store volatile i32 %a6, i32* %p32
+  store volatile i8* %a7, i8** %pp
   ret void
 }
 
 ; CHECK-LABEL: call_intarg:
 ; CHECK: save %sp, -104, %sp
 ; Use %o0-%o5 for outgoing arguments
 ; CHECK: mov 5, %o5
 ; CHECK: st %i0, [%sp+92]

llvm/trunk/test/CodeGen/SPARC/64abi.ll

 define void @intarg(i8  %a0,   ; %i0
                     i8  %a1,   ; %i1
                     i16 %a2,   ; %i2
                     i32 %a3,   ; %i3
                     i8* %a4,   ; %i4
                     i32 %a5,   ; %i5
                     i32 signext %a6,   ; [%fp+BIAS+176]
                     i8* %a7) { ; [%fp+BIAS+184]
-  store i8 %a0, i8* %a4
-  store i8 %a1, i8* %a4
+  store volatile i8 %a0, i8* %a4
+  store volatile i8 %a1, i8* %a4
   %p16 = bitcast i8* %a4 to i16*
-  store i16 %a2, i16* %p16
+  store volatile i16 %a2, i16* %p16
   %p32 = bitcast i8* %a4 to i32*
-  store i32 %a3, i32* %p32
+  store volatile i32 %a3, i32* %p32
   %pp = bitcast i8* %a4 to i8**
-  store i8* %a4, i8** %pp
-  store i32 %a5, i32* %p32
-  store i32 %a6, i32* %p32
-  store i8* %a7, i8** %pp
+  store volatile i8* %a4, i8** %pp
+  store volatile i32 %a5, i32* %p32
+  store volatile i32 %a6, i32* %p32
+  store volatile i8* %a7, i8** %pp
   ret void
 }
 
 ; CHECK-LABEL: call_intarg:
 ; 16 saved + 8 args.
 ; CHECK: save %sp, -192, %sp
 ; Sign-extend and store the full 64 bits.
 ; CHECK: sra %i0, 0, [[R:%[gilo][0-7]]]

 ; CHECK-LABEL: call_ret_i64_pair:
 ; CHECK: call ret_i64_pair
 ; CHECK: stx %o0, [%i0]
 ; CHECK: stx %o1, [%i0]
 define void @call_ret_i64_pair(i64* %i0) {
   %rv = call { i64, i64 } @ret_i64_pair(i32 undef, i32 undef,
                                         i64* undef, i64* undef)
   %e0 = extractvalue { i64, i64 } %rv, 0
-  store i64 %e0, i64* %i0
+  store volatile i64 %e0, i64* %i0
   %e1 = extractvalue { i64, i64 } %rv, 1
   store i64 %e1, i64* %i0
   ret void
 }
 
 ; This is not a C struct, the i32 member uses 8 bytes, but the float only 4.
 ; CHECK-LABEL: ret_i32_float_pair:
 ; CHECK: ld [%i2], %i0

llvm/trunk/test/CodeGen/SystemZ/swift-return.ll

 ; CHECK-O0: nilf %r5, 1
 define void @consume_i1_ret() {
   %call = call swiftcc { i1, i1, i1, i1 } @produce_i1_ret()
   %v3 = extractvalue { i1, i1, i1, i1 } %call, 0
   %v5 = extractvalue { i1, i1, i1, i1 } %call, 1
   %v6 = extractvalue { i1, i1, i1, i1 } %call, 2
   %v7 = extractvalue { i1, i1, i1, i1 } %call, 3
   %val = zext i1 %v3 to i32
-  store i32 %val, i32* @var
+  store volatile i32 %val, i32* @var
   %val2 = zext i1 %v5 to i32
-  store i32 %val2, i32* @var
+  store volatile i32 %val2, i32* @var
   %val3 = zext i1 %v6 to i32
-  store i32 %val3, i32* @var
+  store volatile i32 %val3, i32* @var
   %val4 = zext i1 %v7 to i32
   store i32 %val4, i32* @var
   ret void
 }
 
 declare swiftcc { i1, i1, i1, i1 } @produce_i1_ret()

llvm/trunk/test/CodeGen/Thumb/stack-access.ll

 ; RUN: llc -mtriple=thumb-eabi < %s -o - | FileCheck %s
 
 ; Check that stack addresses are generated using a single ADD
 define void @test1(i8** %p) {
   %x = alloca i8, align 1
   %y = alloca i8, align 1
   %z = alloca i8, align 1
 ; CHECK: add r1, sp, #8
 ; CHECK: str r1, [r0]
-  store i8* %x, i8** %p, align 4
+  store volatile i8* %x, i8** %p, align 4
 ; CHECK: add r1, sp, #4
 ; CHECK: str r1, [r0]
-  store i8* %y, i8** %p, align 4
+  store volatile i8* %y, i8** %p, align 4
 ; CHECK: mov r1, sp
 ; CHECK: str r1, [r0]
-  store i8* %z, i8** %p, align 4
+  store volatile i8* %z, i8** %p, align 4
   ret void
 }
 
 ; Stack offsets larger than 1020 still need two ADDs
 define void @test2([1024 x i8]** %p) {
   %arr1 = alloca [1024 x i8], align 1
   %arr2 = alloca [1024 x i8], align 1
 ; CHECK: add r1, sp, #1020
 ; CHECK: adds r1, #4
 ; CHECK: str r1, [r0]
-  store [1024 x i8]* %arr1, [1024 x i8]** %p, align 4
+  store volatile [1024 x i8]* %arr1, [1024 x i8]** %p, align 4
 ; CHECK: mov r1, sp
 ; CHECK: str r1, [r0]
-  store [1024 x i8]* %arr2, [1024 x i8]** %p, align 4
+  store volatile [1024 x i8]* %arr2, [1024 x i8]** %p, align 4
   ret void
 }
 
 ; If possible stack-based lrdb/ldrh are widened to use SP-based addressing
 define i32 @test3() #0 {
   %x = alloca i8, align 1
   %y = alloca i8, align 1
 ; CHECK: ldr r0, [sp]

llvm/trunk/test/CodeGen/Thumb2/ldr-str-imm12.ll

bb119:                                            ; preds = %bb20, %bb20
   unreachable
 
 bb420:                                            ; preds = %bb20, %bb20
 ; CHECK: bb420
 ; CHECK: str{{(.w)?}} r{{[0-9]+}}, [sp
 ; CHECK: str{{(.w)?}} r{{[0-9]+}}, [sp
 ; CHECK: str{{(.w)?}} r{{[0-9]+}}, [sp
 ; CHECK: str{{(.w)?}} r{{[0-9]+}}, [sp
-  store %union.rec* null, %union.rec** @zz_hold, align 4
+  store volatile %union.rec* null, %union.rec** @zz_hold, align 4
   store %union.rec* null, %union.rec** @zz_res, align 4
-  store %union.rec* %x, %union.rec** @zz_hold, align 4
+  store volatile %union.rec* %x, %union.rec** @zz_hold, align 4
   %0 = call  %union.rec* @Manifest(%union.rec* undef, %union.rec* %env, %struct.STYLE* %style, %union.rec** %bthr, %union.rec** %fthr, %union.rec** %target, %union.rec** %crs, i32 %ok, i32 %need_expand, %union.rec** %enclose, i32 %fcr) nounwind ; <%union.rec*> [#uses=0]
   unreachable
 
 bb438:                                            ; preds = %bb20, %bb20
   unreachable
 
 bb533:                                            ; preds = %bb20
   ret %union.rec* %x

llvm/trunk/test/CodeGen/X86/arg-copy-elide.ll

 }
 
 ; CHECK-LABEL: _avoid_inalloca:
 ; CHECK: leal {{[0-9]+}}(%esp), %[[reg:[^ ]*]]
 ; CHECK: pushl %[[reg]]
 ; CHECK: calll _addrof_i32
 ; CHECK: retl
 
-
 ; Don't elide the copy when the alloca is escaped with a store.
-
 define void @escape_with_store(i32 %x) {
   %x1 = alloca i32
   %x2 = alloca i32*
   store i32* %x1, i32** %x2
   %x3 = load i32*, i32** %x2
   store i32 0, i32* %x3
   store i32 %x, i32* %x1
   call void @addrof_i32(i32* %x1)
   ret void
 }
 
 ; CHECK-LABEL: _escape_with_store:
-; CHECK-DAG: movl {{.*}}(%esp), %[[reg:[^ ]*]]
-; CHECK-DAG: movl $0, [[offs:[0-9]*]](%esp)
-; CHECK: movl %[[reg]], [[offs]](%esp)
+; CHECK: movl {{.*}}(%esp), %[[reg:[^ ]*]]
+; CHECK: movl %[[reg]], [[offs:[0-9]*]](%esp)
 ; CHECK: calll _addrof_i32
 
 
 ; This test case exposed issues with the use of TokenFactor.
 
 define void @sret_and_elide(i32* sret %sret, i32 %v) {
   %v.p = alloca i32
   store i32 %v, i32* %v.p

llvm/trunk/test/CodeGen/X86/nontemporal.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X32-AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64-SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X64-AVX
 
 define void @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E, <4 x i32> %F, <8 x i16> %G, <16 x i8> %H, i64 %I) nounwind {
 ; X32-SSE-LABEL: f:
 ; X32-SSE:       # BB#0:
 ; X32-SSE-NEXT:    pushl %ebp
 ; X32-SSE-NEXT:    movl %esp, %ebp
-; X32-SSE-NEXT:    pushl %esi
 ; X32-SSE-NEXT:    andl $-16, %esp
 ; X32-SSE-NEXT:    subl $16, %esp
 ; X32-SSE-NEXT:    movl 72(%ebp), %eax
 ; X32-SSE-NEXT:    movl 76(%ebp), %ecx
-; X32-SSE-NEXT:    movl 12(%ebp), %edx
 ; X32-SSE-NEXT:    movdqa 56(%ebp), %xmm3
 ; X32-SSE-NEXT:    movdqa 40(%ebp), %xmm4
 ; X32-SSE-NEXT:    movdqa 24(%ebp), %xmm5
-; X32-SSE-NEXT:    movl 8(%ebp), %esi
-; X32-SSE-NEXT:    addps .LCPI0_0, %xmm0
-; X32-SSE-NEXT:    movntps %xmm0, (%esi)
-; X32-SSE-NEXT:    paddq .LCPI0_1, %xmm2
-; X32-SSE-NEXT:    movntdq %xmm2, (%esi)
-; X32-SSE-NEXT:    addpd .LCPI0_2, %xmm1
-; X32-SSE-NEXT:    movntpd %xmm1, (%esi)
-; X32-SSE-NEXT:    paddd .LCPI0_3, %xmm5
-; X32-SSE-NEXT:    movntdq %xmm5, (%esi)
-; X32-SSE-NEXT:    paddw .LCPI0_4, %xmm4
-; X32-SSE-NEXT:    movntdq %xmm4, (%esi)
-; X32-SSE-NEXT:    paddb .LCPI0_5, %xmm3
-; X32-SSE-NEXT:    movntdq %xmm3, (%esi)
-; X32-SSE-NEXT:    movntil %edx, (%esi)
-; X32-SSE-NEXT:    movntil %ecx, 4(%esi)
-; X32-SSE-NEXT:    movntil %eax, (%esi)
-; X32-SSE-NEXT:    leal -4(%ebp), %esp
-; X32-SSE-NEXT:    popl %esi
+; X32-SSE-NEXT:    movl 8(%ebp), %edx
+; X32-SSE-NEXT:    addps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    movntps %xmm0, (%edx)
+; X32-SSE-NEXT:    paddq {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movntdq %xmm2, (%edx)
+; X32-SSE-NEXT:    addpd {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    movntpd %xmm1, (%edx)
+; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm5
+; X32-SSE-NEXT:    movntdq %xmm5, (%edx)
+; X32-SSE-NEXT:    paddw {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    movntdq %xmm4, (%edx)
+; X32-SSE-NEXT:    paddb {{\.LCPI.*}}, %xmm3
+; X32-SSE-NEXT:    movntdq %xmm3, (%edx)
+; X32-SSE-NEXT:    movntil %ecx, 4(%edx)
+; X32-SSE-NEXT:    movntil %eax, (%edx)
+; X32-SSE-NEXT:    movl %ebp, %esp
 ; X32-SSE-NEXT:    popl %ebp
 ; X32-SSE-NEXT:    retl
 ;
 ; X32-AVX-LABEL: f:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    pushl %ebp
 ; X32-AVX-NEXT:    movl %esp, %ebp
-; X32-AVX-NEXT:    pushl %esi
 ; X32-AVX-NEXT:    andl $-16, %esp
 ; X32-AVX-NEXT:    subl $16, %esp
 ; X32-AVX-NEXT:    movl 72(%ebp), %eax
 ; X32-AVX-NEXT:    movl 76(%ebp), %ecx
-; X32-AVX-NEXT:    movl 12(%ebp), %edx
 ; X32-AVX-NEXT:    vmovdqa 56(%ebp), %xmm3
 ; X32-AVX-NEXT:    vmovdqa 40(%ebp), %xmm4
 ; X32-AVX-NEXT:    vmovdqa 24(%ebp), %xmm5
-; X32-AVX-NEXT:    movl 8(%ebp), %esi
-; X32-AVX-NEXT:    vaddps .LCPI0_0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vmovntps %xmm0, (%esi)
-; X32-AVX-NEXT:    vpaddq .LCPI0_1, %xmm2, %xmm0
-; X32-AVX-NEXT:    vmovntdq %xmm0, (%esi)
-; X32-AVX-NEXT:    vaddpd .LCPI0_2, %xmm1, %xmm0
-; X32-AVX-NEXT:    vmovntpd %xmm0, (%esi)
-; X32-AVX-NEXT:    vpaddd .LCPI0_3, %xmm5, %xmm0
-; X32-AVX-NEXT:    vmovntdq %xmm0, (%esi)
-; X32-AVX-NEXT:    vpaddw .LCPI0_4, %xmm4, %xmm0
-; X32-AVX-NEXT:    vmovntdq %xmm0, (%esi)
-; X32-AVX-NEXT:    vpaddb .LCPI0_5, %xmm3, %xmm0
-; X32-AVX-NEXT:    vmovntdq %xmm0, (%esi)
-; X32-AVX-NEXT:    movntil %edx, (%esi)
-; X32-AVX-NEXT:    movntil %ecx, 4(%esi)
-; X32-AVX-NEXT:    movntil %eax, (%esi)
-; X32-AVX-NEXT:    leal -4(%ebp), %esp
-; X32-AVX-NEXT:    popl %esi
+; X32-AVX-NEXT:    movl 8(%ebp), %edx
+; X32-AVX-NEXT:    vaddps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-AVX-NEXT:    vmovntps %xmm0, (%edx)
+; X32-AVX-NEXT:    vpaddq {{\.LCPI.*}}, %xmm2, %xmm0
+; X32-AVX-NEXT:    vmovntdq %xmm0, (%edx)
+; X32-AVX-NEXT:    vaddpd {{\.LCPI.*}}, %xmm1, %xmm0
+; X32-AVX-NEXT:    vmovntpd %xmm0, (%edx)
+; X32-AVX-NEXT:    vpaddd {{\.LCPI.*}}, %xmm5, %xmm0
+; X32-AVX-NEXT:    vmovntdq %xmm0, (%edx)
+; X32-AVX-NEXT:    vpaddw {{\.LCPI.*}}, %xmm4, %xmm0
+; X32-AVX-NEXT:    vmovntdq %xmm0, (%edx)
+; X32-AVX-NEXT:    vpaddb {{\.LCPI.*}}, %xmm3, %xmm0
+; X32-AVX-NEXT:    vmovntdq %xmm0, (%edx)
+; X32-AVX-NEXT:    movntil %ecx, 4(%edx)
+; X32-AVX-NEXT:    movntil %eax, (%edx)
+; X32-AVX-NEXT:    movl %ebp, %esp
 ; X32-AVX-NEXT:    popl %ebp
 ; X32-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: f:
 ; X64-SSE:       # BB#0:
 ; X64-SSE-NEXT:    addps {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movntps %xmm0, (%rdi)
 ; X64-SSE-NEXT:    paddq {{.*}}(%rip), %xmm2

llvm/trunk/test/CodeGen/X86/store-narrow.ll

 ; X32: movb %[[REG]], 5(%{{.*}})
 }
 
 ; PR7833
 
 @g_16 = internal global i32 -1
 
 ; X64-LABEL: test8:
-; X64-NEXT: movl _g_16(%rip), %eax
-; X64-NEXT: movl $0, _g_16(%rip)
-; X64-NEXT: orl  $1, %eax
-; X64-NEXT: movl %eax, _g_16(%rip)
+; X64-NEXT: orb  $1, _g_16(%rip)
 ; X64-NEXT: ret
 define void @test8() nounwind {
   %tmp = load i32, i32* @g_16
   store i32 0, i32* @g_16
   %or = or i32 %tmp, 1
   store i32 %or, i32* @g_16
   ret void
 }

llvm/trunk/test/CodeGen/X86/swift-return.ll

 ; CHECK-O0: andb $1, %r8b
 define void @consume_i1_ret() {
   %call = call swiftcc { i1, i1, i1, i1 } @produce_i1_ret()
   %v3 = extractvalue { i1, i1, i1, i1 } %call, 0
   %v5 = extractvalue { i1, i1, i1, i1 } %call, 1
   %v6 = extractvalue { i1, i1, i1, i1 } %call, 2
   %v7 = extractvalue { i1, i1, i1, i1 } %call, 3
   %val = zext i1 %v3 to i32
-  store i32 %val, i32* @var
+  store volatile i32 %val, i32* @var
   %val2 = zext i1 %v5 to i32
-  store i32 %val2, i32* @var
+  store volatile i32 %val2, i32* @var
   %val3 = zext i1 %v6 to i32
-  store i32 %val3, i32* @var
+  store volatile i32 %val3, i32* @var
   %val4 = zext i1 %v7 to i32
   store i32 %val4, i32* @var
   ret void
 }
 
 declare swiftcc { i1, i1, i1, i1 } @produce_i1_ret()
 
 ; CHECK-LABEL: foo:

llvm/trunk/test/CodeGen/X86/win32-spill-xmm.ll

entry:
   ret void
 }
 
 declare void @bar(<16 x float> %a, i32 %b)
 
 ; Check that proper alignment of spilled vector does not affect vargs
 
 ; CHECK-LABEL: vargs_not_affected
-; CHECK: leal    28(%ebp), %eax
+; CHECK: movl 28(%ebp), %eax
 define i32 @vargs_not_affected(<4 x float> %v, i8* %f, ...) {
 entry:
   %ap = alloca i8*, align 4
   %0 = bitcast i8** %ap to i8*
   call void @llvm.va_start(i8* %0)
   %argp.cur = load i8*, i8** %ap, align 4
   %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
   store i8* %argp.next, i8** %ap, align 4

llvm/trunk/test/CodeGen/X86/win64_sibcall.ll

 ; RUN: llc < %s -mtriple=x86_64-pc-win32-coreclr | FileCheck %s -check-prefix=WIN_X64
 ; RUN: llc < %s -mtriple=x86_64-pc-linux         | FileCheck %s -check-prefix=LINUX
 
 %Object = type <{ [0 x i64*]* }>
 
 define void @C1(%Object addrspace(1)* %param0) gc "coreclr" {
 entry:
 
 ; WIN_X64: # BB#0:
 ; WIN_X64:  pushq %rax
 ; LINUX:   # BB#0:                                 # %entry
 ; LINUX:  movq  $0, -8(%rsp)
 
   %this = alloca %Object addrspace(1)*
-  store %Object addrspace(1)* null, %Object addrspace(1)** %this
-  store %Object addrspace(1)* %param0, %Object addrspace(1)** %this
+  store volatile %Object addrspace(1)* null, %Object addrspace(1)** %this
+  store volatile %Object addrspace(1)* %param0, %Object addrspace(1)** %this
   br label %0
 
 ; <label>:0                                       ; preds = %entry
   %1 = load %Object addrspace(1)*, %Object addrspace(1)** %this, align 8
 
 ; WIN_X64:  xorl  %r8d, %r8d
 ; WIN_X64:  popq  %rax
 ; WIN_X64:  jmp   C2                  # TAILCALL

llvm/trunk/test/CodeGen/X86/win64_vararg.ll

entry:
   %cp1 = bitcast i8** %cp to i8*
   call void @llvm.va_start(i8* %ap1)
   call void @llvm.va_copy(i8* %cp1, i8* %ap1)
   ret void
 }
 
 ; CHECK-LABEL: arg4:
 ; CHECK: pushq
-; va_start:
-; CHECK: leaq 48(%rsp), [[REG_arg4_1:%[a-z]+]]
-; CHECK: movq [[REG_arg4_1]], (%rsp)
+; va_start (optimized away as overwritten by va_arg)
 ; va_arg:
 ; CHECK: leaq 52(%rsp), [[REG_arg4_2:%[a-z]+]]
 ; CHECK: movq [[REG_arg4_2]], (%rsp)
 ; CHECK: movl 48(%rsp), %eax
 ; CHECK: popq
 ; CHECK: ret
 define i32 @arg4(i64 %a0, i64 %a1, i64 %a2, i64 %a3, ...) nounwind {
 entry:

llvm/trunk/test/CodeGen/X86/x86-64-ms_abi-vararg.ll

entry:
   %ap.0 = bitcast i8** %ap to i8*
   %cp.0 = bitcast i8** %cp to i8*
   call void @llvm.va_start(i8* %ap.0)
   call void @llvm.va_copy(i8* %cp.0, i8* %ap.0)
   ret void
 }
 
 ; CHECK-LABEL: arg4:
-; va_start:
-; CHECK: leaq 48(%rsp), [[REG_arg4_1:%[a-z]+]]
-; CHECK: movq [[REG_arg4_1]], (%rsp)
+; va_start (optimized away as overwritten by va_arg)
 ; va_arg:
 ; CHECK: leaq 52(%rsp), [[REG_arg4_2:%[a-z]+]]
 ; CHECK: movq [[REG_arg4_2]], (%rsp)
 ; CHECK: movl 48(%rsp), %eax
 ; CHECK: ret
 define x86_64_win64cc i32 @arg4(i64 %a0, i64 %a1, i64 %a2, i64 %a3, ...) nounwind {
 entry:
   %ap = alloca i8*, align 8
   %ap.0 = bitcast i8** %ap to i8*
   call void @llvm.va_start(i8* %ap.0)
   %tmp = va_arg i8** %ap, i32
   ret i32 %tmp
 }