Index: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
===================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10265,11 +10265,22 @@
   // TODO: Handle TRUNCSTORE/LOADEXT
   if (OptLevel != CodeGenOpt::None &&
       ISD::isNormalLoad(N) && !LD->isVolatile()) {
-    if (ISD::isNON_TRUNCStore(Chain.getNode())) {
+    // Either a direct store, or a store off of a TokenFactor can be
+    // forwarded.
+    if (Chain->getOpcode() == ISD::TokenFactor) {
+      for (const SDValue &ChainOp : Chain->op_values()) {
+        if (ISD::isNON_TRUNCStore(ChainOp.getNode())) {
+          StoreSDNode *PrevST = cast<StoreSDNode>(ChainOp);
+          if (PrevST->getBasePtr() == Ptr &&
+              PrevST->getValue().getValueType() == N->getValueType(0))
+            return CombineTo(N, PrevST->getOperand(1), Chain);
+        }
+      }
+    } else if (ISD::isNON_TRUNCStore(Chain.getNode())) {
       StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
       if (PrevST->getBasePtr() == Ptr &&
           PrevST->getValue().getValueType() == N->getValueType(0))
-      return CombineTo(N, Chain.getOperand(1), Chain);
+        return CombineTo(N, PrevST->getOperand(1), Chain);
     }
   }
 
Index: llvm/trunk/test/CodeGen/Mips/o32_cc_byval.ll
===================================================================
--- llvm/trunk/test/CodeGen/Mips/o32_cc_byval.ll
+++ llvm/trunk/test/CodeGen/Mips/o32_cc_byval.ll
@@ -45,20 +45,21 @@
 define void @f2(float %f, %struct.S1* nocapture byval %s1) nounwind {
 entry:
 ; CHECK: addiu $sp, $sp, -48
-; CHECK: sw  $7, 60($sp)
+; CHECK: sw  $[[R7:[0-9]+]], 60($sp)
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: lw  $4, 80($sp)
 ; CHECK: ldc1 $f[[F0:[0-9]+]], 72($sp)
-; CHECK: lw  $[[R3:[0-9]+]], 64($sp)
-; CHECK: lw  $[[R4:[0-9]+]], 68($sp)
-; CHECK: lw  $[[R2:[0-9]+]], 60($sp)
-; CHECK: lh  $[[R1:[0-9]+]], 58($sp)
-; CHECK: lb  $[[R0:[0-9]+]], 56($sp)
-; CHECK: sw  $[[R0]], 32($sp)
-; CHECK: sw  $[[R1]], 28($sp)
-; CHECK: sw  $[[R2]], 24($sp)
-; CHECK: sw  $[[R4]], 20($sp)
-; CHECK: sw  $[[R3]], 16($sp)
+
+; CHECK: lw  $[[R1:[0-9]+]], 64($sp)
+; CHECK: lw  $[[R2:[0-9]+]], 68($sp)
+; CHECK: lh  $[[R3:[0-9]+]], 58($sp)
+; CHECK: lb  $[[R5:[0-9]+]], 56($sp)
+
+; CHECK-DAG: sw  $[[R5]], 32($sp)
+; CHECK-DAG: sw  $[[R3]], 28($sp)
+; CHECK-DAG: sw  $[[R7]], 24($sp)
+; CHECK-DAG: sw  $[[R2]], 20($sp)
+; CHECK-DAG: sw  $[[R1]], 16($sp)
 ; CHECK: mfc1 $6, $f[[F0]]
 
   %i2 = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 5
@@ -86,9 +87,7 @@
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: sw  $5, 52($sp)
 ; CHECK: sw  $4, 48($sp)
-; CHECK: lw  $4, 48($sp)
-; CHECK: lw  $[[R0:[0-9]+]], 60($sp)
-; CHECK: sw  $[[R0]], 24($sp)
+; CHECK: sw  $7, 24($sp)
 
   %arrayidx = getelementptr inbounds %struct.S2, %struct.S2* %s2, i32 0, i32 0, i32 0
   %tmp = load i32, i32* %arrayidx, align 4
@@ -104,7 +103,6 @@
 ; CHECK: sw  $7, 60($sp)
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: sw  $5, 52($sp)
-; CHECK: lw  $4, 60($sp)
 ; CHECK: lw  $[[R1:[0-9]+]], 80($sp)
 ; CHECK: lb  $[[R0:[0-9]+]], 52($sp)
 ; CHECK: sw  $[[R0]], 32($sp)
Index: llvm/trunk/test/CodeGen/PowerPC/anon_aggr.ll
===================================================================
--- llvm/trunk/test/CodeGen/PowerPC/anon_aggr.ll
+++ llvm/trunk/test/CodeGen/PowerPC/anon_aggr.ll
@@ -62,10 +62,10 @@
 }
 
 ; CHECK-LABEL: func2:
-; CHECK: ld [[REG2:[0-9]+]], 72(1)
-; CHECK: cmpld {{([0-9]+,)?}}4, [[REG2]]
-; CHECK-DAG: std [[REG2]], -[[OFFSET1:[0-9]+]]
-; CHECK-DAG: std 4, -[[OFFSET2:[0-9]+]]
+; CHECK: cmpld 4, [[REG:[0-9]+]]
+; CHECK: mr 3, [[REG]]
+; CHECK-DAG: std 3, -[[OFFSET1:[0-9]+]](1)
+; CHECK-DAG: std 4, -[[OFFSET2:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET2]](1)
 ; CHECK: ld 3, -[[OFFSET1]](1)
 
@@ -106,11 +106,11 @@
 }
 
 ; CHECK-LABEL: func3:
-; CHECK: ld [[REG3:[0-9]+]], 72(1)
-; CHECK: ld [[REG4:[0-9]+]], 56(1)
-; CHECK: cmpld {{([0-9]+,)?}}[[REG4]], [[REG3]]
-; CHECK: std [[REG3]], -[[OFFSET1:[0-9]+]](1)
-; CHECK: std [[REG4]], -[[OFFSET2:[0-9]+]](1)
+; CHECK: cmpld 4, 6
+; CHECK: mr [[REGA:[0-9]+]], 6
+; CHECK: mr [[REGB:[0-9]+]], 4
+; CHECK-DAG: std [[REGA]], -[[OFFSET1:[0-9]+]](1)
+; CHECK-DAG: std [[REGB]], -[[OFFSET2:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET2]](1)
 ; CHECK: ld 3, -[[OFFSET1]](1)
 
Index: llvm/trunk/test/CodeGen/PowerPC/complex-return.ll
===================================================================
--- llvm/trunk/test/CodeGen/PowerPC/complex-return.ll
+++ llvm/trunk/test/CodeGen/PowerPC/complex-return.ll
@@ -24,10 +24,10 @@
 }
 
 ; CHECK-LABEL: foo:
-; CHECK: lfd 1
-; CHECK: lfd 2
-; CHECK: lfd 3
-; CHECK: lfd 4
+; CHECK-DAG: lfd 1
+; CHECK-DAG: lfd 2
+; CHECK-DAG: lfd 3
+; CHECK-DAG: lfd 4
 
 define { float, float } @oof() nounwind {
 entry:
Index: llvm/trunk/test/CodeGen/PowerPC/ppc64-align-long-double.ll
===================================================================
--- llvm/trunk/test/CodeGen/PowerPC/ppc64-align-long-double.ll
+++ llvm/trunk/test/CodeGen/PowerPC/ppc64-align-long-double.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O0 -fast-isel=false -mattr=-vsx < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-VSX %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr9 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-PWR9 %s
 
 ; Verify internal alignment of long double in a struct.  The double
 ; argument comes in in GPR3; GPR4 is skipped; GPRs 5 and 6 contain
@@ -23,15 +23,27 @@
 ; CHECK-DAG: std 5, 64(1)
 ; CHECK-DAG: std 4, 56(1)
 ; CHECK-DAG: std 3, 48(1)
-; CHECK: lfd 1, 64(1)
-; CHECK: lfd 2, 72(1)
+; CHECK: std 5, -16(1)
+; CHECK: std 6, -8(1)
+; CHECK: lfd 1, -16(1)
+; CHECK: lfd 2, -8(1)
+
 
 ; CHECK-VSX-DAG: std 6, 72(1)
 ; CHECK-VSX-DAG: std 5, 64(1)
 ; CHECK-VSX-DAG: std 4, 56(1)
 ; CHECK-VSX-DAG: std 3, 48(1)
-; CHECK-VSX: li 3, 16
-; CHECK-VSX: addi 4, 1, 48
-; CHECK-VSX: lxsdx 1, 4, 3
-; CHECK-VSX: li 3, 24
-; CHECK-VSX: lxsdx 2, 4, 3
+; CHECK-VSX:	std 5, -16(1)
+; CHECK-VSX:	std 6, -8(1)
+; CHECK-VSX:	addi 3, 1, -16
+; CHECK-VSX:	lxsdx 1, 0, 3
+; CHECK-VSX:	addi 3, 1, -8
+; CHECK-VSX:	lxsdx 2, 0, 3
+
+
+; CHECK-PWR9-DAG:	std 6, 72(1)
+; CHECK-PWR9-DAG:	std 5, 64(1)
+; CHECK-PWR9-DAG:	std 4, 56(1)
+; CHECK-PWR9-DAG:	std 3, 48(1)
+; CHECK-PWR9:		mtvsrd 1, 5
+; CHECK-PWR9:		mtvsrd 2, 6
\ No newline at end of file