[DAG] Improve loads-from-store forwarding to handle TokenFactor

[niravd]

Summary

Forward store values to matching loads down through token
factors. Factored from D14834.

[hfinkel]

LGTM

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

10256 ↗

(On Diff #76199)

I think this is okay: If the token factor represented dependencies of multiple stores of the same address and type, then the relative order would be undefined, and we'd have other problems.

llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue DAGCombiner::visitLOAD(SDNode *N) {
   }
 
   // If this load is directly stored, replace the load value with the stored
   // value.
   // TODO: Handle store large -> read small portion.
   // TODO: Handle TRUNCSTORE/LOADEXT
   if (OptLevel != CodeGenOpt::None &&
       ISD::isNormalLoad(N) && !LD->isVolatile()) {
-    if (ISD::isNON_TRUNCStore(Chain.getNode())) {
+    // Either a direct store, or a store off of a TokenFactor can be
+    // forwarded.
+    if (Chain->getOpcode() == ISD::TokenFactor) {
+      for (const SDValue &ChainOp : Chain->op_values()) {
+        if (ISD::isNON_TRUNCStore(ChainOp.getNode())) {
+          StoreSDNode *PrevST = cast<StoreSDNode>(ChainOp);
+          if (PrevST->getBasePtr() == Ptr &&
+              PrevST->getValue().getValueType() == N->getValueType(0))
+            return CombineTo(N, PrevST->getOperand(1), Chain);
+        }
+      }
+    } else if (ISD::isNON_TRUNCStore(Chain.getNode())) {
       StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
       if (PrevST->getBasePtr() == Ptr &&
           PrevST->getValue().getValueType() == N->getValueType(0))
-      return CombineTo(N, Chain.getOperand(1), Chain);
+        return CombineTo(N, PrevST->getOperand(1), Chain);
     }
   }
 
   // Try to infer better alignment information than the load already has.
   if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
     if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
       if (Align > LD->getMemOperand()->getBaseAlignment()) {
         SDValue NewLoad = DAG.getExtLoad(

llvm/trunk/test/CodeGen/Mips/o32_cc_byval.ll

 
 declare void @callee2(%struct.S2* byval)
 
 declare void @callee3(float, %struct.S3* byval, %struct.S1* byval)
 
 define void @f2(float %f, %struct.S1* nocapture byval %s1) nounwind {
 entry:
 ; CHECK: addiu $sp, $sp, -48
-; CHECK: sw  $7, 60($sp)
+; CHECK: sw  $[[R7:[0-9]+]], 60($sp)
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: lw  $4, 80($sp)
 ; CHECK: ldc1 $f[[F0:[0-9]+]], 72($sp)
-; CHECK: lw  $[[R3:[0-9]+]], 64($sp)
-; CHECK: lw  $[[R4:[0-9]+]], 68($sp)
-; CHECK: lw  $[[R2:[0-9]+]], 60($sp)
-; CHECK: lh  $[[R1:[0-9]+]], 58($sp)
-; CHECK: lb  $[[R0:[0-9]+]], 56($sp)
-; CHECK: sw  $[[R0]], 32($sp)
-; CHECK: sw  $[[R1]], 28($sp)
-; CHECK: sw  $[[R2]], 24($sp)
-; CHECK: sw  $[[R4]], 20($sp)
-; CHECK: sw  $[[R3]], 16($sp)
+
+; CHECK: lw  $[[R1:[0-9]+]], 64($sp)
+; CHECK: lw  $[[R2:[0-9]+]], 68($sp)
+; CHECK: lh  $[[R3:[0-9]+]], 58($sp)
+; CHECK: lb  $[[R5:[0-9]+]], 56($sp)
+
+; CHECK-DAG: sw  $[[R5]], 32($sp)
+; CHECK-DAG: sw  $[[R3]], 28($sp)
+; CHECK-DAG: sw  $[[R7]], 24($sp)
+; CHECK-DAG: sw  $[[R2]], 20($sp)
+; CHECK-DAG: sw  $[[R1]], 16($sp)
 ; CHECK: mfc1 $6, $f[[F0]]
 
   %i2 = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 5
   %tmp = load i32, i32* %i2, align 4
   %d = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 4
   %tmp1 = load double, double* %d, align 8
   %ll = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 3
   %tmp2 = load i64, i64* %ll, align 8

 
 define void @f3(%struct.S2* nocapture byval %s2) nounwind {
 entry:
 ; CHECK: addiu $sp, $sp, -48
 ; CHECK: sw  $7, 60($sp)
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: sw  $5, 52($sp)
 ; CHECK: sw  $4, 48($sp)
-; CHECK: lw  $4, 48($sp)
-; CHECK: lw  $[[R0:[0-9]+]], 60($sp)
-; CHECK: sw  $[[R0]], 24($sp)
+; CHECK: sw  $7, 24($sp)
 
   %arrayidx = getelementptr inbounds %struct.S2, %struct.S2* %s2, i32 0, i32 0, i32 0
   %tmp = load i32, i32* %arrayidx, align 4
   %arrayidx2 = getelementptr inbounds %struct.S2, %struct.S2* %s2, i32 0, i32 0, i32 3
   %tmp3 = load i32, i32* %arrayidx2, align 4
   tail call void @callee4(i32 %tmp, double 2.000000e+00, i64 3, i32 %tmp3, i16 signext 4, i8 signext 5, float 6.000000e+00) nounwind
   ret void
 }
 
 define void @f4(float %f, %struct.S3* nocapture byval %s3, %struct.S1* nocapture byval %s1) nounwind {
 entry:
 ; CHECK: addiu $sp, $sp, -48
 ; CHECK: sw  $7, 60($sp)
 ; CHECK: sw  $6, 56($sp)
 ; CHECK: sw  $5, 52($sp)
-; CHECK: lw  $4, 60($sp)
 ; CHECK: lw  $[[R1:[0-9]+]], 80($sp)
 ; CHECK: lb  $[[R0:[0-9]+]], 52($sp)
 ; CHECK: sw  $[[R0]], 32($sp)
 ; CHECK: sw  $[[R1]], 24($sp)
 
   %i = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 2
   %tmp = load i32, i32* %i, align 4
   %i2 = getelementptr inbounds %struct.S1, %struct.S1* %s1, i32 0, i32 5

llvm/trunk/test/CodeGen/PowerPC/anon_aggr.ll

entry:
   br i1 %cond, label %equal, label %unequal
 equal:
   ret i8* %array1_ptr
 unequal:
   ret i8* %array2_ptr
 }
 
 ; CHECK-LABEL: func2:
-; CHECK: ld [[REG2:[0-9]+]], 72(1)
-; CHECK: cmpld {{([0-9]+,)?}}4, [[REG2]]
-; CHECK-DAG: std [[REG2]], -[[OFFSET1:[0-9]+]]
-; CHECK-DAG: std 4, -[[OFFSET2:[0-9]+]]
+; CHECK: cmpld 4, [[REG:[0-9]+]]
+; CHECK: mr 3, [[REG]]
+; CHECK-DAG: std 3, -[[OFFSET1:[0-9]+]](1)
+; CHECK-DAG: std 4, -[[OFFSET2:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET2]](1)
 ; CHECK: ld 3, -[[OFFSET1]](1)
 
 ; DARWIN32: _func2:
 ; DARWIN32: addi r[[REG1:[0-9]+]], r[[REGSP:[0-9]+]], 36
 ; DARWIN32: lwz r[[REG2:[0-9]+]], 44(r[[REGSP]])
 ; DARWIN32: mr
 ; DARWIN32: mr r[[REG3:[0-9]+]], r[[REGA:[0-9]+]]
entry:
   br i1 %cond, label %equal, label %unequal
 equal:
   ret i8* %array1_ptr
 unequal:
   ret i8* %array2_ptr
 }
 
 ; CHECK-LABEL: func3:
-; CHECK: ld [[REG3:[0-9]+]], 72(1)
-; CHECK: ld [[REG4:[0-9]+]], 56(1)
-; CHECK: cmpld {{([0-9]+,)?}}[[REG4]], [[REG3]]
-; CHECK: std [[REG3]], -[[OFFSET1:[0-9]+]](1)
-; CHECK: std [[REG4]], -[[OFFSET2:[0-9]+]](1)
+; CHECK: cmpld 4, 6
+; CHECK: mr [[REGA:[0-9]+]], 6
+; CHECK: mr [[REGB:[0-9]+]], 4
+; CHECK-DAG: std [[REGA]], -[[OFFSET1:[0-9]+]](1)
+; CHECK-DAG: std [[REGB]], -[[OFFSET2:[0-9]+]](1)
 ; CHECK: ld 3, -[[OFFSET2]](1)
 ; CHECK: ld 3, -[[OFFSET1]](1)
 
 ; DARWIN32: _func3:
 ; DARWIN32: addi r[[REG1:[0-9]+]], r[[REGSP:[0-9]+]], 36
 ; DARWIN32: addi r[[REG2:[0-9]+]], r[[REGSP]], 24
 ; DARWIN32: lwz r[[REG3:[0-9]+]], 44(r[[REGSP]])
 ; DARWIN32: lwz r[[REG4:[0-9]+]], 32(r[[REGSP]])

llvm/trunk/test/CodeGen/PowerPC/complex-return.ll

entry:
   %imag2 = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 1
   store ppc_fp128 %x.real, ppc_fp128* %real1
   store ppc_fp128 %x.imag, ppc_fp128* %imag2
   %0 = load { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval
   ret { ppc_fp128, ppc_fp128 } %0
 }
 
 ; CHECK-LABEL: foo:
-; CHECK: lfd 1
-; CHECK: lfd 2
-; CHECK: lfd 3
-; CHECK: lfd 4
+; CHECK-DAG: lfd 1
+; CHECK-DAG: lfd 2
+; CHECK-DAG: lfd 3
+; CHECK-DAG: lfd 4
 
 define { float, float } @oof() nounwind {
 entry:
   %retval = alloca { float, float }, align 4
   %x = alloca { float, float }, align 4
   %real = getelementptr inbounds { float, float }, { float, float }* %x, i32 0, i32 0
   %imag = getelementptr inbounds { float, float }, { float, float }* %x, i32 0, i32 1
   store float 3.500000e+00, float* %real

llvm/trunk/test/CodeGen/PowerPC/ppc64-align-long-double.ll

 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O0 -fast-isel=false -mattr=-vsx < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-VSX %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr9 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -O0 -fast-isel=false -mattr=+vsx < %s | FileCheck -check-prefix=CHECK-PWR9 %s
 
 ; Verify internal alignment of long double in a struct.  The double
 ; argument comes in in GPR3; GPR4 is skipped; GPRs 5 and 6 contain
 ; the long double.  Check that these are stored to proper locations
 ; in the parameter save area and loaded from there for return in FPR1/2.
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 %struct.S = type { double, ppc_fp128 }
 
 define ppc_fp128 @test(%struct.S* byval %x) nounwind {
 entry:
   %b = getelementptr inbounds %struct.S, %struct.S* %x, i32 0, i32 1
   %0 = load ppc_fp128, ppc_fp128* %b, align 16
   ret ppc_fp128 %0
 }
 
 ; CHECK-DAG: std 6, 72(1)
 ; CHECK-DAG: std 5, 64(1)
 ; CHECK-DAG: std 4, 56(1)
 ; CHECK-DAG: std 3, 48(1)
-; CHECK: lfd 1, 64(1)
-; CHECK: lfd 2, 72(1)
+; CHECK: std 5, -16(1)
+; CHECK: std 6, -8(1)
+; CHECK: lfd 1, -16(1)
+; CHECK: lfd 2, -8(1)
+
 
 ; CHECK-VSX-DAG: std 6, 72(1)
 ; CHECK-VSX-DAG: std 5, 64(1)
 ; CHECK-VSX-DAG: std 4, 56(1)
 ; CHECK-VSX-DAG: std 3, 48(1)
-; CHECK-VSX: li 3, 16
-; CHECK-VSX: addi 4, 1, 48
-; CHECK-VSX: lxsdx 1, 4, 3
-; CHECK-VSX: li 3, 24
-; CHECK-VSX: lxsdx 2, 4, 3
+; CHECK-VSX:  std 5, -16(1)
+; CHECK-VSX:  std 6, -8(1)
+; CHECK-VSX:  addi 3, 1, -16
+; CHECK-VSX:  lxsdx 1, 0, 3
+; CHECK-VSX:  addi 3, 1, -8
+; CHECK-VSX:  lxsdx 2, 0, 3
+
+
+; CHECK-PWR9-DAG: std 6, 72(1)
+; CHECK-PWR9-DAG: std 5, 64(1)
+; CHECK-PWR9-DAG: std 4, 56(1)
+; CHECK-PWR9-DAG: std 3, 48(1)
+; CHECK-PWR9:   mtvsrd 1, 5
+; CHECK-PWR9:   mtvsrd 2, 6
 No newline at end of file