[DAGCombiner][X86] Teach SimplifyVBinOp to fold VBinOp (concat X, undef/constant), (concat Y, undef/constant) -> concat (VBinOp X, Y), VecC

[craig.topper]

Summary

This improves the combine I included in D66504 to handle constants in the upper operands of the concat. If we can constant fold them away we can pull the concat after the bin op. This helps with chains of madd reductions on X86 from loop unrolling. The loop madd reduction pattern creates pmaddwd with half the width of the add that follows it using zeroes to fill the upper bits. If we have two of these added together we can pull the zeroes through the accumulating add and then shrink it.

[craig.topper]

llvm/test/CodeGen/X86/madd.ll

2761

We were just barely able to prove the bits were all 0 and therefore disjoint, but weren't able to remove the ADD/OR completely because the disjoint bits code gets to go one level deeper in computeKnownBits then SimplifyDemandedBits. This is because the disjoint check starts at a depth of 0 for the operands, but the SimplifyDemandedBits starts at a depth of 0 for the OR node itself.

[craig.topper]

Rebase after D66504.

[RKSimon]

LGTM - cheers

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

if (NarrowVT == Y.getValueType() &&
       SDLoc DL(N);
       SDValue VecC =
           DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT));
       SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y);
       return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z);
     }
   }
 
+  // Make sure all but the first op are undef or constant.
+  auto ConcatWithConstantOrUndef = [](SDValue Concat) {
+    assert(Concat.getOpcode() == ISD::CONCAT_VECTORS && "Expected concat!");
+    return std::all_of(std::next(Concat->op_begin()), Concat->op_end(),
+                     [](const SDValue &Op) {
+                       return Op.isUndef() ||
+                              ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
+                     });
+  };
+
+  // The following pattern is likely to emerge with vector reduction ops. Moving
+  // the binary operation ahead of the concat may allow using a narrower vector
+  // instruction that has better performance than the wide version of the op:
+  // VBinOp (concat X, undef/constant), (concat Y, undef/constant) -->
+  //   concat (VBinOp X, Y), VecC
+  if (LHS.getOpcode() == ISD::CONCAT_VECTORS && LHS.hasOneUse() &&
+      RHS.getOpcode() == ISD::CONCAT_VECTORS && RHS.hasOneUse() &&
+      ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS)) {
+    EVT NarrowVT = LHS.getOperand(0).getValueType();
+    if (NarrowVT == RHS.getOperand(0).getValueType() &&
+        TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT)) {
+      SDLoc DL(N);
+      unsigned NumOperands = LHS.getNumOperands();
+      SmallVector<SDValue, 4> Ops;
+      for (unsigned i = 0; i != NumOperands; ++i) {
+        // This constant fold for operands 1 and up.
+        Ops.push_back(DAG.getNode(Opcode, DL, NarrowVT, LHS.getOperand(i),
+                                  RHS.getOperand(i)));
+      }
+
+      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops);
+    }
+  }
+
   if (SDValue V = scalarizeBinOpOfSplats(N, DAG))
     return V;
 
   return SDValue();
 }
 
 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
                                     SDValue N2) {

llvm/test/CodeGen/X86/madd.ll

 ; SSE2-NEXT:    paddd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: madd_quad_reduction:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX1-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
-; AVX1-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqu (%r8), %xmm2
-; AVX1-NEXT:    vpmaddwd (%r9), %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqu (%rax), %xmm2
-; AVX1-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX256-LABEL: madd_quad_reduction:
-; AVX256:       # %bb.0:
-; AVX256-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX256-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX256-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX256-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX256-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
-; AVX256-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
-; AVX256-NEXT:    vmovdqu (%r8), %xmm2
-; AVX256-NEXT:    vpmaddwd (%r9), %xmm2, %xmm2
-; AVX256-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; AVX256-NEXT:    vmovdqu (%rax), %xmm2
-; AVX256-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
-; AVX256-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX256-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vmovd %xmm0, %eax
-; AVX256-NEXT:    vzeroupper
-; AVX256-NEXT:    retq
+; AVX-LABEL: madd_quad_reduction:
+; AVX:       # %bb.0:
+; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
+; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqu (%r8), %xmm2
+; AVX-NEXT:    vpmaddwd (%r9), %xmm2, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vmovdqu (%rax), %xmm2
+; AVX-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
   %tmp = load <8 x i16>, <8 x i16>* %arg, align 1
   %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1
   %tmp7 = sext <8 x i16> %tmp to <8 x i32>
   %tmp17 = sext <8 x i16> %tmp6 to <8 x i32>
   %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17
   %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1
   %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1
   %tmp22 = sext <8 x i16> %tmp20 to <8 x i32>