[DAGCombiner] Enable UADDO/USUBO vector combine support

[RKSimon]

test/CodeGen/X86/combine-addo.ll

145

Looks like we're missing a blendv + ~cmp combine -> commuted blendv - I'll add this separately.

[nikic]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

2463

Should this also become isNullorNullSplat? (Does ADDCARRY support vectors?)

[RKSimon]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

2463

AFAICT we don't have ISD::ADDCARRY/SUBCARRY support for vectors - it might be safest to just early-out from visitUADDOLike if VT.isVector()?

[nikic]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

2463

Bailing out early on isVector() sounds reasonable. The code should be safe as-is, but that would make it more obvious that this transform doesn't apply to vectors.

[RKSimon]

Early out from visitUADDOLike for vector types - we don't have ISD::ADDCARRY vector support

[nikic]

LGTM

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) {
   }
   llvm_unreachable("Unsupported boolean content");
 }
 
 SDValue DAGCombiner::visitUADDO(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N0.getValueType();
-  if (VT.isVector())
-    return SDValue();
 
   EVT CarryVT = N->getValueType(1);
   SDLoc DL(N);
 
   // If the flag result is dead, turn this into an ADD.
   if (!N->hasAnyUseOfValue(1))
     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                      DAG.getUNDEF(CarryVT));
 
   // canonicalize constant to RHS.
-  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (N0C && !N1C)
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
 
   // fold (uaddo x, 0) -> x + no carry out
-  if (isNullConstant(N1))
+  if (isNullOrNullSplat(N1))
     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
 
   // If it cannot overflow, transform into an add.
   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                      DAG.getConstant(0, DL, CarryVT));
 
   // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
SDValue DAGCombiner::visitUADDO(SDNode *N) {
   return SDValue();
 }
 
 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
   auto VT = N0.getValueType();
 
   // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
   // If Y + 1 cannot overflow.
   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
     SDValue Y = N1.getOperand(0);
     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
                          N1.getOperand(2));
   }
 
   // (uaddo X, Carry) -> (addcarry X, 0, Carry)
SDValue DAGCombiner::visitSUBC(SDNode *N) {
 
   return SDValue();
 }
 
 SDValue DAGCombiner::visitUSUBO(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N0.getValueType();
-  if (VT.isVector())
-    return SDValue();
 
   EVT CarryVT = N->getValueType(1);
   SDLoc DL(N);
 
   // If the flag result is dead, turn this into an SUB.
   if (!N->hasAnyUseOfValue(1))
     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
                      DAG.getUNDEF(CarryVT));
 
   // fold (usubo x, x) -> 0 + no borrow
   if (N0 == N1)
     return CombineTo(N, DAG.getConstant(0, DL, VT),
                      DAG.getConstant(0, DL, CarryVT));
 
   // fold (usubo x, 0) -> x + no borrow
-  if (isNullConstant(N1))
+  if (isNullOrNullSplat(N1))
     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
 
   // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
-  if (isAllOnesConstant(N0))
+  if (isAllOnesOrAllOnesSplat(N0))
     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
                      DAG.getConstant(0, DL, CarryVT));
 
   return SDValue();
 }
 
 SDValue DAGCombiner::visitSUBE(SDNode *N) {
   SDValue N0 = N->getOperand(0);

test/CodeGen/X86/combine-addo.ll

; AVX-NEXT:    retq
   %3 = extractvalue {i32, i1} %1, 1
   %4 = select i1 %3, i32 %a1, i32 %2
   ret i32 %4
 }
 
 define <4 x i32> @combine_vec_uadd_zero(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_uadd_zero:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    pmaxud %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_uadd_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmaxud %xmm0, %xmm0, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
   %3 = extractvalue {<4 x i32>, <4 x i1>} %1, 1
   %4 = select <4 x i1> %3, <4 x i32> %a1, <4 x i32> %2
   ret <4 x i32> %4
 }
 
; AVX-NEXT:    retq
   %4 = extractvalue {i32, i1} %2, 1
   %5 = select i1 %4, i32 %a1, i32 %3
   ret i32 %5
 }
 
 define <4 x i32> @combine_vec_uadd_not(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_uadd_not:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm3, %xmm0
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    psubd %xmm3, %xmm2
-; SSE-NEXT:    pmaxud %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    psubd %xmm0, %xmm2
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,1,1,1]
+; SSE-NEXT:    pmaxud %xmm2, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE-NEXT:    pxor %xmm3, %xmm0
 ; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_uadd_not:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpsubd %xmm2, %xmm0, %xmm3
-; AVX-NEXT:    vpmaxud %xmm0, %xmm3, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm3, %xmm0
-; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm3, %xmm0
+; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
+; AVX-NEXT:    vpmaxud %xmm2, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = xor <4 x i32> %a0, <i32 -1, i32 -1, i32 -1, i32 -1>
   %2 = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32> %1, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
   %3 = extractvalue {<4 x i32>, <4 x i1>} %2, 0
   %4 = extractvalue {<4 x i32>, <4 x i1>} %2, 1
   %5 = select <4 x i1> %4, <4 x i32> %a1, <4 x i32> %3
   ret <4 x i32> %5
 }

test/CodeGen/X86/combine-subo.ll

; AVX-NEXT:    retq
   %3 = extractvalue {i32, i1} %1, 1
   %4 = select i1 %3, i32 %a1, i32 %2
   ret i32 %4
 }
 
 define <4 x i32> @combine_vec_usub_zero(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_usub_zero:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    pminud %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_usub_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpminud %xmm0, %xmm0, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> zeroinitializer)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
   %3 = extractvalue {<4 x i32>, <4 x i1>} %1, 1
   %4 = select <4 x i1> %3, <4 x i32> %a1, <4 x i32> %2
   ret <4 x i32> %4
 }
 
; AVX-NEXT:    retq
   %3 = extractvalue {i32, i1} %1, 1
   %4 = select i1 %3, i32 %a1, i32 %2
   ret i32 %4
 }
 
 define <4 x i32> @combine_vec_usub_self(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_usub_self:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    psubd %xmm0, %xmm2
-; SSE-NEXT:    pminud %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    xorps %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_usub_self:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpsubd %xmm0, %xmm0, %xmm2
-; AVX-NEXT:    vpminud %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a0)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
   %3 = extractvalue {<4 x i32>, <4 x i1>} %1, 1
   %4 = select <4 x i1> %3, <4 x i32> %a1, <4 x i32> %2
   ret <4 x i32> %4
 }
 
; AVX-NEXT:    retq
   %3 = extractvalue {i32, i1} %1, 1
   %4 = select i1 %3, i32 %a1, i32 %2
   ret i32 %4
 }
 
 define <4 x i32> @combine_vec_usub_negone(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_usub_negone:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm0
-; SSE-NEXT:    pminud %xmm3, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_usub_negone:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpminud %xmm2, %xmm0, %xmm3
-; AVX-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm3
-; AVX-NEXT:    vpxor %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a0)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
   %3 = extractvalue {<4 x i32>, <4 x i1>} %1, 1
   %4 = select <4 x i1> %3, <4 x i32> %a1, <4 x i32> %2
   ret <4 x i32> %4
 }