Index: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2706,7 +2706,19 @@ return DAG.getNode(ISD::XOR, DL, VT, V, Cst); } -static SDValue extractBooleanFlip(SDValue V, const TargetLowering &TLI) { +/** + * Flips a boolean if it is cheaper to compute. If the Force parameters is set, + * then the flip also occurs if computing the inverse is the same cost. + * This function returns an empty SDValue in case it cannot flip the boolean + * without increasing the cost of the computation. If you want to flip a boolean + * no matter what, use flipBoolean. + */ +static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, + const TargetLowering &TLI, + bool Force) { + if (Force && isa(V)) + return flipBoolean(V, SDLoc(V), DAG, TLI); + if (V.getOpcode() != ISD::XOR) return SDValue(); @@ -2731,6 +2743,8 @@ if (IsFlip) return V.getOperand(0); + if (Force) + return flipBoolean(V, SDLoc(V), DAG, TLI); return SDValue(); } @@ -2843,11 +2857,10 @@ return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); } - EVT CarryVT = CarryIn.getValueType(); - // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry. if (isNullConstant(N0) && isNullConstant(N1)) { EVT VT = N0.getValueType(); + EVT CarryVT = CarryIn.getValueType(); SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); AddToWorklist(CarryExt.getNode()); return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, @@ -2855,17 +2868,6 @@ DAG.getConstant(0, DL, CarryVT)); } - // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry. - if (isBitwiseNot(N0) && isNullConstant(N1)) { - if (SDValue B = extractBooleanFlip(CarryIn, TLI)) { - SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), - DAG.getConstant(0, DL, N0.getValueType()), - N0.getOperand(0), B); - return CombineTo(N, Sub, - flipBoolean(Sub.getValue(1), DL, DAG, TLI)); - } - } - if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N)) return Combined; @@ -2964,6 +2966,16 @@ SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N) { + // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry. + if (isBitwiseNot(N0)) + if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) { + SDLoc DL(N); + SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1, + N0.getOperand(0), NotC); + return CombineTo(N, Sub, + flipBoolean(Sub.getValue(1), DL, DAG, TLI)); + } + // Iff the flag result is dead: // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry) // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo @@ -8302,7 +8314,7 @@ } // select (not Cond), N1, N2 -> select Cond, N2, N1 - if (SDValue F = extractBooleanFlip(N0, TLI)) { + if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) { SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1); SelectOp->setFlags(Flags); return SelectOp; @@ -8797,7 +8809,7 @@ return V; // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1 - if (SDValue F = extractBooleanFlip(N0, TLI)) + if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) return DAG.getSelect(DL, VT, F, N2, N1); // Canonicalize integer abs. Index: llvm/trunk/test/CodeGen/X86/addcarry.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/addcarry.ll +++ llvm/trunk/test/CodeGen/X86/addcarry.ll @@ -391,11 +391,10 @@ ; CHECK-LABEL: addcarry_to_subcarry: ; CHECK: # %bb.0: ; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: cmpq %rsi, %rdi ; CHECK-NEXT: notq %rsi -; CHECK-NEXT: movb $1, %cl +; CHECK-NEXT: setae %cl ; CHECK-NEXT: addb $-1, %cl -; CHECK-NEXT: movq %rdi, %rcx -; CHECK-NEXT: adcq %rsi, %rcx ; CHECK-NEXT: adcq $0, %rax ; CHECK-NEXT: setb %cl ; CHECK-NEXT: movzbl %cl, %edx Index: llvm/trunk/test/CodeGen/X86/subcarry.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/subcarry.ll +++ llvm/trunk/test/CodeGen/X86/subcarry.ll @@ -90,37 +90,29 @@ define %S @sub(%S* nocapture readonly %this, %S %arg.b) local_unnamed_addr { ; CHECK-LABEL: sub: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset %rbx, -16 ; CHECK-NEXT: movq %rdi, %rax ; CHECK-NEXT: movq (%rsi), %r10 ; CHECK-NEXT: movq 8(%rsi), %rdi -; CHECK-NEXT: movq %r10, %r11 -; CHECK-NEXT: subq %rdx, %r11 -; CHECK-NEXT: notq %rdx -; CHECK-NEXT: movb $1, %bl -; CHECK-NEXT: addb $-1, %bl -; CHECK-NEXT: adcq %r10, %rdx +; CHECK-NEXT: subq %rdx, %r10 +; CHECK-NEXT: setae %dl +; CHECK-NEXT: addb $-1, %dl ; CHECK-NEXT: adcq $0, %rdi ; CHECK-NEXT: setb %dl -; CHECK-NEXT: movzbl %dl, %edx +; CHECK-NEXT: movzbl %dl, %r11d ; CHECK-NEXT: notq %rcx ; CHECK-NEXT: addq %rdi, %rcx -; CHECK-NEXT: adcq 16(%rsi), %rdx -; CHECK-NEXT: setb %bl -; CHECK-NEXT: movzbl %bl, %edi +; CHECK-NEXT: adcq 16(%rsi), %r11 +; CHECK-NEXT: setb %dl +; CHECK-NEXT: movzbl %dl, %edx ; CHECK-NEXT: notq %r8 -; CHECK-NEXT: addq %rdx, %r8 -; CHECK-NEXT: adcq 24(%rsi), %rdi +; CHECK-NEXT: addq %r11, %r8 +; CHECK-NEXT: adcq 24(%rsi), %rdx ; CHECK-NEXT: notq %r9 -; CHECK-NEXT: addq %rdi, %r9 -; CHECK-NEXT: movq %r11, (%rax) +; CHECK-NEXT: addq %rdx, %r9 +; CHECK-NEXT: movq %r10, (%rax) ; CHECK-NEXT: movq %rcx, 8(%rax) ; CHECK-NEXT: movq %r8, 16(%rax) ; CHECK-NEXT: movq %r9, 24(%rax) -; CHECK-NEXT: popq %rbx -; CHECK-NEXT: .cfi_def_cfa_offset 8 ; CHECK-NEXT: retq entry: %0 = extractvalue %S %arg.b, 0