Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -2677,6 +2677,16 @@ SDValue LHS, SDValue RHS, std::function Match, bool AllowUndefs = false, bool AllowTypeMismatch = false); + + /// Returns true if the specified value is the overflow result from one + /// of the overflow intrinsic nodes. + inline bool isOverflowIntrOpRes(SDValue Op) { + unsigned Opc = Op.getOpcode(); + return (Op.getResNo() == 1 && + (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || + Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); + } + } // end namespace ISD } // end namespace llvm Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2344,15 +2344,6 @@ return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result; } -// Returns true if the given Op is the overflow flag result of an overflow -// intrinsic operation. -static bool isOverflowIntrOpRes(SDValue Op) { - unsigned Opc = Op.getOpcode(); - return (Op.getResNo() == 1 && - (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || - Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); -} - static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) { SDValue Sel = Op.getOperand(0); SDValue Other = Op.getOperand(1); @@ -2365,7 +2356,7 @@ // (csel 1, 0, invert(cc), overflow_op_bool) // ... which later gets transformed to just a cset instruction with an // inverted condition code, rather than a cset + eor sequence. - if (isOneConstant(Other) && isOverflowIntrOpRes(Sel)) { + if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) { // Only lower legal XALUO ops. if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0))) return SDValue(); @@ -5052,7 +5043,7 @@ // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch // instruction. - if (isOverflowIntrOpRes(LHS) && isOneConstant(RHS) && + if (ISD::isOverflowIntrOpRes(LHS) && isOneConstant(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { // Only lower legal XALUO ops. if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0))) @@ -5584,7 +5575,7 @@ // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select // instruction. - if (isOverflowIntrOpRes(CCVal)) { + if (ISD::isOverflowIntrOpRes(CCVal)) { // Only lower legal XALUO ops. if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0))) return SDValue(); Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -23132,163 +23132,44 @@ Op.getOperand(1).hasOneUse()); } -/// Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the -/// SETCC node has a single use. -static bool isXor1OfSetCC(SDValue Op) { - if (Op.getOpcode() != ISD::XOR) - return false; - if (isOneConstant(Op.getOperand(1))) - return Op.getOperand(0).getOpcode() == X86ISD::SETCC && - Op.getOperand(0).hasOneUse(); - return false; -} - SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { - bool addTest = true; SDValue Chain = Op.getOperand(0); SDValue Cond = Op.getOperand(1); SDValue Dest = Op.getOperand(2); SDLoc dl(Op); - SDValue CC; - bool Inverted = false; - if (Cond.getOpcode() == ISD::SETCC) { - // Check for setcc([su]{add,sub,mul}o == 0). - if (cast(Cond.getOperand(2))->get() == ISD::SETEQ && - isNullConstant(Cond.getOperand(1)) && - Cond.getOperand(0).getResNo() == 1 && - (Cond.getOperand(0).getOpcode() == ISD::SADDO || - Cond.getOperand(0).getOpcode() == ISD::UADDO || - Cond.getOperand(0).getOpcode() == ISD::SSUBO || - Cond.getOperand(0).getOpcode() == ISD::USUBO || - Cond.getOperand(0).getOpcode() == ISD::SMULO || - Cond.getOperand(0).getOpcode() == ISD::UMULO)) { - Inverted = true; - Cond = Cond.getOperand(0); - } else { - if (SDValue NewCond = LowerSETCC(Cond, DAG)) - Cond = NewCond; - } - } -#if 0 - // FIXME: LowerXALUO doesn't handle these!! - else if (Cond.getOpcode() == X86ISD::ADD || - Cond.getOpcode() == X86ISD::SUB || - Cond.getOpcode() == X86ISD::SMUL || - Cond.getOpcode() == X86ISD::UMUL) - Cond = LowerXALUO(Cond, DAG); -#endif + if (Cond.getOpcode() == ISD::SETCC && + Cond.getOperand(0).getValueType() != MVT::f128) { + SDValue LHS = Cond.getOperand(0); + SDValue RHS = Cond.getOperand(1); + ISD::CondCode CC = cast(Cond.getOperand(2))->get(); - // Look pass (and (setcc_carry (cmp ...)), 1). - if (Cond.getOpcode() == ISD::AND && - Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY && - isOneConstant(Cond.getOperand(1))) - Cond = Cond.getOperand(0); + // Special case for + // setcc([su]{add,sub,mul}o == 0) + // setcc([su]{add,sub,mul}o != 1) + if (ISD::isOverflowIntrOpRes(LHS) && + (CC == ISD::SETEQ || CC == ISD::SETNE) && + (isNullConstant(RHS) || isOneConstant(RHS))) { + SDValue Value, Overflow; + X86::CondCode X86Cond; + std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG); - // If condition flag is set by a X86ISD::CMP, then use it as the condition - // setting operand in place of the X86ISD::SETCC. - unsigned CondOpcode = Cond.getOpcode(); - if (CondOpcode == X86ISD::SETCC || - CondOpcode == X86ISD::SETCC_CARRY) { - CC = Cond.getOperand(0); + if (CC == ISD::SETEQ == isNullConstant(RHS)) + X86Cond = X86::GetOppositeBranchCondition(X86Cond); - SDValue Cmp = Cond.getOperand(1); - unsigned Opc = Cmp.getOpcode(); - // FIXME: WHY THE SPECIAL CASING OF LogicalCmp?? - if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) { - Cond = Cmp; - addTest = false; - } else { - switch (cast(CC)->getZExtValue()) { - default: break; - case X86::COND_O: - case X86::COND_B: - // These can only come from an arithmetic instruction with overflow, - // e.g. SADDO, UADDO. - Cond = Cond.getOperand(1); - addTest = false; - break; - } + SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + Overflow); } - } - CondOpcode = Cond.getOpcode(); - if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO || - CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO || - CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) { - SDValue Value; - X86::CondCode X86Cond; - std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG); - if (Inverted) - X86Cond = X86::GetOppositeBranchCondition(X86Cond); + if (LHS.getSimpleValueType().isInteger()) { + SDValue CCVal; + SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + EFLAGS); + } - CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8); - addTest = false; - } else { - unsigned CondOpc; - if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) { - SDValue Cmp = Cond.getOperand(0).getOperand(1); - if (CondOpc == ISD::OR) { - // Also, recognize the pattern generated by an FCMP_UNE. We can emit - // two branches instead of an explicit OR instruction with a - // separate test. - if (Cmp == Cond.getOperand(1).getOperand(1) && - isX86LogicalCmp(Cmp)) { - CC = Cond.getOperand(0).getOperand(0); - Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), - Chain, Dest, CC, Cmp); - CC = Cond.getOperand(1).getOperand(0); - Cond = Cmp; - addTest = false; - } - } else { // ISD::AND - // Also, recognize the pattern generated by an FCMP_OEQ. We can emit - // two branches instead of an explicit AND instruction with a - // separate test. However, we only do this if this block doesn't - // have a fall-through edge, because this requires an explicit - // jmp when the condition is false. - if (Cmp == Cond.getOperand(1).getOperand(1) && - isX86LogicalCmp(Cmp) && - Op.getNode()->hasOneUse()) { - X86::CondCode CCode0 = - (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); - CCode0 = X86::GetOppositeBranchCondition(CCode0); - CC = DAG.getTargetConstant(CCode0, dl, MVT::i8); - SDNode *User = *Op.getNode()->use_begin(); - // Look for an unconditional branch following this conditional branch. - // We need this because we need to reverse the successors in order - // to implement FCMP_OEQ. - if (User->getOpcode() == ISD::BR) { - SDValue FalseBB = User->getOperand(1); - SDNode *NewBR = - DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); - assert(NewBR == User); - (void)NewBR; - Dest = FalseBB; - - Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), Chain, - Dest, CC, Cmp); - X86::CondCode CCode1 = - (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0); - CCode1 = X86::GetOppositeBranchCondition(CCode1); - CC = DAG.getTargetConstant(CCode1, dl, MVT::i8); - Cond = Cmp; - addTest = false; - } - } - } - } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) { - // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition. - // It should be transformed during dag combiner except when the condition - // is set by a arithmetics with overflow node. - X86::CondCode CCode = - (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); - CCode = X86::GetOppositeBranchCondition(CCode); - CC = DAG.getTargetConstant(CCode, dl, MVT::i8); - Cond = Cond.getOperand(0).getOperand(1); - addTest = false; - } else if (Cond.getOpcode() == ISD::SETCC && - cast(Cond.getOperand(2))->get() == ISD::SETOEQ) { + if (CC == ISD::SETOEQ) { // For FCMP_OEQ, we can emit // two branches instead of an explicit AND instruction with a // separate test. However, we only do this if this block doesn't @@ -23307,56 +23188,65 @@ (void)NewBR; Dest = FalseBB; - SDValue Cmp = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, - Cond.getOperand(0), Cond.getOperand(1)); - CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); - Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), - Chain, Dest, CC, Cmp); - CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); - Cond = Cmp; - addTest = false; + SDValue Cmp = + DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); + SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); + Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, + CCVal, Cmp); + CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + Cmp); } } - } else if (Cond.getOpcode() == ISD::SETCC && - cast(Cond.getOperand(2))->get() == ISD::SETUNE) { + } else if (CC == ISD::SETUNE) { // For FCMP_UNE, we can emit // two branches instead of an explicit OR instruction with a // separate test. - SDValue Cmp = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, - Cond.getOperand(0), Cond.getOperand(1)); - CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); - Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), - Chain, Dest, CC, Cmp); - CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); - Cond = Cmp; - addTest = false; + SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); + SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); + Chain = + DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp); + CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + Cmp); + } else { + X86::CondCode X86Cond = + TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG); + SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); + SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + Cmp); } } - if (addTest) { - // Look pass the truncate if the high bits are known zero. - if (isTruncWithZeroHighBitsInput(Cond, DAG)) - Cond = Cond.getOperand(0); + if (ISD::isOverflowIntrOpRes(Cond)) { + SDValue Value, Overflow; + X86::CondCode X86Cond; + std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG); - // We know the result of AND is compared against zero. Try to match - // it to BT. - if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { - SDValue BTCC; - if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, dl, DAG, BTCC)) { - CC = BTCC; - Cond = BT; - addTest = false; - } - } + SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + Overflow); } - if (addTest) { - X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE; - CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8); - Cond = EmitTest(Cond, X86Cond, dl, DAG, Subtarget); - } - return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), - Chain, Dest, CC, Cond); + // Look past the truncate if the high bits are known zero. + if (isTruncWithZeroHighBitsInput(Cond, DAG)) + Cond = Cond.getOperand(0); + + EVT CondVT = Cond.getValueType(); + + // Add an AND with 1 if we don't already have one. + if (!(Cond.getOpcode() == ISD::AND && isOneConstant(Cond.getOperand(1)))) + Cond = + DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT)); + + SDValue LHS = Cond; + SDValue RHS = DAG.getConstant(0, dl, CondVT); + + SDValue CCVal; + SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal); + return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, + EFLAGS); } // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.