diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -255,6 +255,13 @@ // or custom. }; + /// Enum that specifies when a float negation is beneficial. + enum class NegatibleKind { + Expensive = 0, // Negated expression is more expensive. + Neutral = 1, // Negated expression has the same cost. + Cheaper = 2 // Negated expression is cheaper. + }; + class ArgListEntry { public: Value *Val = nullptr; @@ -3495,12 +3502,12 @@ llvm_unreachable("Not Implemented"); } - /// Return 1 if we can compute the negated form of the specified expression - /// for the same cost as the expression itself, or 2 if we can compute the - /// negated form more cheaply than the expression itself. Else return 0. - virtual char isNegatibleForFree(SDValue Op, SelectionDAG &DAG, - bool LegalOperations, bool ForCodeSize, - unsigned Depth = 0) const; + /// Returns whether computing the negated form of the specified expression is + /// more expensive, the same cost or cheaper. + virtual NegatibleKind isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, + bool ForCodeSize, + unsigned Depth = 0) const; /// If isNegatibleForFree returns true, return the newly negated expression. virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -12164,14 +12164,16 @@ // fold (fadd A, (fneg B)) -> (fsub A, B) if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && - TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) == 2) + TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) == + TargetLowering::NegatibleKind::Cheaper) return DAG.getNode( ISD::FSUB, DL, VT, N0, TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags); // fold (fadd (fneg A), B) -> (fsub B, A) if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && - TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) == 2) + TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) == + TargetLowering::NegatibleKind::Cheaper) return DAG.getNode( ISD::FSUB, DL, VT, N1, TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize), Flags); @@ -12353,7 +12355,8 @@ if (N0CFP && N0CFP->isZero()) { if (N0CFP->isNegative() || (Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros())) { - if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize)) + if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) != + TargetLowering::NegatibleKind::Expensive) return TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize); if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags); @@ -12372,7 +12375,8 @@ } // fold (fsub A, (fneg B)) -> (fadd A, B) - if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize)) + if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) != + TargetLowering::NegatibleKind::Expensive) return DAG.getNode( ISD::FADD, DL, VT, N0, TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags); @@ -12389,16 +12393,20 @@ /// Return true if both inputs are at least as cheap in negated form and at /// least one input is strictly cheaper in negated form. bool DAGCombiner::isCheaperToUseNegatedFPOps(SDValue X, SDValue Y) { - if (char LHSNeg = - TLI.isNegatibleForFree(X, DAG, LegalOperations, ForCodeSize)) - if (char RHSNeg = - TLI.isNegatibleForFree(Y, DAG, LegalOperations, ForCodeSize)) - // Both negated operands are at least as cheap as their counterparts. - // Check to see if at least one is cheaper negated. - if (LHSNeg == 2 || RHSNeg == 2) - return true; + TargetLowering::NegatibleKind LHSNeg = + TLI.isNegatibleForFree(X, DAG, LegalOperations, ForCodeSize); + if (TargetLowering::NegatibleKind::Expensive == LHSNeg) + return false; - return false; + TargetLowering::NegatibleKind RHSNeg = + TLI.isNegatibleForFree(Y, DAG, LegalOperations, ForCodeSize); + if (TargetLowering::NegatibleKind::Expensive == RHSNeg) + return false; + + // Both negated operands are at least as cheap as their counterparts. + // Check to see if at least one is cheaper negated. + return (TargetLowering::NegatibleKind::Cheaper == LHSNeg || + TargetLowering::NegatibleKind::Cheaper == RHSNeg); } SDValue DAGCombiner::visitFMUL(SDNode *N) { @@ -12651,7 +12659,8 @@ // fold ((fma X, (fneg Y), (fneg Z)) -> fneg (fma X, Y, Z)) if (!TLI.isFNegFree(VT) && TLI.isNegatibleForFree(SDValue(N, 0), DAG, LegalOperations, - ForCodeSize) == 2) + ForCodeSize) == + TargetLowering::NegatibleKind::Cheaper) return DAG.getNode(ISD::FNEG, DL, VT, TLI.getNegatedExpression(SDValue(N, 0), DAG, LegalOperations, ForCodeSize), @@ -13386,7 +13395,8 @@ if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); - if (TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize)) + if (TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) != + TargetLowering::NegatibleKind::Expensive) return TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize); // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 FIXME: This is diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -5545,12 +5545,13 @@ return false; } -char TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, - bool LegalOperations, bool ForCodeSize, - unsigned Depth) const { +TargetLowering::NegatibleKind +TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth) const { // fneg is removable even if it has multiple uses. if (Op.getOpcode() == ISD::FNEG) - return 2; + return NegatibleKind::Cheaper; // Don't allow anything with multiple uses unless we know it is free. EVT VT = Op.getValueType(); @@ -5558,97 +5559,105 @@ const TargetOptions &Options = DAG.getTarget().Options; if (!Op.hasOneUse() && !(Op.getOpcode() == ISD::FP_EXTEND && isFPExtFree(VT, Op.getOperand(0).getValueType()))) - return 0; + return NegatibleKind::Expensive; // Don't recurse exponentially. if (Depth > SelectionDAG::MaxRecursionDepth) - return 0; + return NegatibleKind::Expensive; switch (Op.getOpcode()) { case ISD::ConstantFP: { if (!LegalOperations) - return 1; + return NegatibleKind::Neutral; // Don't invert constant FP values after legalization unless the target says // the negated constant is legal. - return isOperationLegal(ISD::ConstantFP, VT) || - isFPImmLegal(neg(cast(Op)->getValueAPF()), VT, - ForCodeSize); + if (isOperationLegal(ISD::ConstantFP, VT) || + isFPImmLegal(neg(cast(Op)->getValueAPF()), VT, + ForCodeSize)) + return NegatibleKind::Neutral; + break; } case ISD::BUILD_VECTOR: { // Only permit BUILD_VECTOR of constants. if (llvm::any_of(Op->op_values(), [&](SDValue N) { return !N.isUndef() && !isa(N); })) - return 0; + return NegatibleKind::Expensive; if (!LegalOperations) - return 1; + return NegatibleKind::Neutral; if (isOperationLegal(ISD::ConstantFP, VT) && isOperationLegal(ISD::BUILD_VECTOR, VT)) - return 1; - return llvm::all_of(Op->op_values(), [&](SDValue N) { - return N.isUndef() || - isFPImmLegal(neg(cast(N)->getValueAPF()), VT, - ForCodeSize); - }); + return NegatibleKind::Neutral; + if (llvm::all_of(Op->op_values(), [&](SDValue N) { + return N.isUndef() || + isFPImmLegal(neg(cast(N)->getValueAPF()), VT, + ForCodeSize); + })) + return NegatibleKind::Neutral; + break; } - case ISD::FADD: + case ISD::FADD: { if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros()) - return 0; + return NegatibleKind::Expensive; // After operation legalization, it might not be legal to create new FSUBs. if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT)) - return 0; + return NegatibleKind::Expensive; // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) - if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, - ForCodeSize, Depth + 1)) - return V; + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V0 != NegatibleKind::Expensive) + return V0; // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations, ForCodeSize, Depth + 1); + } case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros()) - return 0; + return NegatibleKind::Expensive; // fold (fneg (fsub A, B)) -> (fsub B, A) - return 1; - + return NegatibleKind::Neutral; case ISD::FMUL: - case ISD::FDIV: + case ISD::FDIV: { // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y)) - if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, - ForCodeSize, Depth + 1)) - return V; + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V0 != NegatibleKind::Expensive) + return V0; // Ignore X * 2.0 because that is expected to be canonicalized to X + X. if (auto *C = isConstOrConstSplatFP(Op.getOperand(1))) if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL) - return 0; + return NegatibleKind::Expensive; return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations, ForCodeSize, Depth + 1); - + } case ISD::FMA: case ISD::FMAD: { if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros()) - return 0; + return NegatibleKind::Expensive; // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z)) // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z)) - char V2 = isNegatibleForFree(Op.getOperand(2), DAG, LegalOperations, - ForCodeSize, Depth + 1); - if (!V2) - return 0; + NegatibleKind V2 = isNegatibleForFree( + Op.getOperand(2), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (NegatibleKind::Expensive == V2) + return NegatibleKind::Expensive; // One of Op0/Op1 must be cheaply negatible, then select the cheapest. - char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, - ForCodeSize, Depth + 1); - char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations, - ForCodeSize, Depth + 1); - char V01 = std::max(V0, V1); - return V01 ? std::max(V01, V2) : 0; + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + NegatibleKind V1 = isNegatibleForFree( + Op.getOperand(1), DAG, LegalOperations, ForCodeSize, Depth + 1); + NegatibleKind V01 = std::max(V0, V1); + if (V01 == NegatibleKind::Expensive) + return NegatibleKind::Expensive; + return std::max(V01, V2); } case ISD::FP_EXTEND: @@ -5658,7 +5667,7 @@ ForCodeSize, Depth + 1); } - return 0; + return NegatibleKind::Expensive; } SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, @@ -5692,14 +5701,15 @@ } return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops); } - case ISD::FADD: + case ISD::FADD: { assert((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && "Expected NSZ fp-flag"); // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) - if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize, - Depth + 1)) + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V0 != NegatibleKind::Expensive) return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), getNegatedExpression(Op.getOperand(0), DAG, LegalOperations, ForCodeSize, @@ -5711,6 +5721,7 @@ LegalOperations, ForCodeSize, Depth + 1), Op.getOperand(0), Flags); + } case ISD::FSUB: // fold (fneg (fsub 0, B)) -> B if (ConstantFPSDNode *N0CFP = @@ -5723,10 +5734,11 @@ Op.getOperand(1), Op.getOperand(0), Flags); case ISD::FMUL: - case ISD::FDIV: + case ISD::FDIV: { // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) - if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize, - Depth + 1)) + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V0 != NegatibleKind::Expensive) return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), getNegatedExpression(Op.getOperand(0), DAG, LegalOperations, ForCodeSize, @@ -5739,7 +5751,7 @@ getNegatedExpression(Op.getOperand(1), DAG, LegalOperations, ForCodeSize, Depth + 1), Flags); - + } case ISD::FMA: case ISD::FMAD: { assert((DAG.getTarget().Options.NoSignedZerosFPMath || @@ -5749,10 +5761,10 @@ SDValue Neg2 = getNegatedExpression(Op.getOperand(2), DAG, LegalOperations, ForCodeSize, Depth + 1); - char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, - ForCodeSize, Depth + 1); - char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations, - ForCodeSize, Depth + 1); + NegatibleKind V0 = isNegatibleForFree( + Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1); + NegatibleKind V1 = isNegatibleForFree( + Op.getOperand(1), DAG, LegalOperations, ForCodeSize, Depth + 1); // TODO: This is a hack. It is possible that costs have changed between now // and the initial calls to isNegatibleForFree(). That is because we // are rewriting the expression, and that may change the number of diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -172,8 +172,9 @@ bool isZExtFree(EVT Src, EVT Dest) const override; bool isZExtFree(SDValue Val, EVT VT2) const override; - char isNegatibleForFree(SDValue Op, SelectionDAG &DAG, bool LegalOperations, - bool ForCodeSize, unsigned Depth) const override; + NegatibleKind isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth) const override; bool isNarrowingProfitable(EVT VT1, EVT VT2) const override; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -734,16 +734,16 @@ } } -char AMDGPUTargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, - bool LegalOperations, - bool ForCodeSize, - unsigned Depth) const { +TargetLowering::NegatibleKind +AMDGPUTargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth) const { switch (Op.getOpcode()) { case ISD::FMA: case ISD::FMAD: { // Negating a fma is not free if it has users without source mods. if (!allUsesHaveSourceMods(Op.getNode())) - return 0; + return NegatibleKind::Expensive; break; } default: diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -814,11 +814,11 @@ /// and some i16 instructions are slow. bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override; - /// Return 1 if we can compute the negated form of the specified expression - /// for the same cost as the expression itself, or 2 if we can compute the - /// negated form more cheaply than the expression itself. Else return 0. - char isNegatibleForFree(SDValue Op, SelectionDAG &DAG, bool LegalOperations, - bool ForCodeSize, unsigned Depth) const override; + /// Returns whether computing the negated form of the specified expression + /// is more expensive, the same cost or cheaper. + NegatibleKind isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth) const override; /// If isNegatibleForFree returns true, return the newly negated expression. SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -42902,17 +42902,17 @@ return SDValue(); } -char X86TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, - bool LegalOperations, - bool ForCodeSize, - unsigned Depth) const { +TargetLowering::NegatibleKind +X86TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, bool ForCodeSize, + unsigned Depth) const { // fneg patterns are removable even if they have multiple uses. if (isFNEG(DAG, Op.getNode(), Depth)) - return 2; + return NegatibleKind::Cheaper; // Don't recurse exponentially. if (Depth > SelectionDAG::MaxRecursionDepth) - return 0; + return NegatibleKind::Expensive; EVT VT = Op.getValueType(); EVT SVT = VT.getScalarType(); @@ -42932,12 +42932,12 @@ // This is always negatible for free but we might be able to remove some // extra operand negations as well. for (int i = 0; i != 3; ++i) { - char V = isNegatibleForFree(Op.getOperand(i), DAG, LegalOperations, - ForCodeSize, Depth + 1); - if (V == 2) + NegatibleKind V = isNegatibleForFree( + Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V == NegatibleKind::Cheaper) return V; } - return 1; + return NegatibleKind::Neutral; } } @@ -42973,9 +42973,9 @@ // extra operand negations as well. SmallVector NewOps(Op.getNumOperands(), SDValue()); for (int i = 0; i != 3; ++i) { - char V = isNegatibleForFree(Op.getOperand(i), DAG, LegalOperations, - ForCodeSize, Depth + 1); - if (V == 2) + NegatibleKind V = isNegatibleForFree( + Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1); + if (V == NegatibleKind::Cheaper) NewOps[i] = getNegatedExpression(Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1); } @@ -43829,7 +43829,8 @@ auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) { bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); bool LegalOperations = !DCI.isBeforeLegalizeOps(); - if (TLI.isNegatibleForFree(V, DAG, LegalOperations, CodeSize) == 2) { + if (TLI.isNegatibleForFree(V, DAG, LegalOperations, CodeSize) == + TargetLowering::NegatibleKind::Cheaper) { V = TLI.getNegatedExpression(V, DAG, LegalOperations, CodeSize); return true; } @@ -43838,7 +43839,8 @@ if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT && isNullConstant(V.getOperand(1))) { SDValue Vec = V.getOperand(0); - if (TLI.isNegatibleForFree(Vec, DAG, LegalOperations, CodeSize) == 2) { + if (TLI.isNegatibleForFree(Vec, DAG, LegalOperations, CodeSize) == + TargetLowering::NegatibleKind::Cheaper) { SDValue NegVal = TLI.getNegatedExpression(Vec, DAG, LegalOperations, CodeSize); V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(), @@ -43884,7 +43886,8 @@ bool LegalOperations = !DCI.isBeforeLegalizeOps(); SDValue N2 = N->getOperand(2); - if (TLI.isNegatibleForFree(N2, DAG, LegalOperations, CodeSize) != 2) + if (TLI.isNegatibleForFree(N2, DAG, LegalOperations, CodeSize) != + TargetLowering::NegatibleKind::Cheaper) return SDValue(); SDValue NegN2 = TLI.getNegatedExpression(N2, DAG, LegalOperations, CodeSize);