Index: llvm/include/llvm/CodeGen/TargetLowering.h
===================================================================
--- llvm/include/llvm/CodeGen/TargetLowering.h
+++ llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2856,6 +2856,13 @@
return false;
}
+ /// Return true if pulling a binary operation into a select with an identity
+ /// constant is profitable. This is the inverse of an IR transform.
+ /// Example: X + (Cond ? Y : 0) --> Cond ? (X + Y) : X
+ virtual bool shouldFoldBinopIntoSelect(unsigned BinOpcode, EVT VT) const {
+ return false;
+ }
+
/// Return true if it is beneficial to convert a load of a constant to
/// just the constant itself.
/// On some targets it might be more efficient to use a combination of
Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2101,10 +2101,77 @@
VT.getTypeForEVT(*DAG.getContext()), AS);
}
+/// This inverts a canonicalization in IR that replaces a variable select arm
+/// with an identity constant. Codegen improves if we re-use the variable
+/// operand rather than load a constant. This can also be converted into a
+/// masked vector operation if the target supports it.
+static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG,
+ bool ShouldCommuteOperands) {
+ // Match a select as operand 1. The identity constant that we are looking for
+ // is only valid as operand 1 of a non-commutative binop.
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+ if (ShouldCommuteOperands)
+ std::swap(N0, N1);
+
+ // TODO: Should this apply to scalar select too?
+ if (!N1.hasOneUse() || N1.getOpcode() != ISD::VSELECT)
+ return SDValue();
+
+ unsigned Opcode = N->getOpcode();
+ EVT VT = N->getValueType(0);
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (!TLI.shouldFoldBinopIntoSelect(Opcode, VT))
+ return SDValue();
+
+ SDValue Cond = N1.getOperand(0);
+ SDValue TVal = N1.getOperand(1);
+ SDValue FVal = N1.getOperand(2);
+
+ // TODO: The cases should match with IR's ConstantExpr::getBinOpIdentity().
+ // TODO: Target-specific opcodes could be added. Ex: "isCommutativeBinOp()".
+ // TODO: With fast-math (NSZ), allow the opposite-sign form of zero?
+ auto isIdentityConstantForOpcode = [](unsigned Opcode, SDValue V) {
+ if (ConstantFPSDNode *C = isConstOrConstSplatFP(V)) {
+ switch (Opcode) {
+ case ISD::FADD: // X + -0.0 --> X
+ return C->isZero() && C->isNegative();
+ case ISD::FSUB: // X - 0.0 --> X
+ return C->isZero() && !C->isNegative();
+ }
+ }
+ return false;
+ };
+
+ // This transform increases uses of N0, so freeze it to be safe.
+ // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
+ if (isIdentityConstantForOpcode(Opcode, TVal)) {
+ SDValue F0 = DAG.getFreeze(N0);
+ SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags());
+ return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO);
+ }
+ // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
+ if (isIdentityConstantForOpcode(Opcode, FVal)) {
+ SDValue F0 = DAG.getFreeze(N0);
+ SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
+ return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
+ }
+
+ return SDValue();
+}
+
SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&
"Unexpected binary operator");
+ if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false))
+ return Sel;
+
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (TLI.isCommutativeBinOp(BO->getOpcode()))
+ if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true))
+ return Sel;
+
// Don't do this unless the old select is going away. We want to eliminate the
// binary operator, not replace a binop with a select.
// TODO: Handle ISD::SELECT_CC.
Index: llvm/lib/Target/X86/X86ISelLowering.h
===================================================================
--- llvm/lib/Target/X86/X86ISelLowering.h
+++ llvm/lib/Target/X86/X86ISelLowering.h
@@ -1288,6 +1288,8 @@
/// from i32 to i8 but not from i32 to i16.
bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
+ bool shouldFoldBinopIntoSelect(unsigned Opcode, EVT VT) const override;
+
/// Given an intrinsic, checks if on the target the intrinsic will need to map
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and stores the intrinsic information into the IntrinsicInfo that was
Index: llvm/lib/Target/X86/X86ISelLowering.cpp
===================================================================
--- llvm/lib/Target/X86/X86ISelLowering.cpp
+++ llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -33418,6 +33418,20 @@
return !(VT1 == MVT::i32 && VT2 == MVT::i16);
}
+bool X86TargetLowering::shouldFoldBinopIntoSelect(unsigned Opcode,
+ EVT VT) const {
+ // TODO: This is too general. There are cases where pre-AVX512 codegen would
+ // benefit. The transform may also be profitable for scalar code.
+ if (!Subtarget.hasAVX512())
+ return false;
+ if (!Subtarget.hasVLX() && !VT.is512BitVector())
+ return false;
+ if (!VT.isVector())
+ return false;
+
+ return true;
+}
+
/// Targets can use this to indicate that they only support *some*
/// VECTOR_SHUFFLE operations, those with specific masks.
/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
@@ -48909,83 +48923,6 @@
return DAG.getBitcast(VT, CFmul);
}
-/// This inverts a canonicalization in IR that replaces a variable select arm
-/// with an identity constant. Codegen improves if we re-use the variable
-/// operand rather than load a constant. This can also be converted into a
-/// masked vector operation if the target supports it.
-static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG,
- bool ShouldCommuteOperands) {
- // Match a select as operand 1. The identity constant that we are looking for
- // is only valid as operand 1 of a non-commutative binop.
- SDValue N0 = N->getOperand(0);
- SDValue N1 = N->getOperand(1);
- if (ShouldCommuteOperands)
- std::swap(N0, N1);
-
- // TODO: Should this apply to scalar select too?
- if (!N1.hasOneUse() || N1.getOpcode() != ISD::VSELECT)
- return SDValue();
-
- unsigned Opcode = N->getOpcode();
- EVT VT = N->getValueType(0);
- SDValue Cond = N1.getOperand(0);
- SDValue TVal = N1.getOperand(1);
- SDValue FVal = N1.getOperand(2);
-
- // TODO: This (and possibly the entire function) belongs in a
- // target-independent location with target hooks.
- // TODO: The cases should match with IR's ConstantExpr::getBinOpIdentity().
- // TODO: With fast-math (NSZ), allow the opposite-sign form of zero?
- auto isIdentityConstantForOpcode = [](unsigned Opcode, SDValue V) {
- if (ConstantFPSDNode *C = isConstOrConstSplatFP(V)) {
- switch (Opcode) {
- case ISD::FADD: // X + -0.0 --> X
- return C->isZero() && C->isNegative();
- case ISD::FSUB: // X - 0.0 --> X
- return C->isZero() && !C->isNegative();
- }
- }
- return false;
- };
-
- // This transform increases uses of N0, so freeze it to be safe.
- // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
- if (isIdentityConstantForOpcode(Opcode, TVal)) {
- SDValue F0 = DAG.getFreeze(N0);
- SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags());
- return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO);
- }
- // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
- if (isIdentityConstantForOpcode(Opcode, FVal)) {
- SDValue F0 = DAG.getFreeze(N0);
- SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
- return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
- }
-
- return SDValue();
-}
-
-static SDValue combineBinopWithSelect(SDNode *N, SelectionDAG &DAG,
- const X86Subtarget &Subtarget) {
- // TODO: This is too general. There are cases where pre-AVX512 codegen would
- // benefit. The transform may also be profitable for scalar code.
- if (!Subtarget.hasAVX512())
- return SDValue();
-
- if (!Subtarget.hasVLX() && !N->getValueType(0).is512BitVector())
- return SDValue();
-
- if (SDValue Sel = foldSelectWithIdentityConstant(N, DAG, false))
- return Sel;
-
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- if (TLI.isCommutativeBinOp(N->getOpcode()))
- if (SDValue Sel = foldSelectWithIdentityConstant(N, DAG, true))
- return Sel;
-
- return SDValue();
-}
-
/// Do target-specific dag combines on floating-point adds/subs.
static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
@@ -48995,9 +48932,6 @@
if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
return COp;
- if (SDValue Sel = combineBinopWithSelect(N, DAG, Subtarget))
- return Sel;
-
return SDValue();
}