Index: include/llvm/Target/TargetLowering.h
===================================================================
--- include/llvm/Target/TargetLowering.h
+++ include/llvm/Target/TargetLowering.h
@@ -142,6 +142,14 @@
CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
};
+ /// Enum that specifies what [US]MUL_LOHI is expanded to.
+ enum class MulExpansion {
+ None, // Do not actually expand anything
+ Split, // Split into MUL + MULH[US]
+ Scalarize, // Scalarize vector operations
+ HalfWidth, // Sequence of half-width multiplications and additions
+ };
+
static ISD::NodeType getExtendForContent(BooleanContent Content) {
switch (Content) {
case UndefinedBooleanContent:
@@ -258,6 +266,13 @@
return BypassSlowDivWidths;
}
+ /// Returns which of a number of possible expansions should be used for
+ /// [US]MUL_LOHI instructions. This is only used when the corresponding
+ /// operation actions is set to Expand.
+ virtual MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const {
+ return MulExpansion::Split;
+ }
+
/// Return true if Flow Control is an expensive operation that should be
/// avoided.
bool isJumpExpensive() const { return JumpIsExpensive; }
@@ -2999,9 +3014,28 @@
// Legalization utility functions
//
+ /// Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes,
+ /// respectively, each computing an n/2-bit part of the result.
+ /// \param Result A vector that will be filled with the parts of the result
+ /// in little-endian order.
+ /// \param HalfVT The value type to use for the result nodes.
+ /// \param OnlyLegalOrCustom Only legal or custom instructions are used.
+ /// \param LL Low bits of the LHS of the MUL. You can use this parameter
+ /// if you want to control how low bits are extracted from the LHS.
+ /// \param LH High bits of the LHS of the MUL. See LL for meaning.
+ /// \param RL Low bits of the RHS of the MUL. See LL for meaning
+ /// \param RH High bits of the RHS of the MUL. See LL for meaning.
+ /// \returns true if the node has been expanded, false if it has not
+ bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS,
+ SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HalfVT,
+ SelectionDAG &DAG, bool OnlyLegalOrCustom,
+ SDValue LL = SDValue(), SDValue LH = SDValue(),
+ SDValue RL = SDValue(), SDValue RH = SDValue()) const;
+
/// Expand a MUL into two nodes. One that computes the high bits of
/// the result and one that computes the low bits.
/// \param HiLoVT The value type to use for the Lo and Hi nodes.
+ /// \param OnlyLegalOrCustom Only legal or custom instructions are used.
/// \param LL Low bits of the LHS of the MUL. You can use this parameter
/// if you want to control how low bits are extracted from the LHS.
/// \param LH High bits of the LHS of the MUL. See LL for meaning.
@@ -3009,9 +3043,9 @@
/// \param RH High bits of the RHS of the MUL. See LL for meaning.
/// \returns true if the node has been expanded. false if it has not
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
- SelectionDAG &DAG, SDValue LL = SDValue(),
- SDValue LH = SDValue(), SDValue RL = SDValue(),
- SDValue RH = SDValue()) const;
+ SelectionDAG &DAG, bool OnlyLegalOrCustom,
+ SDValue LL = SDValue(), SDValue LH = SDValue(),
+ SDValue RL = SDValue(), SDValue RH = SDValue()) const;
/// Expand float(f32) to SINT(i64) conversion
/// \param N Node to expand
Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3302,17 +3302,87 @@
}
case ISD::MULHU:
case ISD::MULHS: {
- unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
- ISD::SMUL_LOHI;
+ unsigned ExpandOpcode =
+ Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
EVT VT = Node->getValueType(0);
SDVTList VTs = DAG.getVTList(VT, VT);
- assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) &&
- "If this wasn't legal, it shouldn't have been created!");
+
Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
Node->getOperand(1));
Results.push_back(Tmp1.getValue(1));
break;
}
+ case ISD::UMUL_LOHI:
+ case ISD::SMUL_LOHI: {
+ SDValue LHS = Node->getOperand(0);
+ SDValue RHS = Node->getOperand(1);
+ MVT VT = LHS.getSimpleValueType();
+ TargetLowering::MulExpansion Kind =
+ TLI.getMulExpansion(Node->getOpcode(), VT);
+
+ switch (Kind) {
+ case TargetLowering::MulExpansion::None:
+ break;
+
+ case TargetLowering::MulExpansion::Split:
+ Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
+ Results.push_back(DAG.getNode(
+ Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS, dl, VT,
+ LHS, RHS));
+ break;
+
+ case TargetLowering::MulExpansion::Scalarize: {
+ MVT ScalarVT = VT.getScalarType();
+ unsigned NumElements = VT.getVectorNumElements();
+ MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
+ SmallVector<SDValue, 16> Scalars;
+
+ for (unsigned i = 0; i < NumElements; ++i) {
+ SDValue Idx = DAG.getConstant(i, dl, IdxTy);
+ Scalars.push_back(DAG.getNode(
+ Node->getOpcode(), dl, DAG.getVTList(ScalarVT, ScalarVT),
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, LHS, Idx),
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, RHS, Idx)));
+ }
+
+ for (unsigned i = 0; i < 2; ++i) {
+ SmallVector<SDValue, 16> Elements;
+ for (unsigned j = 0; j < NumElements; ++j)
+ Elements.push_back(Scalars[j].getValue(i));
+ Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elements));
+ }
+ break;
+ }
+
+ case TargetLowering::MulExpansion::HalfWidth: {
+ SmallVector<SDValue, 4> Halves;
+ EVT HalfType;
+ if (!VT.isVector()) {
+ HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
+ } else {
+ HalfType = EVT::getVectorVT(
+ *DAG.getContext(),
+ EVT(VT.getScalarType()).getHalfSizedIntegerVT(*DAG.getContext()),
+ VT.getVectorNumElements());
+ }
+ if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
+ HalfType, DAG, false)) {
+ for (unsigned i = 0; i < 2; ++i) {
+ SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
+ SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
+ SDValue Shift = DAG.getConstant(
+ HalfType.getSizeInBits(), dl,
+ TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
+ Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
+ Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
+ }
+ break;
+ }
+ break;
+ }
+ }
+ break;
+ }
case ISD::MUL: {
EVT VT = Node->getValueType(0);
SDVTList VTs = DAG.getVTList(VT, VT);
@@ -3347,7 +3417,7 @@
TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
- TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) {
+ TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, true)) {
Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
SDValue Shift =
@@ -4147,6 +4217,24 @@
Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
break;
}
+ case ISD::UMUL_LOHI:
+ case ISD::SMUL_LOHI: {
+ // Promote to a multiply in a wider integer type.
+ unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
+ : ISD::SIGN_EXTEND;
+ Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
+ Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
+ Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
+
+ auto &DL = DAG.getDataLayout();
+ unsigned OriginalSize = OVT.getScalarSizeInBits();
+ Tmp2 = DAG.getNode(
+ ISD::SRL, dl, NVT, Tmp1,
+ DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
+ break;
+ }
case ISD::SELECT: {
unsigned ExtOp, TruncOp;
if (Node->getValueType(0).isVector() ||
Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2176,7 +2176,7 @@
GetExpandedInteger(N->getOperand(0), LL, LH);
GetExpandedInteger(N->getOperand(1), RL, RH);
- if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH))
+ if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, true, LL, LH, RL, RH))
return;
// If nothing else, we can make a libcall.
Index: lib/CodeGen/SelectionDAG/TargetLowering.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3015,108 +3015,190 @@
// Legalization Utilities
//===----------------------------------------------------------------------===//
-bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
- SelectionDAG &DAG, SDValue LL, SDValue LH,
- SDValue RL, SDValue RH) const {
- EVT VT = N->getValueType(0);
- SDLoc dl(N);
+bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
+ SDValue LHS, SDValue RHS,
+ SmallVectorImpl<SDValue> &Result,
+ EVT HalfVT, SelectionDAG &DAG,
+ bool OnlyLegalOrCustom, SDValue LL,
+ SDValue LH, SDValue RL, SDValue RH) const {
+ assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
+ Opcode == ISD::SMUL_LOHI);
+
+ bool HasMULHS =
+ !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::MULHS, HalfVT);
+ bool HasMULHU =
+ !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::MULHU, HalfVT);
+ bool HasSMUL_LOHI =
+ !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::SMUL_LOHI, HalfVT);
+ bool HasUMUL_LOHI =
+ !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::UMUL_LOHI, HalfVT);
+ unsigned OuterBitSize = VT.getSizeInBits();
+ unsigned InnerBitSize = HalfVT.getSizeInBits();
+ unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
+ unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
+
+ // LL, LH, RL, and RH must be either all NULL or all set to a value.
+ assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
+ (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
+
+ if (!HasMULHS && !HasMULHU && !HasSMUL_LOHI && !HasUMUL_LOHI)
+ return false;
- bool HasMULHS = isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
- bool HasMULHU = isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
- bool HasSMUL_LOHI = isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
- bool HasUMUL_LOHI = isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
- if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
- unsigned OuterBitSize = VT.getSizeInBits();
- unsigned InnerBitSize = HiLoVT.getSizeInBits();
- unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0));
- unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1));
-
- // LL, LH, RL, and RH must be either all NULL or all set to a value.
- assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
- (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
-
- if (!LL.getNode() && !RL.getNode() &&
- isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
- LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, N->getOperand(0));
- RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, N->getOperand(1));
+ SDVTList VTs = DAG.getVTList(HalfVT, HalfVT);
+ auto MakeUMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo,
+ SDValue &Hi) -> bool {
+ if (HasUMUL_LOHI) {
+ Lo = DAG.getNode(ISD::UMUL_LOHI, dl, VTs, L, R);
+ Hi = SDValue(Lo.getNode(), 1);
+ return true;
}
-
- if (!LL.getNode())
- return false;
-
- APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
- if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) &&
- DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) {
- // The inputs are both zero-extended.
- if (HasUMUL_LOHI) {
- // We can emit a umul_lohi.
- Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(HiLoVT, HiLoVT), LL,
- RL);
- Hi = SDValue(Lo.getNode(), 1);
- return true;
- }
- if (HasMULHU) {
- // We can emit a mulhu+mul.
- Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
- Hi = DAG.getNode(ISD::MULHU, dl, HiLoVT, LL, RL);
- return true;
- }
+ if (HasMULHU) {
+ Lo = DAG.getNode(ISD::MUL, dl, HalfVT, L, R);
+ Hi = DAG.getNode(ISD::MULHU, dl, HalfVT, L, R);
+ return true;
}
- if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) {
- // The input values are both sign-extended.
- if (HasSMUL_LOHI) {
- // We can emit a smul_lohi.
- Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(HiLoVT, HiLoVT), LL,
- RL);
- Hi = SDValue(Lo.getNode(), 1);
- return true;
- }
- if (HasMULHS) {
- // We can emit a mulhs+mul.
- Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
- Hi = DAG.getNode(ISD::MULHS, dl, HiLoVT, LL, RL);
- return true;
- }
+ return false;
+ };
+ auto MakeSMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo,
+ SDValue &Hi) -> bool {
+ if (HasSMUL_LOHI) {
+ Lo = DAG.getNode(ISD::SMUL_LOHI, dl, VTs, L, R);
+ Hi = SDValue(Lo.getNode(), 1);
+ return true;
}
-
- if (!LH.getNode() && !RH.getNode() &&
- isOperationLegalOrCustom(ISD::SRL, VT) &&
- isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
- auto &DL = DAG.getDataLayout();
- unsigned ShiftAmt = VT.getSizeInBits() - HiLoVT.getSizeInBits();
- SDValue Shift = DAG.getConstant(ShiftAmt, dl, getShiftAmountTy(VT, DL));
- LH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(0), Shift);
- LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
- RH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(1), Shift);
- RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
+ if (HasMULHS) {
+ Lo = DAG.getNode(ISD::MUL, dl, HalfVT, L, R);
+ Hi = DAG.getNode(ISD::MULHS, dl, HalfVT, L, R);
+ return true;
}
+ return false;
+ };
- if (!LH.getNode())
- return false;
+ SDValue Lo, Hi;
- if (HasUMUL_LOHI) {
- // Lo,Hi = umul LHS, RHS.
- SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl,
- DAG.getVTList(HiLoVT, HiLoVT), LL, RL);
- Lo = UMulLOHI;
- Hi = UMulLOHI.getValue(1);
- RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
- LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
- Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
- Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
+ if (!LL.getNode() && !RL.getNode() &&
+ isOperationLegalOrCustom(ISD::TRUNCATE, HalfVT)) {
+ LL = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, LHS);
+ RL = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, RHS);
+ }
+
+ if (!LL.getNode())
+ return false;
+
+ APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
+ if (DAG.MaskedValueIsZero(LHS, HighMask) &&
+ DAG.MaskedValueIsZero(RHS, HighMask)) {
+ // The inputs are both zero-extended.
+ if (MakeUMUL_LOHI(LL, RL, Lo, Hi)) {
+ Result.push_back(Lo);
+ Result.push_back(Hi);
+ if (Opcode != ISD::MUL) {
+ SDValue Zero = DAG.getConstant(0, dl, HalfVT);
+ Result.push_back(Zero);
+ Result.push_back(Zero);
+ }
return true;
}
- if (HasMULHU) {
- Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
- Hi = DAG.getNode(ISD::MULHU, dl, HiLoVT, LL, RL);
- RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
- LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
- Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
- Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
+ }
+
+ if (LHSSB > InnerBitSize && RHSSB > InnerBitSize && Opcode == ISD::MUL) {
+ // The input values are both sign-extended.
+ // TODO non-MUL case?
+ if (MakeSMUL_LOHI(LL, RL, Lo, Hi)) {
+ Result.push_back(Lo);
+ Result.push_back(Hi);
return true;
}
}
- return false;
+
+ if (!LH.getNode() && !RH.getNode() &&
+ isOperationLegalOrCustom(ISD::SRL, VT) &&
+ isOperationLegalOrCustom(ISD::TRUNCATE, HalfVT)) {
+ auto &DL = DAG.getDataLayout();
+ unsigned ShiftAmt = VT.getSizeInBits() - HalfVT.getSizeInBits();
+ SDValue Shift = DAG.getConstant(ShiftAmt, dl, getShiftAmountTy(VT, DL));
+ LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
+ LH = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, LH);
+ RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
+ RH = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, RH);
+ }
+
+ if (!LH.getNode())
+ return false;
+
+ if (!MakeUMUL_LOHI(LL, RL, Lo, Hi))
+ return false;
+
+ Result.push_back(Lo);
+
+ if (Opcode == ISD::MUL) {
+ RH = DAG.getNode(ISD::MUL, dl, HalfVT, LL, RH);
+ LH = DAG.getNode(ISD::MUL, dl, HalfVT, LH, RL);
+ Hi = DAG.getNode(ISD::ADD, dl, HalfVT, Hi, RH);
+ Hi = DAG.getNode(ISD::ADD, dl, HalfVT, Hi, LH);
+ Result.push_back(Hi);
+ return true;
+ }
+
+ SDValue Next = Hi;
+ if (!MakeUMUL_LOHI(LL, RH, Lo, Hi))
+ return false;
+
+ SDVTList CarryVTs = DAG.getVTList(HalfVT, MVT::Glue);
+ SDValue SumLo, SumHi;
+ SumHi = Hi;
+ SumLo = DAG.getNode(ISD::ADDC, dl, CarryVTs, Next, Lo);
+
+ if (!MakeUMUL_LOHI(LH, RL, Lo, Hi))
+ return false;
+
+ SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Hi, SumLo.getValue(1));
+ SumLo = DAG.getNode(ISD::ADDC, dl, CarryVTs, SumLo, Lo);
+ Result.push_back(SumLo);
+
+ if (!(Opcode == ISD::UMUL_LOHI ? MakeUMUL_LOHI(LH, RH, Lo, Hi)
+ : MakeSMUL_LOHI(LH, RH, Lo, Hi)))
+ return false;
+
+ SDValue Zero = DAG.getConstant(0, dl, HalfVT);
+ SumLo = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Lo, SumLo.getValue(1));
+ SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, Hi, Zero, SumHi.getValue(1));
+ SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Zero, SumLo.getValue(1));
+
+ if (Opcode == ISD::SMUL_LOHI) {
+ SDValue LoSub = DAG.getNode(ISD::SUBC, dl, CarryVTs, SumLo, RL);
+ SDValue HiSub =
+ DAG.getNode(ISD::SUBE, dl, CarryVTs, SumHi, Zero, LoSub.getValue(1));
+
+ SumLo = DAG.getSelectCC(dl, LH, Zero, LoSub, SumLo, ISD::SETLT);
+ SumHi = DAG.getSelectCC(dl, LH, Zero, HiSub, SumHi, ISD::SETLT);
+
+ LoSub = DAG.getNode(ISD::SUBC, dl, CarryVTs, SumLo, LL);
+ HiSub =
+ DAG.getNode(ISD::SUBE, dl, CarryVTs, SumHi, Zero, LoSub.getValue(1));
+
+ SumLo = DAG.getSelectCC(dl, RH, Zero, LoSub, SumLo, ISD::SETLT);
+ SumHi = DAG.getSelectCC(dl, RH, Zero, HiSub, SumHi, ISD::SETLT);
+ }
+
+ Result.push_back(SumLo);
+ Result.push_back(SumHi);
+ return true;
+}
+
+bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
+ SelectionDAG &DAG, bool OnlyLegalOrCustom,
+ SDValue LL, SDValue LH, SDValue RL,
+ SDValue RH) const {
+ SmallVector<SDValue, 2> Result;
+ bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
+ N->getOperand(0), N->getOperand(1), Result, HiLoVT,
+ DAG, OnlyLegalOrCustom, LL, LH, RL, RH);
+ if (Result.size() >= 2) {
+ Lo = Result[0];
+ Hi = Result[1];
+ }
+ return Ok;
}
bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
Index: lib/Target/AMDGPU/AMDGPUISelLowering.h
===================================================================
--- lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -121,6 +121,8 @@
bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
+ MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
MVT getVectorIdxTy(const DataLayout &) const override;
bool isSelectSupported(SelectSupportKind) const override;
Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp
===================================================================
--- lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -622,6 +622,13 @@
return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
}
+TargetLowering::MulExpansion
+AMDGPUTargetLowering::getMulExpansion(unsigned Opcode, MVT VT) const {
+ if (VT.getSizeInBits() > 32)
+ return MulExpansion::HalfWidth;
+ return MulExpansion::Split;
+}
+
//===---------------------------------------------------------------------===//
// TargetLowering Callbacks
//===---------------------------------------------------------------------===//
Index: lib/Target/BPF/BPFISelLowering.h
===================================================================
--- lib/Target/BPF/BPFISelLowering.h
+++ lib/Target/BPF/BPFISelLowering.h
@@ -46,6 +46,8 @@
EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *BB) const override;
+ MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
private:
SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
Index: lib/Target/BPF/BPFISelLowering.cpp
===================================================================
--- lib/Target/BPF/BPFISelLowering.cpp
+++ lib/Target/BPF/BPFISelLowering.cpp
@@ -132,6 +132,11 @@
MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 128;
}
+TargetLowering::MulExpansion BPFTargetLowering::getMulExpansion(unsigned Opcode,
+ MVT VT) const {
+ return MulExpansion::HalfWidth;
+}
+
SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
case ISD::BR_CC:
Index: lib/Target/PowerPC/PPCISelLowering.h
===================================================================
--- lib/Target/PowerPC/PPCISelLowering.h
+++ lib/Target/PowerPC/PPCISelLowering.h
@@ -567,6 +567,8 @@
SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
std::vector<SDNode *> *Created) const override;
+ MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
unsigned getRegisterByName(const char* RegName, EVT VT,
SelectionDAG &DAG) const override;
Index: lib/Target/PowerPC/PPCISelLowering.cpp
===================================================================
--- lib/Target/PowerPC/PPCISelLowering.cpp
+++ lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11435,6 +11435,13 @@
return Op;
}
+TargetLowering::MulExpansion PPCTargetLowering::getMulExpansion(unsigned Opcode,
+ MVT VT) const {
+ if (VT.isVector())
+ return MulExpansion::Scalarize;
+ return MulExpansion::Split;
+}
+
//===----------------------------------------------------------------------===//
// Inline Assembly Support
//===----------------------------------------------------------------------===//
Index: lib/Target/Sparc/SparcISelLowering.h
===================================================================
--- lib/Target/Sparc/SparcISelLowering.h
+++ lib/Target/Sparc/SparcISelLowering.h
@@ -205,6 +205,8 @@
return true;
}
+ MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
void ReplaceNodeResults(SDNode *N,
Index: lib/Target/Sparc/SparcISelLowering.cpp
===================================================================
--- lib/Target/Sparc/SparcISelLowering.cpp
+++ lib/Target/Sparc/SparcISelLowering.cpp
@@ -1394,6 +1394,13 @@
// TargetLowering Implementation
//===----------------------------------------------------------------------===//
+TargetLowering::MulExpansion
+SparcTargetLowering::getMulExpansion(unsigned Opcode, MVT VT) const {
+ if (VT.getSizeInBits() > 32)
+ return MulExpansion::HalfWidth;
+ return MulExpansion::Split;
+}
+
TargetLowering::AtomicExpansionKind SparcTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
if (AI->getOperation() == AtomicRMWInst::Xchg &&
AI->getType()->getPrimitiveSizeInBits() == 32)
@@ -1685,11 +1692,11 @@
setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
- // FIXME: Sparc provides these multiplies, but we don't have them yet.
- setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
- setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
-
if (Subtarget->is64Bit()) {
+ // FIXME: Sparc provides these multiplies, but we don't have them yet.
+ setOperationAction(ISD::UMUL_LOHI, MVT::i32, Promote);
+ setOperationAction(ISD::SMUL_LOHI, MVT::i32, Promote);
+
setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
setOperationAction(ISD::MULHU, MVT::i64, Expand);
@@ -1701,6 +1708,10 @@
setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
+ } else {
+ // FIXME: Sparc provides these multiplies, but we don't have them yet.
+ setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
+ setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
}
// VASTART needs to be custom lowered to use the VarArgsFrameIndex.
Index: lib/Target/X86/X86ISelLowering.h
===================================================================
--- lib/Target/X86/X86ISelLowering.h
+++ lib/Target/X86/X86ISelLowering.h
@@ -1026,6 +1026,8 @@
bool isIntDivCheap(EVT VT, AttributeSet Attr) const override;
+ MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
bool supportSwiftError() const override;
protected:
Index: lib/Target/X86/X86ISelLowering.cpp
===================================================================
--- lib/Target/X86/X86ISelLowering.cpp
+++ lib/Target/X86/X86ISelLowering.cpp
@@ -32526,6 +32526,13 @@
return OptSize && !VT.isVector();
}
+TargetLowering::MulExpansion X86TargetLowering::getMulExpansion(unsigned Opcode,
+ MVT VT) const {
+ if (VT.getScalarSizeInBits() >= 64)
+ return MulExpansion::Scalarize;
+ return MulExpansion::Split;
+}
+
void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
if (!Subtarget.is64Bit())
return;