diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -4955,11 +4955,6 @@ EVT ShSVT = ShVT.getScalarType(); unsigned EltBits = VT.getScalarSizeInBits(); - // Check to see if we can do this. - // FIXME: We should be more aggressive here. - if (!isTypeLegal(VT)) - return SDValue(); - // If the sdiv has an 'exact' bit we can use a simpler lowering. if (N->getFlags().hasExact()) return BuildExactSDIV(*this, N, dl, DAG, Created); @@ -5017,7 +5012,6 @@ } // Multiply the numerator (operand 0) by the magic value. - // FIXME: We should support doing a MUL in a wider type. SDValue Q; if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT) : isOperationLegalOrCustom(ISD::MULHS, VT)) @@ -5027,8 +5021,22 @@ SDValue LoHi = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor); Q = SDValue(LoHi.getNode(), 1); - } else - return SDValue(); // No mulhs or equivalent. + } else { + // No MULHU or UMUL_LOHI. Multiply in a wider integer and take the upper part. + // Even if targets do not support the other two instructions well, they are pretty + // good at supporting plain `MUL` at any width. + EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), EltBits * 2); + if (VT.isVector()) + WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, VT.getVectorNumElements()); + SDValue DoubleMul = DAG.getNode(ISD::MUL, dl, WideVT, + DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N0), + DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, MagicFactor)); + SDValue Upper = DAG.getNode(ISD::SRL, dl, WideVT, + DoubleMul, + DAG.getConstant(EltBits, dl, WideVT)); + SDValue Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Upper); + Q = Hi; + } Created.push_back(Q.getNode()); // (Optionally) Add/subtract the numerator using Factor. @@ -5064,11 +5072,6 @@ EVT ShSVT = ShVT.getScalarType(); unsigned EltBits = VT.getScalarSizeInBits(); - // Check to see if we can do this. - // FIXME: We should be more aggressive here. - if (!isTypeLegal(VT)) - return SDValue(); - bool UseNPQ = false; SmallVector PreShifts, PostShifts, MagicFactors, NPQFactors; @@ -5137,7 +5140,6 @@ Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); Created.push_back(Q.getNode()); - // FIXME: We should support doing a MUL in a wider type. auto GetMULHU = [&](SDValue X, SDValue Y) { if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT) : isOperationLegalOrCustom(ISD::MULHU, VT)) @@ -5148,7 +5150,20 @@ DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); return SDValue(LoHi.getNode(), 1); } - return SDValue(); // No mulhu or equivalent + // No MULHU or UMUL_LOHI. Multiply in a wider integer and take the upper part. + // Even if targets do not support the other two instructions well, they are pretty + // good at supporting plain `MUL` at any width. + EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), EltBits * 2); + if (VT.isVector()) + WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, VT.getVectorNumElements()); + SDValue DoubleMul = DAG.getNode(ISD::MUL, dl, WideVT, + DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X), + DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y)); + SDValue Upper = DAG.getNode(ISD::SRL, dl, WideVT, + DoubleMul, + DAG.getConstant(EltBits, dl, WideVT)); + SDValue Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Upper); + return Hi; }; // Multiply the numerator (operand 0) by the magic value.