Index: include/llvm/Target/TargetLowering.h
===================================================================
--- include/llvm/Target/TargetLowering.h
+++ include/llvm/Target/TargetLowering.h
@@ -142,6 +142,14 @@
     CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
   };
 
+  /// Enum that specifies what [US]MUL_LOHI is expanded to.
+  enum class MulExpansion {
+    None,      // Do not actually expand anything
+    Split,     // Split into MUL + MULH[US]
+    Scalarize, // Scalarize vector operations
+    HalfWidth, // Sequence of half-width multiplications and additions
+  };
+
   static ISD::NodeType getExtendForContent(BooleanContent Content) {
     switch (Content) {
     case UndefinedBooleanContent:
@@ -258,6 +266,13 @@
     return BypassSlowDivWidths;
   }
 
+  /// Returns which of a number of possible expansions should be used for
+  /// [US]MUL_LOHI instructions. This is only used when the corresponding
+  /// operation actions is set to Expand.
+  virtual MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const {
+    return MulExpansion::Split;
+  }
+
   /// Return true if Flow Control is an expensive operation that should be
   /// avoided.
   bool isJumpExpensive() const { return JumpIsExpensive; }
@@ -2999,9 +3014,28 @@
   // Legalization utility functions
   //
 
+  /// Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes,
+  /// respectively, each computing an n/2-bit part of the result.
+  /// \param Result A vector that will be filled with the parts of the result
+  ///        in little-endian order.
+  /// \param HalfVT The value type to use for the result nodes.
+  /// \param OnlyLegalOrCustom Only legal or custom instructions are used.
+  /// \param LL Low bits of the LHS of the MUL.  You can use this parameter
+  ///        if you want to control how low bits are extracted from the LHS.
+  /// \param LH High bits of the LHS of the MUL.  See LL for meaning.
+  /// \param RL Low bits of the RHS of the MUL.  See LL for meaning
+  /// \param RH High bits of the RHS of the MUL.  See LL for meaning.
+  /// \returns true if the node has been expanded, false if it has not
+  bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS,
+                      SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HalfVT,
+                      SelectionDAG &DAG, bool OnlyLegalOrCustom,
+                      SDValue LL = SDValue(), SDValue LH = SDValue(),
+                      SDValue RL = SDValue(), SDValue RH = SDValue()) const;
+
   /// Expand a MUL into two nodes.  One that computes the high bits of
   /// the result and one that computes the low bits.
   /// \param HiLoVT The value type to use for the Lo and Hi nodes.
+  /// \param OnlyLegalOrCustom Only legal or custom instructions are used.
   /// \param LL Low bits of the LHS of the MUL.  You can use this parameter
   ///        if you want to control how low bits are extracted from the LHS.
   /// \param LH High bits of the LHS of the MUL.  See LL for meaning.
@@ -3009,9 +3043,9 @@
   /// \param RH High bits of the RHS of the MUL.  See LL for meaning.
   /// \returns true if the node has been expanded. false if it has not
   bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
-                 SelectionDAG &DAG, SDValue LL = SDValue(),
-                 SDValue LH = SDValue(), SDValue RL = SDValue(),
-                 SDValue RH = SDValue()) const;
+                 SelectionDAG &DAG, bool OnlyLegalOrCustom,
+                 SDValue LL = SDValue(), SDValue LH = SDValue(),
+                 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
 
   /// Expand float(f32) to SINT(i64) conversion
   /// \param N Node to expand
Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3302,17 +3302,87 @@
   }
   case ISD::MULHU:
   case ISD::MULHS: {
-    unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
-                                                              ISD::SMUL_LOHI;
+    unsigned ExpandOpcode =
+        Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
     EVT VT = Node->getValueType(0);
     SDVTList VTs = DAG.getVTList(VT, VT);
-    assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) &&
-           "If this wasn't legal, it shouldn't have been created!");
+
     Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
                        Node->getOperand(1));
     Results.push_back(Tmp1.getValue(1));
     break;
   }
+  case ISD::UMUL_LOHI:
+  case ISD::SMUL_LOHI: {
+    SDValue LHS = Node->getOperand(0);
+    SDValue RHS = Node->getOperand(1);
+    MVT VT = LHS.getSimpleValueType();
+    TargetLowering::MulExpansion Kind =
+        TLI.getMulExpansion(Node->getOpcode(), VT);
+
+    switch (Kind) {
+    case TargetLowering::MulExpansion::None:
+      break;
+
+    case TargetLowering::MulExpansion::Split:
+      Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
+      Results.push_back(DAG.getNode(
+          Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS, dl, VT,
+          LHS, RHS));
+      break;
+
+    case TargetLowering::MulExpansion::Scalarize: {
+      MVT ScalarVT = VT.getScalarType();
+      unsigned NumElements = VT.getVectorNumElements();
+      MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
+      SmallVector<SDValue, 16> Scalars;
+
+      for (unsigned i = 0; i < NumElements; ++i) {
+        SDValue Idx = DAG.getConstant(i, dl, IdxTy);
+        Scalars.push_back(DAG.getNode(
+            Node->getOpcode(), dl, DAG.getVTList(ScalarVT, ScalarVT),
+            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, LHS, Idx),
+            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, RHS, Idx)));
+      }
+
+      for (unsigned i = 0; i < 2; ++i) {
+        SmallVector<SDValue, 16> Elements;
+        for (unsigned j = 0; j < NumElements; ++j)
+          Elements.push_back(Scalars[j].getValue(i));
+        Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elements));
+      }
+      break;
+    }
+
+    case TargetLowering::MulExpansion::HalfWidth: {
+      SmallVector<SDValue, 4> Halves;
+      EVT HalfType;
+      if (!VT.isVector()) {
+        HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
+      } else {
+        HalfType = EVT::getVectorVT(
+            *DAG.getContext(),
+            EVT(VT.getScalarType()).getHalfSizedIntegerVT(*DAG.getContext()),
+            VT.getVectorNumElements());
+      }
+      if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
+                             HalfType, DAG, false)) {
+        for (unsigned i = 0; i < 2; ++i) {
+          SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
+          SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
+          SDValue Shift = DAG.getConstant(
+              HalfType.getSizeInBits(), dl,
+              TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
+          Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
+          Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
+        }
+        break;
+      }
+      break;
+    }
+    }
+    break;
+  }
   case ISD::MUL: {
     EVT VT = Node->getValueType(0);
     SDVTList VTs = DAG.getVTList(VT, VT);
@@ -3347,7 +3417,7 @@
         TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
         TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
         TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
-        TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) {
+        TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, true)) {
       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
       SDValue Shift =
@@ -4147,6 +4217,24 @@
     Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
     break;
   }
+  case ISD::UMUL_LOHI:
+  case ISD::SMUL_LOHI: {
+    // Promote to a multiply in a wider integer type.
+    unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
+                                                         : ISD::SIGN_EXTEND;
+    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
+    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
+    Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
+
+    auto &DL = DAG.getDataLayout();
+    unsigned OriginalSize = OVT.getScalarSizeInBits();
+    Tmp2 = DAG.getNode(
+        ISD::SRL, dl, NVT, Tmp1,
+        DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
+    break;
+  }
   case ISD::SELECT: {
     unsigned ExtOp, TruncOp;
     if (Node->getValueType(0).isVector() ||
Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2176,7 +2176,7 @@
   GetExpandedInteger(N->getOperand(0), LL, LH);
   GetExpandedInteger(N->getOperand(1), RL, RH);
 
-  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH))
+  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, true, LL, LH, RL, RH))
     return;
 
   // If nothing else, we can make a libcall.
Index: lib/CodeGen/SelectionDAG/TargetLowering.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3015,108 +3015,190 @@
 // Legalization Utilities
 //===----------------------------------------------------------------------===//
 
-bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
-                               SelectionDAG &DAG, SDValue LL, SDValue LH,
-                               SDValue RL, SDValue RH) const {
-  EVT VT = N->getValueType(0);
-  SDLoc dl(N);
+bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
+                                    SDValue LHS, SDValue RHS,
+                                    SmallVectorImpl<SDValue> &Result,
+                                    EVT HalfVT, SelectionDAG &DAG,
+                                    bool OnlyLegalOrCustom, SDValue LL,
+                                    SDValue LH, SDValue RL, SDValue RH) const {
+  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
+         Opcode == ISD::SMUL_LOHI);
+
+  bool HasMULHS =
+      !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::MULHS, HalfVT);
+  bool HasMULHU =
+      !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::MULHU, HalfVT);
+  bool HasSMUL_LOHI =
+      !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::SMUL_LOHI, HalfVT);
+  bool HasUMUL_LOHI =
+      !OnlyLegalOrCustom || isOperationLegalOrCustom(ISD::UMUL_LOHI, HalfVT);
+  unsigned OuterBitSize = VT.getSizeInBits();
+  unsigned InnerBitSize = HalfVT.getSizeInBits();
+  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
+  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
+
+  // LL, LH, RL, and RH must be either all NULL or all set to a value.
+  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
+         (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
+
+  if (!HasMULHS && !HasMULHU && !HasSMUL_LOHI && !HasUMUL_LOHI)
+    return false;
 
-  bool HasMULHS = isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
-  bool HasMULHU = isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
-  bool HasSMUL_LOHI = isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
-  bool HasUMUL_LOHI = isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
-  if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
-    unsigned OuterBitSize = VT.getSizeInBits();
-    unsigned InnerBitSize = HiLoVT.getSizeInBits();
-    unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0));
-    unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1));
-
-    // LL, LH, RL, and RH must be either all NULL or all set to a value.
-    assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
-           (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
-
-    if (!LL.getNode() && !RL.getNode() &&
-        isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
-      LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, N->getOperand(0));
-      RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, N->getOperand(1));
+  SDVTList VTs = DAG.getVTList(HalfVT, HalfVT);
+  auto MakeUMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo,
+                           SDValue &Hi) -> bool {
+    if (HasUMUL_LOHI) {
+      Lo = DAG.getNode(ISD::UMUL_LOHI, dl, VTs, L, R);
+      Hi = SDValue(Lo.getNode(), 1);
+      return true;
     }
-
-    if (!LL.getNode())
-      return false;
-
-    APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
-    if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) &&
-        DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) {
-      // The inputs are both zero-extended.
-      if (HasUMUL_LOHI) {
-        // We can emit a umul_lohi.
-        Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(HiLoVT, HiLoVT), LL,
-                         RL);
-        Hi = SDValue(Lo.getNode(), 1);
-        return true;
-      }
-      if (HasMULHU) {
-        // We can emit a mulhu+mul.
-        Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
-        Hi = DAG.getNode(ISD::MULHU, dl, HiLoVT, LL, RL);
-        return true;
-      }
+    if (HasMULHU) {
+      Lo = DAG.getNode(ISD::MUL, dl, HalfVT, L, R);
+      Hi = DAG.getNode(ISD::MULHU, dl, HalfVT, L, R);
+      return true;
     }
-    if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) {
-      // The input values are both sign-extended.
-      if (HasSMUL_LOHI) {
-        // We can emit a smul_lohi.
-        Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(HiLoVT, HiLoVT), LL,
-                         RL);
-        Hi = SDValue(Lo.getNode(), 1);
-        return true;
-      }
-      if (HasMULHS) {
-        // We can emit a mulhs+mul.
-        Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
-        Hi = DAG.getNode(ISD::MULHS, dl, HiLoVT, LL, RL);
-        return true;
-      }
+    return false;
+  };
+  auto MakeSMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo,
+                           SDValue &Hi) -> bool {
+    if (HasSMUL_LOHI) {
+      Lo = DAG.getNode(ISD::SMUL_LOHI, dl, VTs, L, R);
+      Hi = SDValue(Lo.getNode(), 1);
+      return true;
     }
-
-    if (!LH.getNode() && !RH.getNode() &&
-        isOperationLegalOrCustom(ISD::SRL, VT) &&
-        isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
-      auto &DL = DAG.getDataLayout();
-      unsigned ShiftAmt = VT.getSizeInBits() - HiLoVT.getSizeInBits();
-      SDValue Shift = DAG.getConstant(ShiftAmt, dl, getShiftAmountTy(VT, DL));
-      LH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(0), Shift);
-      LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
-      RH = DAG.getNode(ISD::SRL, dl, VT, N->getOperand(1), Shift);
-      RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
+    if (HasMULHS) {
+      Lo = DAG.getNode(ISD::MUL, dl, HalfVT, L, R);
+      Hi = DAG.getNode(ISD::MULHS, dl, HalfVT, L, R);
+      return true;
     }
+    return false;
+  };
 
-    if (!LH.getNode())
-      return false;
+  SDValue Lo, Hi;
 
-    if (HasUMUL_LOHI) {
-      // Lo,Hi = umul LHS, RHS.
-      SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl,
-                                     DAG.getVTList(HiLoVT, HiLoVT), LL, RL);
-      Lo = UMulLOHI;
-      Hi = UMulLOHI.getValue(1);
-      RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
-      LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
-      Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
-      Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
+  if (!LL.getNode() && !RL.getNode() &&
+      isOperationLegalOrCustom(ISD::TRUNCATE, HalfVT)) {
+    LL = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, LHS);
+    RL = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, RHS);
+  }
+
+  if (!LL.getNode())
+    return false;
+
+  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
+  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
+      DAG.MaskedValueIsZero(RHS, HighMask)) {
+    // The inputs are both zero-extended.
+    if (MakeUMUL_LOHI(LL, RL, Lo, Hi)) {
+      Result.push_back(Lo);
+      Result.push_back(Hi);
+      if (Opcode != ISD::MUL) {
+        SDValue Zero = DAG.getConstant(0, dl, HalfVT);
+        Result.push_back(Zero);
+        Result.push_back(Zero);
+      }
       return true;
     }
-    if (HasMULHU) {
-      Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RL);
-      Hi = DAG.getNode(ISD::MULHU, dl, HiLoVT, LL, RL);
-      RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
-      LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
-      Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
-      Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
+  }
+
+  if (LHSSB > InnerBitSize && RHSSB > InnerBitSize && Opcode == ISD::MUL) {
+    // The input values are both sign-extended.
+    // TODO non-MUL case?
+    if (MakeSMUL_LOHI(LL, RL, Lo, Hi)) {
+      Result.push_back(Lo);
+      Result.push_back(Hi);
       return true;
     }
   }
-  return false;
+
+  if (!LH.getNode() && !RH.getNode() &&
+      isOperationLegalOrCustom(ISD::SRL, VT) &&
+      isOperationLegalOrCustom(ISD::TRUNCATE, HalfVT)) {
+    auto &DL = DAG.getDataLayout();
+    unsigned ShiftAmt = VT.getSizeInBits() - HalfVT.getSizeInBits();
+    SDValue Shift = DAG.getConstant(ShiftAmt, dl, getShiftAmountTy(VT, DL));
+    LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
+    LH = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, LH);
+    RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
+    RH = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, RH);
+  }
+
+  if (!LH.getNode())
+    return false;
+
+  if (!MakeUMUL_LOHI(LL, RL, Lo, Hi))
+    return false;
+
+  Result.push_back(Lo);
+
+  if (Opcode == ISD::MUL) {
+    RH = DAG.getNode(ISD::MUL, dl, HalfVT, LL, RH);
+    LH = DAG.getNode(ISD::MUL, dl, HalfVT, LH, RL);
+    Hi = DAG.getNode(ISD::ADD, dl, HalfVT, Hi, RH);
+    Hi = DAG.getNode(ISD::ADD, dl, HalfVT, Hi, LH);
+    Result.push_back(Hi);
+    return true;
+  }
+
+  SDValue Next = Hi;
+  if (!MakeUMUL_LOHI(LL, RH, Lo, Hi))
+    return false;
+
+  SDVTList CarryVTs = DAG.getVTList(HalfVT, MVT::Glue);
+  SDValue SumLo, SumHi;
+  SumHi = Hi;
+  SumLo = DAG.getNode(ISD::ADDC, dl, CarryVTs, Next, Lo);
+
+  if (!MakeUMUL_LOHI(LH, RL, Lo, Hi))
+    return false;
+
+  SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Hi, SumLo.getValue(1));
+  SumLo = DAG.getNode(ISD::ADDC, dl, CarryVTs, SumLo, Lo);
+  Result.push_back(SumLo);
+
+  if (!(Opcode == ISD::UMUL_LOHI ? MakeUMUL_LOHI(LH, RH, Lo, Hi)
+                                 : MakeSMUL_LOHI(LH, RH, Lo, Hi)))
+    return false;
+
+  SDValue Zero = DAG.getConstant(0, dl, HalfVT);
+  SumLo = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Lo, SumLo.getValue(1));
+  SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, Hi, Zero, SumHi.getValue(1));
+  SumHi = DAG.getNode(ISD::ADDE, dl, CarryVTs, SumHi, Zero, SumLo.getValue(1));
+
+  if (Opcode == ISD::SMUL_LOHI) {
+    SDValue LoSub = DAG.getNode(ISD::SUBC, dl, CarryVTs, SumLo, RL);
+    SDValue HiSub =
+        DAG.getNode(ISD::SUBE, dl, CarryVTs, SumHi, Zero, LoSub.getValue(1));
+
+    SumLo = DAG.getSelectCC(dl, LH, Zero, LoSub, SumLo, ISD::SETLT);
+    SumHi = DAG.getSelectCC(dl, LH, Zero, HiSub, SumHi, ISD::SETLT);
+
+    LoSub = DAG.getNode(ISD::SUBC, dl, CarryVTs, SumLo, LL);
+    HiSub =
+        DAG.getNode(ISD::SUBE, dl, CarryVTs, SumHi, Zero, LoSub.getValue(1));
+
+    SumLo = DAG.getSelectCC(dl, RH, Zero, LoSub, SumLo, ISD::SETLT);
+    SumHi = DAG.getSelectCC(dl, RH, Zero, HiSub, SumHi, ISD::SETLT);
+  }
+
+  Result.push_back(SumLo);
+  Result.push_back(SumHi);
+  return true;
+}
+
+bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
+                               SelectionDAG &DAG, bool OnlyLegalOrCustom,
+                               SDValue LL, SDValue LH, SDValue RL,
+                               SDValue RH) const {
+  SmallVector<SDValue, 2> Result;
+  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
+                           N->getOperand(0), N->getOperand(1), Result, HiLoVT,
+                           DAG, OnlyLegalOrCustom, LL, LH, RL, RH);
+  if (Result.size() >= 2) {
+    Lo = Result[0];
+    Hi = Result[1];
+  }
+  return Ok;
 }
 
 bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
Index: lib/Target/AMDGPU/AMDGPUISelLowering.h
===================================================================
--- lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -121,6 +121,8 @@
 
   bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
 
+  MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
   MVT getVectorIdxTy(const DataLayout &) const override;
   bool isSelectSupported(SelectSupportKind) const override;
 
Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp
===================================================================
--- lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -622,6 +622,13 @@
   return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
 }
 
+TargetLowering::MulExpansion
+AMDGPUTargetLowering::getMulExpansion(unsigned Opcode, MVT VT) const {
+  if (VT.getSizeInBits() > 32)
+    return MulExpansion::HalfWidth;
+  return MulExpansion::Split;
+}
+
 //===---------------------------------------------------------------------===//
 // TargetLowering Callbacks
 //===---------------------------------------------------------------------===//
Index: lib/Target/BPF/BPFISelLowering.h
===================================================================
--- lib/Target/BPF/BPFISelLowering.h
+++ lib/Target/BPF/BPFISelLowering.h
@@ -46,6 +46,8 @@
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *BB) const override;
 
+  MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
 private:
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
Index: lib/Target/BPF/BPFISelLowering.cpp
===================================================================
--- lib/Target/BPF/BPFISelLowering.cpp
+++ lib/Target/BPF/BPFISelLowering.cpp
@@ -132,6 +132,11 @@
   MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 128;
 }
 
+TargetLowering::MulExpansion BPFTargetLowering::getMulExpansion(unsigned Opcode,
+                                                                MVT VT) const {
+  return MulExpansion::HalfWidth;
+}
+
 SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   case ISD::BR_CC:
Index: lib/Target/PowerPC/PPCISelLowering.h
===================================================================
--- lib/Target/PowerPC/PPCISelLowering.h
+++ lib/Target/PowerPC/PPCISelLowering.h
@@ -567,6 +567,8 @@
     SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                           std::vector<SDNode *> *Created) const override;
 
+    MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
     unsigned getRegisterByName(const char* RegName, EVT VT,
                                SelectionDAG &DAG) const override;
 
Index: lib/Target/PowerPC/PPCISelLowering.cpp
===================================================================
--- lib/Target/PowerPC/PPCISelLowering.cpp
+++ lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11435,6 +11435,13 @@
   return Op;
 }
 
+TargetLowering::MulExpansion PPCTargetLowering::getMulExpansion(unsigned Opcode,
+                                                                MVT VT) const {
+  if (VT.isVector())
+    return MulExpansion::Scalarize;
+  return MulExpansion::Split;
+}
+
 //===----------------------------------------------------------------------===//
 // Inline Assembly Support
 //===----------------------------------------------------------------------===//
Index: lib/Target/Sparc/SparcISelLowering.h
===================================================================
--- lib/Target/Sparc/SparcISelLowering.h
+++ lib/Target/Sparc/SparcISelLowering.h
@@ -205,6 +205,8 @@
       return true;
     }
 
+    MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
     AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
 
     void ReplaceNodeResults(SDNode *N,
Index: lib/Target/Sparc/SparcISelLowering.cpp
===================================================================
--- lib/Target/Sparc/SparcISelLowering.cpp
+++ lib/Target/Sparc/SparcISelLowering.cpp
@@ -1394,6 +1394,13 @@
 // TargetLowering Implementation
 //===----------------------------------------------------------------------===//
 
+TargetLowering::MulExpansion
+SparcTargetLowering::getMulExpansion(unsigned Opcode, MVT VT) const {
+  if (VT.getSizeInBits() > 32)
+    return MulExpansion::HalfWidth;
+  return MulExpansion::Split;
+}
+
 TargetLowering::AtomicExpansionKind SparcTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   if (AI->getOperation() == AtomicRMWInst::Xchg &&
       AI->getType()->getPrimitiveSizeInBits() == 32)
@@ -1685,11 +1692,11 @@
   setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
   setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
 
-  // FIXME: Sparc provides these multiplies, but we don't have them yet.
-  setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
-  setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
-
   if (Subtarget->is64Bit()) {
+    // FIXME: Sparc provides these multiplies, but we don't have them yet.
+    setOperationAction(ISD::UMUL_LOHI, MVT::i32, Promote);
+    setOperationAction(ISD::SMUL_LOHI, MVT::i32, Promote);
+
     setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
     setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
     setOperationAction(ISD::MULHU,     MVT::i64, Expand);
@@ -1701,6 +1708,10 @@
     setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
     setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
     setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
+  } else {
+    // FIXME: Sparc provides these multiplies, but we don't have them yet.
+    setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
+    setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
   }
 
   // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
Index: lib/Target/X86/X86ISelLowering.h
===================================================================
--- lib/Target/X86/X86ISelLowering.h
+++ lib/Target/X86/X86ISelLowering.h
@@ -1026,6 +1026,8 @@
 
     bool isIntDivCheap(EVT VT, AttributeSet Attr) const override;
 
+    MulExpansion getMulExpansion(unsigned Opcode, MVT VT) const override;
+
     bool supportSwiftError() const override;
 
   protected:
Index: lib/Target/X86/X86ISelLowering.cpp
===================================================================
--- lib/Target/X86/X86ISelLowering.cpp
+++ lib/Target/X86/X86ISelLowering.cpp
@@ -32526,6 +32526,13 @@
   return OptSize && !VT.isVector();
 }
 
+TargetLowering::MulExpansion X86TargetLowering::getMulExpansion(unsigned Opcode,
+                                                                MVT VT) const {
+  if (VT.getScalarSizeInBits() >= 64)
+    return MulExpansion::Scalarize;
+  return MulExpansion::Split;
+}
+
 void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
   if (!Subtarget.is64Bit())
     return;