Index: llvm/docs/LangRef.rst
===================================================================
--- llvm/docs/LangRef.rst
+++ llvm/docs/LangRef.rst
@@ -13182,6 +13182,86 @@
%res = call i4 @llvm.umul.fix.i4(i4 15, i4 1, i32 1) ; %res = 7 (or 8) (7.5 x 0.5 = 3.75)
+Fixed Point Saturation Arithmetic Intrinsics
+---------------------------------
+
+These operations share the same qualities as Fixed Point Arithmetic and
+Saturation Arithmetic. The following intrinsics perform fixed point arithmeric
+on two operands of the same scale (specified with the third argument). The
+result of the operation is limited to a fixed range between a minimum and
+maximum value (see Saturation Arithmetic).
+
+
+'``llvm.smul.fix.sat.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.smul.fix.sat``
+on any integer bit width or vectors of integers.
+
+::
+
+ declare i16 @llvm.smul.fix.sat.i16(i16 %a, i16 %b, i32 %scale)
+ declare i32 @llvm.smul.fix.sat.i32(i32 %a, i32 %b, i32 %scale)
+ declare i64 @llvm.smul.fix.sat.i64(i64 %a, i64 %b, i32 %scale)
+ declare <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %a, <4 x i32> %b, i32 %scale)
+
+Overview
+"""""""""
+
+The '``llvm.smul.fix.sat``' family of intrinsic functions perform signed
+fixed point saturation multiplication on 2 arguments of the same scale.
+
+Arguments
+""""""""""
+
+The arguments (%a and %b) and the result may be of integer types of any bit
+width, but they must have the same bit width. ``%a`` and ``%b`` are the two
+values that will undergo signed fixed point multiplication. The argument
+``%scale`` represents the scale of both operands, and must be a constant
+integer.
+
+Semantics:
+""""""""""
+
+This operation performs fixed point multiplication on the 2 arguments of a
+specified scale. The result will also be returned in the same scale specified
+in the third argument.
+
+If the result value cannot be precisely represented in the given scale, the
+value is rounded up or down to the closest representable value. The rounding
+direction is unspecified.
+
+The maximum value this operation can clamp to is the largest signed value
+representable by the bit width of the first 2 arguments. The minimum value is the
+smallest signed value representable by this bit width.
+
+
+Examples
+"""""""""
+
+.. code-block:: llvm
+
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 3, i4 2, i32 0) ; %res = 6 (2 x 3 = 6)
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 3, i4 2, i32 1) ; %res = 3 (1.5 x 1 = 1.5)
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 3, i4 -2, i32 1) ; %res = -3 (1.5 x -1 = -1.5)
+
+ ; The result in the following could be rounded up to -2 or down to -2.5
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 3, i4 -3, i32 1) ; %res = -5 (or -4) (1.5 x -1.5 = -2.25)
+
+ ; Saturation
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 7, i4 2, i32 0) ; %res = 7
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 7, i4 2, i32 2) ; %res = 7
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 -8, i4 2, i32 2) ; %res = -8
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 -8, i4 -2, i32 2) ; %res = 7
+
+ ; Scale can affect the saturation result
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 2, i4 4, i32 0) ; %res = 7 (2 x 4 -> clamped to 7)
+ %res = call i4 @llvm.smul.fix.sat.i4(i4 2, i4 4, i32 1) ; %res = 4 (1 x 2 = 2)
+
+
Specialised Arithmetic Intrinsics
---------------------------------
Index: llvm/include/llvm/CodeGen/ISDOpcodes.h
===================================================================
--- llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -278,6 +278,11 @@
/// multiplication on 2 integers.
SMULFIX, UMULFIX,
+ /// Same as the corresponding unsaturated fixed point instructions, but the
+ /// result is clamped between the min and max values representable by the
+ /// bits of the first 2 operands.
+ SMULFIXSAT,
+
/// Simple binary floating point operators.
FADD, FSUB, FMUL, FDIV, FREM,
Index: llvm/include/llvm/CodeGen/TargetLowering.h
===================================================================
--- llvm/include/llvm/CodeGen/TargetLowering.h
+++ llvm/include/llvm/CodeGen/TargetLowering.h
@@ -836,6 +836,7 @@
default:
llvm_unreachable("Unexpected fixed point operation.");
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX:
Supported = isSupportedFixedPointOperation(Op, VT, Scale);
break;
Index: llvm/include/llvm/IR/Intrinsics.td
===================================================================
--- llvm/include/llvm/IR/Intrinsics.td
+++ llvm/include/llvm/IR/Intrinsics.td
@@ -849,11 +849,16 @@
def int_smul_fix : Intrinsic<[llvm_anyint_ty],
[LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
[IntrNoMem, IntrSpeculatable, Commutative]>;
-
def int_umul_fix : Intrinsic<[llvm_anyint_ty],
[LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
[IntrNoMem, IntrSpeculatable, Commutative]>;
+//===------------------- Fixed Point Saturation Arithmetic Intrinsics ----------------===//
+//
+def int_smul_fix_sat : Intrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
+ [IntrNoMem, IntrSpeculatable, Commutative]>;
+
//===------------------------- Memory Use Markers -------------------------===//
//
def int_lifetime_start : Intrinsic<[],
Index: llvm/include/llvm/Target/TargetSelectionDAG.td
===================================================================
--- llvm/include/llvm/Target/TargetSelectionDAG.td
+++ llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -391,6 +391,7 @@
def usubsat : SDNode<"ISD::USUBSAT" , SDTIntBinOp>;
def smulfix : SDNode<"ISD::SMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>;
+def smulfixsat : SDNode<"ISD::SMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>;
def umulfix : SDNode<"ISD::UMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>;
def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>;
Index: llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1128,6 +1128,7 @@
break;
}
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX: {
unsigned Scale = Node->getConstantOperandVal(2);
Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
@@ -3291,6 +3292,7 @@
Results.push_back(TLI.expandAddSubSat(Node, DAG));
break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX:
Results.push_back(TLI.expandFixedPointMul(Node, DAG));
break;
Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -149,6 +149,7 @@
case ISD::SSUBSAT:
case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
case ISD::ATOMIC_LOAD:
@@ -650,14 +651,35 @@
// Can just promote the operands then continue with operation.
SDLoc dl(N);
SDValue Op1Promoted, Op2Promoted;
- if (N->getOpcode() == ISD::SMULFIX) {
+ bool Signed =
+ N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
+ if (Signed) {
Op1Promoted = SExtPromotedInteger(N->getOperand(0));
Op2Promoted = SExtPromotedInteger(N->getOperand(1));
} else {
Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
}
+ EVT OldType = N->getOperand(0).getValueType();
EVT PromotedType = Op1Promoted.getValueType();
+ unsigned DiffSize =
+ PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
+
+ bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
+ if (Saturating) {
+ // Promoting the operand and result values changes the saturation width,
+ // which is extends the values that we clamp to on saturation. This could be
+ // resolved by shifting one of the operands the same amount, which would
+ // also shift the result we compare against, then shifting back.
+ EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
+ Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
+ DAG.getConstant(DiffSize, dl, ShiftTy));
+ SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
+ Op2Promoted, N->getOperand(2));
+ unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
+ return DAG.getNode(ShiftOp, dl, PromotedType, Result,
+ DAG.getConstant(DiffSize, dl, ShiftTy));
+ }
return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
N->getOperand(2));
}
@@ -1098,6 +1120,7 @@
case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
@@ -1628,7 +1651,9 @@
case ISD::UADDSAT:
case ISD::SSUBSAT:
case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
+
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
}
@@ -2599,17 +2624,33 @@
void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
SDValue &Hi) {
- assert(
- (N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::UMULFIX) &&
- "Expected operand to be signed or unsigned fixed point multiplication");
-
SDLoc dl(N);
EVT VT = N->getValueType(0);
+ unsigned VTSize = VT.getScalarSizeInBits();
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
uint64_t Scale = N->getConstantOperandVal(2);
+ bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
+ EVT BoolVT =
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue Zero = DAG.getConstant(0, dl, VT);
if (!Scale) {
- SDValue Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+ SDValue Result;
+ if (!Saturating) {
+ Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+ } else {
+ Result = DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
+ SDValue Product = Result.getValue(0);
+ SDValue Overflow = Result.getValue(1);
+
+ APInt MinVal = APInt::getSignedMinValue(VTSize);
+ APInt MaxVal = APInt::getSignedMaxValue(VTSize);
+ SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
+ SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
+ SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
+ Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
+ Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
+ }
SplitInteger(Result, Lo, Hi);
return;
}
@@ -2620,7 +2661,8 @@
GetExpandedInteger(RHS, RL, RH);
SmallVector<SDValue, 4> Result;
- bool Signed = N->getOpcode() == ISD::SMULFIX;
+ bool Signed = (N->getOpcode() == ISD::SMULFIX ||
+ N->getOpcode() == ISD::SMULFIXSAT);
unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
@@ -2629,8 +2671,9 @@
return;
}
- unsigned VTSize = VT.getScalarSizeInBits();
unsigned NVTSize = NVT.getScalarSizeInBits();
+ assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
+ "the size of the current value type");
EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
// Shift whole amount by scale.
@@ -2639,6 +2682,12 @@
SDValue ResultHL = Result[2];
SDValue ResultHH = Result[3];
+ SDValue SatMax, SatMin;
+ SDValue NVTZero = DAG.getConstant(0, dl, NVT);
+ SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
+ EVT BoolNVT =
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), NVT);
+
// After getting the multplication result in 4 parts, we need to perform a
// shift right by the amount of the scale to get the result in that scale.
// Let's say we multiply 2 64 bit numbers. The resulting value can be held in
@@ -2667,11 +2716,60 @@
Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
+
+ // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
+ // highest bit of HH determines saturation direction in the event of
+ // saturation.
+ // The number of overflow bits we can check are VTSize - Scale + 1 (we
+ // include the sign bit). If these top bits are > 0, then we overflowed past
+ // the max value. If these top bits are < -1, then we overflowed past the
+ // min value. Otherwise, we did not overflow.
+ if (Saturating) {
+ unsigned OverflowBits = VTSize - Scale + 1;
+ assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
+ "Extent of overflow bits must start within HL");
+ SDValue HLHiMask = DAG.getConstant(
+ APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
+ SDValue HLLoMask = DAG.getConstant(
+ APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
+
+ // HH > 0 or HH == 0 && HL > HLLoMask
+ SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
+ SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
+ SDValue HLPos =
+ DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
+ SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
+ DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLPos));
+
+ // HH < -1 or HH == -1 && HL < HLHiMask
+ SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
+ SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
+ SDValue HLNeg =
+ DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
+ SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
+ DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLNeg));
+ }
} else if (Scale == NVTSize) {
// If the scales are equal, Lo and Hi are ResultLH and Result HL,
// respectively. Avoid shifting to prevent undefined behavior.
Lo = ResultLH;
Hi = ResultHL;
+
+ // We overflow max if HH > 0 or HH == 0 && HL sign is negative.
+ // We overflow min if HH < -1 or HH == -1 && HL sign is 0.
+ if (Saturating) {
+ SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
+ SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
+ SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
+ SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
+ DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLNeg));
+
+ SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
+ SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
+ SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGT);
+ SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
+ DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLPos));
+ }
} else if (Scale < VTSize) {
// If the scale is instead less than the old VT size, but greater than or
// equal to the expanded VT size, the first part of the result (ResultLL) is
@@ -2686,6 +2784,19 @@
Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
+
+ // This is similar to the case when we saturate if Scale < NVTSize, but we
+ // only need to chech HH.
+ if (Saturating) {
+ unsigned OverflowBits = VTSize - Scale + 1;
+ SDValue HHHiMask = DAG.getConstant(
+ APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
+ SDValue HHLoMask = DAG.getConstant(
+ APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
+
+ SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
+ SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
+ }
} else if (Scale == VTSize) {
assert(
!Signed &&
@@ -2697,6 +2808,16 @@
llvm_unreachable("Expected the scale to be less than or equal to the width "
"of the operands");
}
+
+ if (Saturating) {
+ APInt LHMax = APInt::getSignedMaxValue(NVTSize);
+ APInt LLMax = APInt::getAllOnesValue(NVTSize);
+ APInt LHMin = APInt::getSignedMinValue(NVTSize);
+ Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LHMax, dl, NVT), Hi);
+ Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(LHMin, dl, NVT), Hi);
+ Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LLMax, dl, NVT), Lo);
+ Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
+ }
}
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
Index: llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -426,6 +426,7 @@
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX: {
unsigned Scale = Node->getConstantOperandVal(2);
Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
Index: llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -172,6 +172,7 @@
R = ScalarizeVecRes_StrictFPOp(N);
break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX:
R = ScalarizeVecRes_MULFIX(N);
break;
@@ -860,6 +861,7 @@
SplitVecRes_StrictFPOp(N, Lo, Hi);
break;
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX:
SplitVecRes_MULFIX(N, Lo, Hi);
break;
Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5900,6 +5900,14 @@
Op1.getValueType(), Op1, Op2, Op3));
return nullptr;
}
+ case Intrinsic::smul_fix_sat: {
+ SDValue Op1 = getValue(I.getArgOperand(0));
+ SDValue Op2 = getValue(I.getArgOperand(1));
+ SDValue Op3 = getValue(I.getArgOperand(2));
+ setValue(&I, DAG.getNode(ISD::SMULFIXSAT, sdl, Op1.getValueType(), Op1, Op2,
+ Op3));
+ return nullptr;
+ }
case Intrinsic::stacksave: {
SDValue Op = getRoot();
Res = DAG.getNode(
Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -299,7 +299,9 @@
case ISD::UADDSAT: return "uaddsat";
case ISD::SSUBSAT: return "ssubsat";
case ISD::USUBSAT: return "usubsat";
+
case ISD::SMULFIX: return "smulfix";
+ case ISD::SMULFIXSAT: return "smulfixsat";
case ISD::UMULFIX: return "umulfix";
// Conversion operators.
Index: llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5416,25 +5416,43 @@
SDValue
TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
assert((Node->getOpcode() == ISD::SMULFIX ||
- Node->getOpcode() == ISD::UMULFIX) &&
- "Expected opcode to be SMULFIX or UMULFIX.");
+ Node->getOpcode() == ISD::UMULFIX ||
+ Node->getOpcode() == ISD::SMULFIXSAT) &&
+ "Expected a fixed point multiplication opcode");
SDLoc dl(Node);
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
unsigned Scale = Node->getConstantOperandVal(2);
+ bool Saturating = Node->getOpcode() == ISD::SMULFIXSAT;
+ EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ unsigned VTSize = VT.getScalarSizeInBits();
- // [us]mul.fix(a, b, 0) -> mul(a, b)
if (!Scale) {
- if (VT.isVector() && !isOperationLegalOrCustom(ISD::MUL, VT))
- return SDValue();
- return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+ // [us]mul.fix(a, b, 0) -> mul(a, b)
+ if (!Saturating && isOperationLegalOrCustom(ISD::MUL, VT)) {
+ return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+ } else if (Saturating && isOperationLegalOrCustom(ISD::SMULO, VT)) {
+ SDValue Result =
+ DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
+ SDValue Product = Result.getValue(0);
+ SDValue Overflow = Result.getValue(1);
+ SDValue Zero = DAG.getConstant(0, dl, VT);
+
+ APInt MinVal = APInt::getSignedMinValue(VTSize);
+ APInt MaxVal = APInt::getSignedMaxValue(VTSize);
+ SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
+ SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
+ SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
+ Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
+ return DAG.getSelect(dl, VT, Overflow, Result, Product);
+ }
+ return SDValue();
}
- unsigned VTSize = VT.getScalarSizeInBits();
- bool Signed = Node->getOpcode() == ISD::SMULFIX;
-
+ bool Signed =
+ Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::SMULFIXSAT;
assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
"Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.");
@@ -5467,9 +5485,27 @@
// are scaled. The result is given to us in 2 halves, so we only want part of
// both in the result.
EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
- Lo = DAG.getNode(ISD::SRL, dl, VT, Lo, DAG.getConstant(Scale, dl, ShiftTy));
- Hi = DAG.getNode(
+ SDValue ShiftedLo =
+ DAG.getNode(ISD::SRL, dl, VT, Lo, DAG.getConstant(Scale, dl, ShiftTy));
+ SDValue ShiftedHi = DAG.getNode(
ISD::SHL, dl, VT, Hi,
DAG.getConstant(VT.getScalarSizeInBits() - Scale, dl, ShiftTy));
- return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
+ SDValue Result = DAG.getNode(ISD::OR, dl, VT, ShiftedLo, ShiftedHi);
+
+ if (!Saturating)
+ return Result;
+
+ unsigned OverflowBits = VTSize - Scale + 1; // +1 for the sign
+ SDValue HiMask =
+ DAG.getConstant(APInt::getHighBitsSet(VTSize, OverflowBits), dl, VT);
+ SDValue LoMask = DAG.getConstant(
+ APInt::getLowBitsSet(VTSize, VTSize - OverflowBits), dl, VT);
+ SDValue SatMax = DAG.getSetCC(dl, BoolVT, Hi, LoMask, ISD::SETGT);
+ SDValue SatMin = DAG.getSetCC(dl, BoolVT, Hi, HiMask, ISD::SETLT);
+
+ APInt MaxVal = APInt::getSignedMaxValue(VTSize);
+ APInt MinVal = APInt::getSignedMinValue(VTSize);
+ Result =
+ DAG.getSelect(dl, VT, SatMax, DAG.getConstant(MaxVal, dl, VT), Result);
+ return DAG.getSelect(dl, VT, SatMin, DAG.getConstant(MinVal, dl, VT), Result);
}
Index: llvm/lib/CodeGen/TargetLoweringBase.cpp
===================================================================
--- llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -624,6 +624,7 @@
setOperationAction(ISD::SSUBSAT, VT, Expand);
setOperationAction(ISD::USUBSAT, VT, Expand);
setOperationAction(ISD::SMULFIX, VT, Expand);
+ setOperationAction(ISD::SMULFIXSAT, VT, Expand);
setOperationAction(ISD::UMULFIX, VT, Expand);
// Overflow operations default to expand
Index: llvm/lib/IR/Verifier.cpp
===================================================================
--- llvm/lib/IR/Verifier.cpp
+++ llvm/lib/IR/Verifier.cpp
@@ -4575,28 +4575,29 @@
break;
}
case Intrinsic::smul_fix:
+ case Intrinsic::smul_fix_sat:
case Intrinsic::umul_fix: {
Value *Op1 = Call.getArgOperand(0);
Value *Op2 = Call.getArgOperand(1);
Assert(Op1->getType()->isIntOrIntVectorTy(),
- "first operand of [us]mul_fix must be an int type or vector "
+ "first operand of [us]mul_fix[_sat] must be an int type or vector "
"of ints");
Assert(Op2->getType()->isIntOrIntVectorTy(),
- "second operand of [us]mul_fix must be an int type or vector "
+ "second operand of [us]mul_fix_[sat] must be an int type or vector "
"of ints");
auto *Op3 = dyn_cast<ConstantInt>(Call.getArgOperand(2));
- Assert(Op3, "third argument of [us]mul_fix must be a constant integer");
+ Assert(Op3, "third argument of [us]mul_fix[_sat] must be a constant integer");
Assert(Op3->getType()->getBitWidth() <= 32,
- "third argument of [us]mul_fix must fit within 32 bits");
+ "third argument of [us]mul_fix[_sat] must fit within 32 bits");
- if (ID == Intrinsic::smul_fix) {
+ if (ID == Intrinsic::smul_fix || ID == Intrinsic::smul_fix_sat) {
Assert(
Op3->getZExtValue() < Op1->getType()->getScalarSizeInBits(),
- "the scale of smul_fix must be less than the width of the operands");
+ "the scale of smul_fix[_sat] must be less than the width of the operands");
} else {
Assert(Op3->getZExtValue() <= Op1->getType()->getScalarSizeInBits(),
- "the scale of umul_fix must be less than or equal to the width of "
+ "the scale of umul_fix[_sat] must be less than or equal to the width of "
"the operands");
}
break;
Index: llvm/test/CodeGen/X86/smul_fix_sat.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -0,0 +1,751 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare i4 @llvm.smul.fix.sat.i4 (i4, i4, i32)
+declare i32 @llvm.smul.fix.sat.i32 (i32, i32, i32)
+declare i64 @llvm.smul.fix.sat.i64 (i64, i64, i32)
+declare <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func:
+; X64: # %bb.0:
+; X64-NEXT: movslq %esi, %rax
+; X64-NEXT: movslq %edi, %rcx
+; X64-NEXT: imulq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: shrq $32, %rax
+; X64-NEXT: movl %eax, %edx
+; X64-NEXT: shldl $30, %ecx, %edx
+; X64-NEXT: cmpl $1, %eax
+; X64-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X64-NEXT: cmovlel %edx, %ecx
+; X64-NEXT: cmpl $-2, %eax
+; X64-NEXT: movl $-2147483648, %eax # imm = 0x80000000
+; X64-NEXT: cmovgel %ecx, %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: func:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: shrdl $2, %edx, %eax
+; X86-NEXT: cmpl $1, %edx
+; X86-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: cmpl $-2, %edx
+; X86-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: retl
+ %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2);
+ ret i32 %tmp;
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func2:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: imulq %rsi
+; X64-NEXT: shrdq $2, %rdx, %rax
+; X64-NEXT: cmpq $1, %rdx
+; X64-NEXT: movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: cmpq $-2, %rdx
+; X64-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; X64-NEXT: cmovlq %rcx, %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: func2:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: addl %ebx, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: imull %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: addl %ebp, %eax
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: addl %esi, %edx
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: cmovnsl %ecx, %edi
+; X86-NEXT: cmovnsl %edx, %esi
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edi, %ebp
+; X86-NEXT: sbbl $0, %ebp
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %edi, %ebp
+; X86-NEXT: cmovnsl %esi, %ecx
+; X86-NEXT: testl %ebp, %ebp
+; X86-NEXT: setg %bh
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: cmpl $1, %ecx
+; X86-NEXT: seta %bl
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shldl $30, %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $30, %esi, %eax
+; X86-NEXT: andb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
+; X86-NEXT: orb %bh, %bl
+; X86-NEXT: testb %bl, %bl
+; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movl $-1, %esi
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: cmpl $-1, %ebp
+; X86-NEXT: setl %bl
+; X86-NEXT: sete %bh
+; X86-NEXT: cmpl $-2, %ecx
+; X86-NEXT: setb %cl
+; X86-NEXT: andb %bh, %cl
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: orb %bl, %cl
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: addl $8, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2);
+ ret i64 %tmp;
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func3:
+; X64: # %bb.0:
+; X64-NEXT: shlb $4, %sil
+; X64-NEXT: sarb $4, %sil
+; X64-NEXT: shlb $4, %dil
+; X64-NEXT: movsbl %dil, %eax
+; X64-NEXT: movsbl %sil, %ecx
+; X64-NEXT: imull %eax, %ecx
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: shrl $8, %eax
+; X64-NEXT: cmpb $1, %al
+; X64-NEXT: movb $127, %dl
+; X64-NEXT: jg .LBB2_2
+; X64-NEXT: # %bb.1:
+; X64-NEXT: shrb $2, %cl
+; X64-NEXT: movl %eax, %edx
+; X64-NEXT: shlb $6, %dl
+; X64-NEXT: orb %cl, %dl
+; X64-NEXT: .LBB2_2:
+; X64-NEXT: cmpb $-2, %al
+; X64-NEXT: movb $-128, %al
+; X64-NEXT: jl .LBB2_4
+; X64-NEXT: # %bb.3:
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: .LBB2_4:
+; X64-NEXT: sarb $4, %al
+; X64-NEXT: retq
+;
+; X86-LABEL: func3:
+; X86: # %bb.0:
+; X86-NEXT: movb {{[0-9]+}}(%esp), %al
+; X86-NEXT: shlb $4, %al
+; X86-NEXT: sarb $4, %al
+; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT: shlb $4, %cl
+; X86-NEXT: movsbl %cl, %ecx
+; X86-NEXT: movsbl %al, %eax
+; X86-NEXT: imull %ecx, %eax
+; X86-NEXT: cmpb $1, %ah
+; X86-NEXT: movb $127, %cl
+; X86-NEXT: jg .LBB2_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movb %ah, %dl
+; X86-NEXT: shlb $6, %dl
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrb $2, %cl
+; X86-NEXT: orb %dl, %cl
+; X86-NEXT: .LBB2_2:
+; X86-NEXT: cmpb $-2, %ah
+; X86-NEXT: movb $-128, %al
+; X86-NEXT: jl .LBB2_4
+; X86-NEXT: # %bb.3:
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: .LBB2_4:
+; X86-NEXT: sarb $4, %al
+; X86-NEXT: retl
+ %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2);
+ ret i4 %tmp;
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec:
+; X64: # %bb.0:
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT: movd %xmm2, %eax
+; X64-NEXT: cltq
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT: movd %xmm2, %ecx
+; X64-NEXT: movslq %ecx, %rcx
+; X64-NEXT: imulq %rax, %rcx
+; X64-NEXT: movq %rcx, %rdx
+; X64-NEXT: shrq $32, %rdx
+; X64-NEXT: movl %edx, %esi
+; X64-NEXT: shldl $30, %ecx, %esi
+; X64-NEXT: cmpl $1, %edx
+; X64-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT: cmovgl %eax, %esi
+; X64-NEXT: cmpl $-2, %edx
+; X64-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
+; X64-NEXT: cmovll %ecx, %esi
+; X64-NEXT: movd %esi, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT: movd %xmm3, %edx
+; X64-NEXT: movslq %edx, %rdx
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT: movd %xmm3, %esi
+; X64-NEXT: movslq %esi, %rsi
+; X64-NEXT: imulq %rdx, %rsi
+; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: shrq $32, %rdx
+; X64-NEXT: movl %edx, %edi
+; X64-NEXT: shldl $30, %esi, %edi
+; X64-NEXT: cmpl $1, %edx
+; X64-NEXT: cmovgl %eax, %edi
+; X64-NEXT: cmpl $-2, %edx
+; X64-NEXT: cmovll %ecx, %edi
+; X64-NEXT: movd %edi, %xmm3
+; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT: movd %xmm1, %edx
+; X64-NEXT: movslq %edx, %rdx
+; X64-NEXT: movd %xmm0, %esi
+; X64-NEXT: movslq %esi, %rsi
+; X64-NEXT: imulq %rdx, %rsi
+; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: shrq $32, %rdx
+; X64-NEXT: movl %edx, %edi
+; X64-NEXT: shldl $30, %esi, %edi
+; X64-NEXT: cmpl $1, %edx
+; X64-NEXT: cmovgl %eax, %edi
+; X64-NEXT: cmpl $-2, %edx
+; X64-NEXT: cmovll %ecx, %edi
+; X64-NEXT: movd %edi, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT: movd %xmm1, %edx
+; X64-NEXT: movslq %edx, %rdx
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT: movd %xmm0, %esi
+; X64-NEXT: movslq %esi, %rsi
+; X64-NEXT: imulq %rdx, %rsi
+; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: shrq $32, %rdx
+; X64-NEXT: movl %edx, %edi
+; X64-NEXT: shldl $30, %esi, %edi
+; X64-NEXT: cmpl $1, %edx
+; X64-NEXT: cmovgl %eax, %edi
+; X64-NEXT: cmpl $-2, %edx
+; X64-NEXT: cmovll %ecx, %edi
+; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: retq
+;
+; X86-LABEL: vec:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrdl $2, %edx, %ecx
+; X86-NEXT: cmpl $1, %edx
+; X86-NEXT: movl $2147483647, %ebp # imm = 0x7FFFFFFF
+; X86-NEXT: cmovgl %ebp, %ecx
+; X86-NEXT: cmpl $-2, %edx
+; X86-NEXT: movl $-2147483648, %esi # imm = 0x80000000
+; X86-NEXT: cmovll %esi, %ecx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: shrdl $2, %edx, %edi
+; X86-NEXT: cmpl $1, %edx
+; X86-NEXT: cmovgl %ebp, %edi
+; X86-NEXT: cmpl $-2, %edx
+; X86-NEXT: cmovll %esi, %edi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shrdl $2, %edx, %ebx
+; X86-NEXT: cmpl $1, %edx
+; X86-NEXT: cmovgl %ebp, %ebx
+; X86-NEXT: cmpl $-2, %edx
+; X86-NEXT: cmovll %esi, %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: shrdl $2, %edx, %eax
+; X86-NEXT: cmpl $1, %edx
+; X86-NEXT: cmovgl %ebp, %eax
+; X86-NEXT: cmpl $-2, %edx
+; X86-NEXT: cmovll %esi, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %eax, 12(%edx)
+; X86-NEXT: movl %ebx, 8(%edx)
+; X86-NEXT: movl %edi, 4(%edx)
+; X86-NEXT: movl %ecx, (%edx)
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2);
+ ret <4 x i32> %tmp;
+}
+
+; These result in regular integer multiplication
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func4:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: imull %esi, %ecx
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: testl %ecx, %ecx
+; X64-NEXT: setns %al
+; X64-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT: imull %esi, %edi
+; X64-NEXT: cmovnol %edi, %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: func4:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: imull %edx, %esi
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: setns %cl
+; X86-NEXT: addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: imull %edx, %eax
+; X86-NEXT: cmovol %ecx, %eax
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
+ %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0);
+ ret i32 %tmp;
+}
+
+define i64 @func5(i64 %x, i64 %y) {
+; X64-LABEL: func5:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: imulq %rsi, %rax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testq %rax, %rax
+; X64-NEXT: setns %cl
+; X64-NEXT: movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: imulq %rsi, %rdi
+; X64-NEXT: cmovnoq %rdi, %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: func5:
+; X86: # %bb.0:
+; X86-NEXT: pushl %edi
+; X86-NEXT: .cfi_def_cfa_offset 8
+; X86-NEXT: pushl %esi
+; X86-NEXT: .cfi_def_cfa_offset 12
+; X86-NEXT: pushl %eax
+; X86-NEXT: .cfi_def_cfa_offset 16
+; X86-NEXT: .cfi_offset %esi, -12
+; X86-NEXT: .cfi_offset %edi, -8
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl $0, (%esp)
+; X86-NEXT: movl %esp, %edi
+; X86-NEXT: pushl %edi
+; X86-NEXT: .cfi_adjust_cfa_offset 4
+; X86-NEXT: pushl %esi
+; X86-NEXT: .cfi_adjust_cfa_offset 4
+; X86-NEXT: pushl %edx
+; X86-NEXT: .cfi_adjust_cfa_offset 4
+; X86-NEXT: pushl %ecx
+; X86-NEXT: .cfi_adjust_cfa_offset 4
+; X86-NEXT: pushl %eax
+; X86-NEXT: .cfi_adjust_cfa_offset 4
+; X86-NEXT: calll __mulodi4
+; X86-NEXT: addl $20, %esp
+; X86-NEXT: .cfi_adjust_cfa_offset -20
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: setns %cl
+; X86-NEXT: addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: cmpl $0, (%esp)
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: addl $4, %esp
+; X86-NEXT: .cfi_def_cfa_offset 12
+; X86-NEXT: popl %esi
+; X86-NEXT: .cfi_def_cfa_offset 8
+; X86-NEXT: popl %edi
+; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: retl
+ %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0);
+ ret i64 %tmp;
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func6:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shlb $4, %sil
+; X64-NEXT: sarb $4, %sil
+; X64-NEXT: shlb $4, %al
+; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: imulb %sil
+; X64-NEXT: setno %dl
+; X64-NEXT: testb %al, %al
+; X64-NEXT: setns %cl
+; X64-NEXT: testb %dl, %dl
+; X64-NEXT: jne .LBB6_2
+; X64-NEXT: # %bb.1:
+; X64-NEXT: addb $127, %cl
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: .LBB6_2:
+; X64-NEXT: sarb $4, %al
+; X64-NEXT: retq
+;
+; X86-LABEL: func6:
+; X86: # %bb.0:
+; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT: shlb $4, %cl
+; X86-NEXT: sarb $4, %cl
+; X86-NEXT: movb {{[0-9]+}}(%esp), %al
+; X86-NEXT: shlb $4, %al
+; X86-NEXT: imulb %cl
+; X86-NEXT: setno %dl
+; X86-NEXT: testb %al, %al
+; X86-NEXT: setns %cl
+; X86-NEXT: testb %dl, %dl
+; X86-NEXT: jne .LBB6_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: addb $127, %cl
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: .LBB6_2:
+; X86-NEXT: sarb $4, %al
+; X86-NEXT: retl
+ %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0);
+ ret i4 %tmp;
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec2:
+; X64: # %bb.0:
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; X64-NEXT: movd %xmm2, %ecx
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; X64-NEXT: movd %xmm2, %r8d
+; X64-NEXT: movl %r8d, %edx
+; X64-NEXT: imull %ecx, %edx
+; X64-NEXT: xorl %esi, %esi
+; X64-NEXT: testl %edx, %edx
+; X64-NEXT: setns %sil
+; X64-NEXT: addl $2147483647, %esi # imm = 0x7FFFFFFF
+; X64-NEXT: imull %ecx, %r8d
+; X64-NEXT: cmovol %esi, %r8d
+; X64-NEXT: movd %xmm1, %edx
+; X64-NEXT: movd %xmm0, %ecx
+; X64-NEXT: movl %ecx, %esi
+; X64-NEXT: imull %edx, %esi
+; X64-NEXT: xorl %edi, %edi
+; X64-NEXT: testl %esi, %esi
+; X64-NEXT: setns %dil
+; X64-NEXT: addl $2147483647, %edi # imm = 0x7FFFFFFF
+; X64-NEXT: imull %edx, %ecx
+; X64-NEXT: cmovol %edi, %ecx
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-NEXT: movd %xmm2, %edx
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-NEXT: movd %xmm2, %esi
+; X64-NEXT: movl %esi, %edi
+; X64-NEXT: imull %edx, %edi
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: testl %edi, %edi
+; X64-NEXT: setns %al
+; X64-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT: imull %edx, %esi
+; X64-NEXT: cmovol %eax, %esi
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; X64-NEXT: movd %xmm1, %r9d
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X64-NEXT: movd %xmm0, %edx
+; X64-NEXT: movl %edx, %edi
+; X64-NEXT: imull %r9d, %edi
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: testl %edi, %edi
+; X64-NEXT: setns %al
+; X64-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT: imull %r9d, %edx
+; X64-NEXT: cmovol %eax, %edx
+; X64-NEXT: movd %edx, %xmm0
+; X64-NEXT: movd %esi, %xmm1
+; X64-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: movd %r8d, %xmm2
+; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-NEXT: retq
+;
+; X86-LABEL: vec2:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: imull %edx, %esi
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: setns %al
+; X86-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT: imull %edx, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: cmovol %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: imull %esi, %edi
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: setns %al
+; X86-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT: imull %esi, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: cmovol %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: imull %edi, %ebx
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: setns %al
+; X86-NEXT: addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT: imull %edi, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: cmovol %eax, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %edi, %ebp
+; X86-NEXT: imull %eax, %ebp
+; X86-NEXT: xorl %ebx, %ebx
+; X86-NEXT: testl %ebp, %ebp
+; X86-NEXT: setns %bl
+; X86-NEXT: addl $2147483647, %ebx # imm = 0x7FFFFFFF
+; X86-NEXT: imull %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: cmovol %ebx, %edi
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0);
+ ret <4 x i32> %tmp;
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func7:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: imulq %rsi
+; X64-NEXT: shrdq $32, %rdx, %rax
+; X64-NEXT: cmpq $2147483647, %rdx # imm = 0x7FFFFFFF
+; X64-NEXT: movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: cmpq $-2147483648, %rdx # imm = 0x80000000
+; X64-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; X64-NEXT: cmovlq %rcx, %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: func7:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: addl %ebx, %eax
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl $0, %ebp
+; X86-NEXT: addl %ecx, %edx
+; X86-NEXT: adcl $0, %ebp
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ebp, %esi
+; X86-NEXT: sbbl $0, %esi
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %ebp, %esi
+; X86-NEXT: cmovnsl %edx, %ecx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %esi, %edi
+; X86-NEXT: cmovnsl %ecx, %edx
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: setg %cl
+; X86-NEXT: sets %ch
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: setg %bl
+; X86-NEXT: sete %bh
+; X86-NEXT: andb %ch, %bh
+; X86-NEXT: orb %bl, %bh
+; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movl $-1, %esi
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: cmpl $-1, %edi
+; X86-NEXT: setl %ch
+; X86-NEXT: sete %bl
+; X86-NEXT: andb %cl, %bl
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: orb %ch, %bl
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32);
+ ret i64 %tmp;
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func8:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: imulq %rsi
+; X64-NEXT: shrdq $63, %rdx, %rax
+; X64-NEXT: movabsq $4611686018427387903, %rcx # imm = 0x3FFFFFFFFFFFFFFF
+; X64-NEXT: cmpq %rcx, %rdx
+; X64-NEXT: movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: movabsq $-4611686018427387904, %rcx # imm = 0xC000000000000000
+; X64-NEXT: cmpq %rcx, %rdx
+; X64-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; X64-NEXT: cmovlq %rcx, %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: func8:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: addl %ebx, %ebp
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: imull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: addl %ebp, %eax
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: addl %ecx, %edx
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: sbbl $0, %esi
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %ebx, %esi
+; X86-NEXT: cmovnsl %edx, %ecx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: sbbl $0, %ebx
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %esi, %ebx
+; X86-NEXT: cmovnsl %ecx, %edi
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: shrdl $31, %edi, %eax
+; X86-NEXT: cmpl $1073741823, %ebx # imm = 0x3FFFFFFF
+; X86-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: cmovgl %ecx, %edx
+; X86-NEXT: movl $-1, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: cmpl $-1073741824, %ebx # imm = 0xC0000000
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
+; X86-NEXT: cmovll %ecx, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63);
+ ret i64 %tmp;
+}