diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -16212,6 +16212,120 @@ %a = load i16, i16* @x, align 2 %res = call float @llvm.convert.from.fp16(i16 %a) +Saturating floating-point to integer conversions +------------------------------------------------ + +The ``fptoui`` and ``fptosi`` instructions return a +:ref:`poison value ` if the rounded-towards-zero value is not +representable by the result type. These intrinsics provide an alternative +conversion, which will saturate towards the smallest and largest representable +integer values instead. + +'``llvm.fptoui.sat.*``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use ``llvm.fptoui.sat`` on any +floating-point argument type and any integer result type, or vectors thereof. +Not all targets may support all types, however. + +:: + + declare i32 @llvm.fptoui.sat.i32.f32(float %f) + declare i19 @llvm.fptoui.sat.i19.f64(double %f) + declare <4 x i100> @llvm.fptoui.sat.v4i100.v4f128(<4 x fp128> %f) + +Overview: +""""""""" + +This intrinsic converts the argument into an unsigned integer using saturating +semantics. + +Arguments: +"""""""""" + +The argument may be any floating-point or vector of floating-point type. The +return value may be any integer or vector of integer type. The number of vector +elements in argument and return must be the same. + +Semantics: +"""""""""" + +The conversion to integer is performed subject to the following rules: + +- If the argument is any NaN, zero is returned. +- If the argument is smaller than zero (this includes negative infinity), + zero is returned. +- If the argument is larger than the largest representable integer of the + result type (this includes positive infinity), the largest representable + integer is returned. +- Otherwise, the result of rounding the argument towards zero is returned. + +Example: +"""""""" + +.. code-block:: text + + %a = call i8 @llvm.fptoui.sat.i8.f32(float 123.9) ; yields i8: 123 + %b = call i8 @llvm.fptoui.sat.i8.f32(float -5.7) ; yields i8: 0 + %c = call i8 @llvm.fptoui.sat.i8.f32(float 377.0) ; yields i8: 255 + %d = call i8 @llvm.fptoui.sat.i8.f32(float 0xFFF8000000000000) ; yields i8: 0 + +'``llvm.fptosi.sat.*``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use ``llvm.fptosi.sat`` on any +floating-point argument type and any integer result type, or vectors thereof. +Not all targets may support all types, however. + +:: + + declare i32 @llvm.fptosi.sat.i32.f32(float %f) + declare i19 @llvm.fptosi.sat.i19.f64(double %f) + declare <4 x i100> @llvm.fptosi.sat.v4i100.v4f128(<4 x fp128> %f) + +Overview: +""""""""" + +This intrinsic converts the argument into a signed integer using saturating +semantics. + +Arguments: +"""""""""" + +The argument may be any floating-point or vector of floating-point type. The +return value may be any integer or vector of integer type. The number of vector +elements in argument and return must be the same. + +Semantics: +"""""""""" + +The conversion to integer is performed subject to the following rules: + +- If the argument is any NaN, zero is returned. +- If the argument is smaller than the smallest representable integer of the + result type (this includes negative infinity), the smallest representable + integer is returned. +- If the argument is larger than the largest representable integer of the + result type (this includes positive infinity), the largest representable + integer is returned. +- Otherwise, the result of rounding the argument towards zero is returned. + +Example: +"""""""" + +.. code-block:: text + + %a = call i8 @llvm.fptosi.sat.i8.f32(float 23.9) ; yields i8: 23 + %b = call i8 @llvm.fptosi.sat.i8.f32(float -130.8) ; yields i8: -128 + %c = call i8 @llvm.fptosi.sat.i8.f32(float 999.0) ; yields i8: 127 + %d = call i8 @llvm.fptosi.sat.i8.f32(float 0xFFF8000000000000) ; yields i8: 0 + .. _dbg_intrinsics: Debugger Intrinsics diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -734,6 +734,20 @@ FP_TO_SINT, FP_TO_UINT, + /// FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a + /// signed or unsigned integer type with the bit width given in operand 1 with + /// the following semantics: + /// + /// * If the value is NaN, zero is returned. + /// * If the value is larger/smaller than the largest/smallest integer, + /// the largest/smallest integer is returned (saturation). + /// * Otherwise the result of rounding the value towards zero is returned. + /// + /// The width in operand 1 may be smaller than the scalar result type width + /// as a result of integer type legalization. + FP_TO_SINT_SAT, + FP_TO_UINT_SAT, + /// X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type /// down to the precision of the destination VT. TRUNC is a flag, which is /// always an integer that is zero or one. If TRUNC is 0, this is a diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4341,6 +4341,11 @@ /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs. SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const; + /// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max. + /// \param N Node to expand + /// \returns The expansion result + SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const; + /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes, /// vector nodes can only succeed if all operations are legal/custom. /// \param N Node to expand diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1266,6 +1266,12 @@ def int_convert_from_fp16 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_i16_ty]>; } +// Saturating floating point to integer intrinsics +let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in { +def int_fptoui_sat : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; +def int_fptosi_sat : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; +} + // Clear cache intrinsic, default to ignore (ie. emit nothing) // maps to void __clear_cache() on supporting platforms def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -164,6 +164,9 @@ def SDTFPToIntOp : SDTypeProfile<1, 1, [ // fp_to_[su]int SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1> ]>; +def SDTFPToIntSatOp : SDTypeProfile<1, 2, [ // fp_to_[su]int_sat + SDTCisInt<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisSameNumEltsAs<0, 1> +]>; def SDTExtInreg : SDTypeProfile<1, 2, [ // sext_inreg SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisVT<2, OtherVT>, SDTCisVTSmallerThanOp<2, 1> @@ -484,6 +487,8 @@ def uint_to_fp : SDNode<"ISD::UINT_TO_FP" , SDTIntToFPOp>; def fp_to_sint : SDNode<"ISD::FP_TO_SINT" , SDTFPToIntOp>; def fp_to_uint : SDNode<"ISD::FP_TO_UINT" , SDTFPToIntOp>; +def fp_to_sint_sat : SDNode<"ISD::FP_TO_SINT_SAT" , SDTFPToIntSatOp>; +def fp_to_uint_sat : SDNode<"ISD::FP_TO_UINT_SAT" , SDTFPToIntSatOp>; def f16_to_fp : SDNode<"ISD::FP16_TO_FP" , SDTIntToFPOp>; def fp_to_f16 : SDNode<"ISD::FP_TO_FP16" , SDTFPToIntOp>; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -179,6 +179,7 @@ SmallVectorImpl &Results); void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, SmallVectorImpl &Results); + SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl); SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); @@ -1124,10 +1125,11 @@ case ISD::SSUBSAT: case ISD::USUBSAT: case ISD::SSHLSAT: - case ISD::USHLSAT: { + case ISD::USHLSAT: + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); break; - } case ISD::SMULFIX: case ISD::SMULFIXSAT: case ISD::UMULFIX: @@ -2697,6 +2699,30 @@ Results.push_back(Operation.getValue(1)); } +/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point +/// the result and operand types are legal and there must be a legal +/// FP_TO_*INT_SAT operation for a larger result type. +SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, + const SDLoc &dl) { + unsigned Opcode = Node->getOpcode(); + + // Scan for the appropriate larger type to use. + EVT NewOutTy = Node->getValueType(0); + while (true) { + NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1); + assert(NewOutTy.isInteger() && "Ran out of possibilities!"); + + if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy)) + break; + } + + // Saturation width is determined by second operand, so we don't have to + // perform any fixup and can directly truncate the result. + SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), + Node->getOperand(1)); + return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); +} + /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { EVT VT = Op.getValueType(); @@ -3124,6 +3150,10 @@ return true; } break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); + break; case ISD::VAARG: Results.push_back(DAG.expandVAArg(Node)); Results.push_back(Results[0].getValue(1)); @@ -4464,6 +4494,10 @@ case ISD::STRICT_FP_TO_SINT: PromoteLegalFP_TO_INT(Node, dl, Results); break; + case ISD::FP_TO_UINT_SAT: + case ISD::FP_TO_SINT_SAT: + Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); + break; case ISD::UINT_TO_FP: case ISD::STRICT_UINT_TO_FP: case ISD::SINT_TO_FP: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -811,6 +811,9 @@ case ISD::STRICT_FP_TO_UINT: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; case ISD::STRICT_LROUND: case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; case ISD::STRICT_LLROUND: @@ -946,6 +949,11 @@ return SDValue(); } +SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { + SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); + return Res; +} + SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); ISD::CondCode CCCode = cast(N->getOperand(4))->get(); @@ -2070,6 +2078,9 @@ case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; @@ -2113,6 +2124,13 @@ return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); } +SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, + unsigned OpNo) { + SDValue Op = GetPromotedFloat(N->getOperand(0)); + return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, + N->getOperand(1)); +} + SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { SDValue Op = GetPromotedFloat(N->getOperand(0)); EVT VT = N->getValueType(0); @@ -2837,6 +2855,9 @@ case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; case ISD::STRICT_FP_EXTEND: case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; @@ -2906,6 +2927,20 @@ return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); } +SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { + SDValue Op = N->getOperand(0); + SDLoc dl(N); + + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); + + Op = GetSoftPromotedHalf(Op); + + SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); + + return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, + N->getOperand(1)); +} + SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo) { assert(OpNo == 0 && "Can only soften the comparison values"); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -123,6 +123,10 @@ case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; + case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; @@ -596,6 +600,14 @@ DAG.getValueType(N->getValueType(0).getScalarType())); } +SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { + // Promote the result type, while keeping the original width in Op1. + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + SDLoc dl(N); + return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), + N->getOperand(1)); +} + SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDLoc dl(N); @@ -2022,6 +2034,8 @@ case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; case ISD::STRICT_FP_TO_UINT: case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; case ISD::STRICT_LLROUND: case ISD::STRICT_LLRINT: case ISD::LLROUND: @@ -3017,6 +3031,12 @@ ReplaceValueWith(SDValue(N, 1), Tmp.second); } +void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, + SDValue &Hi) { + SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); + SplitInteger(Res, Lo, Hi); +} + void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -315,6 +315,7 @@ SDValue PromoteIntRes_CTTZ(SDNode *N); SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N); SDValue PromoteIntRes_FP_TO_XINT(SDNode *N); + SDValue PromoteIntRes_FP_TO_XINT_SAT(SDNode *N); SDValue PromoteIntRes_FP_TO_FP16(SDNode *N); SDValue PromoteIntRes_FREEZE(SDNode *N); SDValue PromoteIntRes_INT_EXTEND(SDNode *N); @@ -423,6 +424,7 @@ void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_FP_TO_XINT_SAT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_LLROUND_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -559,6 +561,7 @@ SDValue SoftenFloatOp_BR_CC(SDNode *N); SDValue SoftenFloatOp_FP_ROUND(SDNode *N); SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N); + SDValue SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N); SDValue SoftenFloatOp_LROUND(SDNode *N); SDValue SoftenFloatOp_LLROUND(SDNode *N); SDValue SoftenFloatOp_LRINT(SDNode *N); @@ -676,6 +679,7 @@ SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo); SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo); SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo); + SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo); SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo); SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo); SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo); @@ -714,6 +718,7 @@ SDValue SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, unsigned OpNo); SDValue SoftPromoteHalfOp_FP_EXTEND(SDNode *N); SDValue SoftPromoteHalfOp_FP_TO_XINT(SDNode *N); + SDValue SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N); SDValue SoftPromoteHalfOp_SETCC(SDNode *N); SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo); SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo); @@ -758,6 +763,7 @@ SDValue ScalarizeVecRes_SETCC(SDNode *N); SDValue ScalarizeVecRes_UNDEF(SDNode *N); SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N); + SDValue ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N); SDValue ScalarizeVecRes_FIX(SDNode *N); @@ -824,6 +830,7 @@ void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi); // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>. bool SplitVectorOperand(SDNode *N, unsigned OpNo); @@ -844,6 +851,7 @@ SDValue SplitVecOp_VSETCC(SDNode *N); SDValue SplitVecOp_FP_ROUND(SDNode *N); SDValue SplitVecOp_FCOPYSIGN(SDNode *N); + SDValue SplitVecOp_FP_TO_XINT_SAT(SDNode *N); //===--------------------------------------------------------------------===// // Vector Widening Support: LegalizeVectorTypes.cpp @@ -892,6 +900,7 @@ SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo); SDValue WidenVecRes_Convert(SDNode *N); SDValue WidenVecRes_Convert_StrictFP(SDNode *N); + SDValue WidenVecRes_FP_TO_XINT_SAT(SDNode *N); SDValue WidenVecRes_FCOPYSIGN(SDNode *N); SDValue WidenVecRes_POWI(SDNode *N); SDValue WidenVecRes_Unary(SDNode *N); @@ -913,6 +922,7 @@ SDValue WidenVecOp_VSELECT(SDNode *N); SDValue WidenVecOp_Convert(SDNode *N); + SDValue WidenVecOp_FP_TO_XINT_SAT(SDNode *N); SDValue WidenVecOp_FCOPYSIGN(SDNode *N); SDValue WidenVecOp_VECREDUCE(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -456,6 +456,8 @@ case ISD::USUBSAT: case ISD::SSHLSAT: case ISD::USHLSAT: + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); break; case ISD::SMULFIX: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -162,6 +162,11 @@ R = ScalarizeVecRes_StrictFPOp(N); break; + case ISD::FP_TO_UINT_SAT: + case ISD::FP_TO_SINT_SAT: + R = ScalarizeVecRes_FP_TO_XINT_SAT(N); + break; + case ISD::UADDO: case ISD::SADDO: case ISD::USUBO: @@ -516,6 +521,23 @@ return GetScalarizedVector(N->getOperand(Op)); } +SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { + SDValue Src = N->getOperand(0); + EVT SrcVT = Src.getValueType(); + SDLoc dl(N); + + // Handle case where result is scalarized but operand is not + if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector) + Src = GetScalarizedVector(Src); + else + Src = DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src, + DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + + EVT DstVT = N->getValueType(0).getVectorElementType(); + return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1)); +} + SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { assert(N->getValueType(0).isVector() && N->getOperand(0).getValueType().isVector() && @@ -966,6 +988,11 @@ SplitVecRes_StrictFPOp(N, Lo, Hi); break; + case ISD::FP_TO_UINT_SAT: + case ISD::FP_TO_SINT_SAT: + SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi); + break; + case ISD::UADDO: case ISD::SADDO: case ISD::USUBO: @@ -1977,6 +2004,22 @@ ReplaceValueWith(SDValue(N, 1), Chain); } +void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, + SDValue &Hi) { + EVT DstVTLo, DstVTHi; + std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0)); + SDLoc dl(N); + + SDValue SrcLo, SrcHi; + EVT SrcVT = N->getOperand(0).getValueType(); + if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector) + GetSplitVector(N->getOperand(0), SrcLo, SrcHi); + else + std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0); + + Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1)); + Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1)); +} //===----------------------------------------------------------------------===// // Operand Vector Splitting @@ -2041,6 +2084,10 @@ else Res = SplitVecOp_UnaryOp(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = SplitVecOp_FP_TO_XINT_SAT(N); + break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::STRICT_FP_TO_SINT: @@ -2751,6 +2798,22 @@ return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); } +SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { + EVT ResVT = N->getValueType(0); + SDValue Lo, Hi; + SDLoc dl(N); + GetSplitVector(N->getOperand(0), Lo, Hi); + EVT InVT = Lo.getValueType(); + + EVT NewResVT = + EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), + InVT.getVectorElementCount()); + + Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1)); + Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1)); + + return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); +} //===----------------------------------------------------------------------===// // Result Vector Widening @@ -2895,6 +2958,11 @@ Res = WidenVecRes_Convert(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = WidenVecRes_FP_TO_XINT_SAT(N); + break; + case ISD::FABS: case ISD::FCEIL: case ISD::FCOS: @@ -3404,6 +3472,27 @@ return DAG.getBuildVector(WidenVT, DL, Ops); } +SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { + SDLoc dl(N); + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + ElementCount WidenNumElts = WidenVT.getVectorElementCount(); + + SDValue Src = N->getOperand(0); + EVT SrcVT = Src.getValueType(); + + // Also widen the input. + if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) { + Src = GetWidenedVector(Src); + SrcVT = Src.getValueType(); + } + + // Input and output not widened to the same size, give up. + if (WidenNumElts != SrcVT.getVectorElementCount()) + return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue()); + + return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1)); +} + SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { SDValue InOp = N->getOperand(1); SDLoc DL(N); @@ -4321,6 +4410,11 @@ Res = WidenVecOp_Convert(N); break; + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: + Res = WidenVecOp_FP_TO_XINT_SAT(N); + break; + case ISD::VECREDUCE_FADD: case ISD::VECREDUCE_FMUL: case ISD::VECREDUCE_ADD: @@ -4490,6 +4584,28 @@ return DAG.getBuildVector(VT, dl, Ops); } +SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { + EVT DstVT = N->getValueType(0); + SDValue Src = GetWidenedVector(N->getOperand(0)); + EVT SrcVT = Src.getValueType(); + ElementCount WideNumElts = SrcVT.getVectorElementCount(); + SDLoc dl(N); + + // See if a widened result type would be legal, if so widen the node. + EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(), + DstVT.getVectorElementType(), WideNumElts); + if (TLI.isTypeLegal(WideDstVT)) { + SDValue Res = + DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1)); + return DAG.getNode( + ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res, + DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + } + + // Give up and unroll. + return DAG.UnrollVectorOp(N); +} + SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { EVT VT = N->getValueType(0); SDValue InOp = GetWidenedVector(N->getOperand(0)); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6171,6 +6171,20 @@ DAG.getNode(ISD::BITCAST, sdl, MVT::f16, getValue(I.getArgOperand(0))))); return; + case Intrinsic::fptosi_sat: { + EVT Type = TLI.getValueType(DAG.getDataLayout(), I.getType()); + SDValue SatW = DAG.getConstant(Type.getScalarSizeInBits(), sdl, MVT::i32); + setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, Type, + getValue(I.getArgOperand(0)), SatW)); + return; + } + case Intrinsic::fptoui_sat: { + EVT Type = TLI.getValueType(DAG.getDataLayout(), I.getType()); + SDValue SatW = DAG.getConstant(Type.getScalarSizeInBits(), sdl, MVT::i32); + setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, Type, + getValue(I.getArgOperand(0)), SatW)); + return; + } case Intrinsic::pcmarker: { SDValue Tmp = getValue(I.getArgOperand(0)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -348,6 +348,8 @@ case ISD::STRICT_FP_TO_SINT: return "strict_fp_to_sint"; case ISD::FP_TO_UINT: return "fp_to_uint"; case ISD::STRICT_FP_TO_UINT: return "strict_fp_to_uint"; + case ISD::FP_TO_SINT_SAT: return "fp_to_sint_sat"; + case ISD::FP_TO_UINT_SAT: return "fp_to_uint_sat"; case ISD::BITCAST: return "bitcast"; case ISD::ADDRSPACECAST: return "addrspacecast"; case ISD::FP16_TO_FP: return "fp16_to_fp"; diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -8069,3 +8069,105 @@ } return false; } + +SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node, + SelectionDAG &DAG) const { + bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; + SDLoc dl(SDValue(Node, 0)); + SDValue Src = Node->getOperand(0); + + // DstVT is the result type, while SatVT is the size to which we saturate + EVT SrcVT = Src.getValueType(); + EVT DstVT = Node->getValueType(0); + + unsigned SatWidth = Node->getConstantOperandVal(1); + unsigned DstWidth = DstVT.getScalarSizeInBits(); + assert(SatWidth <= DstWidth && + "Expected saturation width smaller than result width"); + + // Determine minimum and maximum integer values and their corresponding + // floating-point values. + APInt MinInt, MaxInt; + if (IsSigned) { + MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth); + MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth); + } else { + MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth); + MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth); + } + + // We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as + // libcall emission cannot handle this. Large result types will fail. + if (SrcVT == MVT::f16) { + Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src); + SrcVT = Src.getValueType(); + } + + APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT)); + APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT)); + + APFloat::opStatus MinStatus = + MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero); + APFloat::opStatus MaxStatus = + MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero); + bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) && + !(MaxStatus & APFloat::opStatus::opInexact); + + SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT); + SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT); + + // If the integer bounds are exactly representable as floats and min/max are + // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence + // of comparisons and selects. + bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) && + isOperationLegal(ISD::FMAXNUM, SrcVT); + if (AreExactFloatBounds && MinMaxLegal) { + SDValue Clamped = Src; + + // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat. + Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode); + // Clamp by MaxFloat from above. NaN cannot occur. + Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode); + // Convert clamped value to integer. + SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, + dl, DstVT, Clamped); + + // In the unsigned case we're done, because we mapped NaN to MinFloat, + // which will cast to zero. + if (!IsSigned) + return FpToInt; + + // Otherwise, select 0 if Src is NaN. + SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); + return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt, + ISD::CondCode::SETUO); + } + + SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT); + SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT); + + // Result of direct conversion. The assumption here is that the operation is + // non-trapping and it's fine to apply it to an out-of-range value if we + // select it away later. + SDValue FpToInt = + DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src); + + SDValue Select = FpToInt; + + // If Src ULT MinFloat, select MinInt. In particular, this also selects + // MinInt if Src is NaN. + Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select, + ISD::CondCode::SETULT); + // If Src OGT MaxFloat, select MaxInt. + Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select, + ISD::CondCode::SETOGT); + + // In the unsigned case we are done, because we mapped NaN to MinInt, which + // is already zero. + if (!IsSigned) + return Select; + + // Otherwise, select 0 if Src is NaN. + SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); + return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO); +} diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -667,6 +667,8 @@ setOperationAction(ISD::SDIVFIXSAT, VT, Expand); setOperationAction(ISD::UDIVFIX, VT, Expand); setOperationAction(ISD::UDIVFIXSAT, VT, Expand); + setOperationAction(ISD::FP_TO_SINT_SAT, VT, Expand); + setOperationAction(ISD::FP_TO_UINT_SAT, VT, Expand); // Overflow operations default to expand setOperationAction(ISD::SADDO, VT, Expand);