[SelectionDAG] Use a VTSDNode to store the saturation width for FP_TO_SINT_SAT/FP_TO_UINT_SAT

[craig.topper]

Summary

Previously we used an i32 constant to store the saturation width, but i32 isn't
legal on RISCV64. This wasn't a big deal to fix, but it is extra work for the
type legalizer.

This patch uses a VTSDNode to store the type similar to SEXT_INREG. This makes
it opaque to the type legalizer.

[nikic]

This is what I did in my original patch -- I think this got switched to a simple integer because it made legalization simpler. Shouldn't this require some changes to the vector legalizer at least? E.g. SplitVector would also have to split the VTSD.

[craig.topper]

-Use scalar type to avoid vector type legalization changes and changes to UnrollVectorOp.
-Add code to getNode to check the rules.

[bjope]

Haven't tried it yet (on our downstream code base), but this looks good to me.

Should probably let @nikic confirm if the worries about vector legalization was solved by the latest update.

[RKSimon]

@nikic Any more comments?

[nikic]

LGTM

llvm/include/llvm/CodeGen/ISDOpcodes.h

enum NodeType {
 
   /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned
   /// integer. These have the same semantics as fptosi and fptoui in IR. If
   /// the FP value cannot fit in the integer type, the results are undefined.
   FP_TO_SINT,
   FP_TO_UINT,
 
   /// FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a
-  /// signed or unsigned integer type with the bit width given in operand 1 with
-  /// the following semantics:
+  /// signed or unsigned scalar integer type given in operand 1 with the
+  /// following semantics:
   ///
   ///  * If the value is NaN, zero is returned.
   ///  * If the value is larger/smaller than the largest/smallest integer,
   ///    the largest/smallest integer is returned (saturation).
   ///  * Otherwise the result of rounding the value towards zero is returned.
   ///
-  /// The width given in operand 1 must be equal to, or smaller than, the scalar
-  /// result type width. It may end up being smaller than the result witdh as a
-  /// result of integer type legalization.
+  /// The scalar width of the type given in operand 1 must be equal to, or
+  /// smaller than, the scalar result type width. It may end up being smaller
+  /// than the result width as a result of integer type legalization.
   FP_TO_SINT_SAT,
   FP_TO_UINT_SAT,
 
   /// X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type
   /// down to the precision of the destination VT.  TRUNC is a flag, which is
   /// always an integer that is zero or one.  If TRUNC is 0, this is a
   /// normal rounding, if it is 1, this FP_ROUND is known to not change the
   /// value of Y.

llvm/include/llvm/Target/TargetSelectionDAG.td

 ]>;
 def SDTIntToFPOp : SDTypeProfile<1, 1, [    // [su]int_to_fp
   SDTCisFP<0>, SDTCisInt<1>, SDTCisSameNumEltsAs<0, 1>
 ]>;
 def SDTFPToIntOp : SDTypeProfile<1, 1, [    // fp_to_[su]int
   SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>
 ]>;
 def SDTFPToIntSatOp : SDTypeProfile<1, 2, [    // fp_to_[su]int_sat
-  SDTCisInt<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisSameNumEltsAs<0, 1>
+  SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>, SDTCisVT<2, OtherVT>
 ]>;
 def SDTExtInreg : SDTypeProfile<1, 2, [     // sext_inreg
   SDTCisSameAs<0, 1>, SDTCisInt<0>, SDTCisVT<2, OtherVT>,
   SDTCisVTSmallerThanOp<2, 1>
 ]>;
 def SDTExtInvec : SDTypeProfile<1, 1, [     // sext_invec
   SDTCisInt<0>, SDTCisVec<0>, SDTCisInt<1>, SDTCisVec<1>,
   SDTCisOpSmallerThanOp<1, 0>

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::UMULFIXSAT:
   case ISD::SDIVFIX:
   case ISD::SDIVFIXSAT:
   case ISD::UDIVFIX:
   case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
 
   case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
 
-  case ISD::FP_TO_SINT_SAT:
-  case ISD::FP_TO_UINT_SAT: PromoteIntOp_FP_TO_XINT_SAT(N); break;
-
   case ISD::VECREDUCE_ADD:
   case ISD::VECREDUCE_MUL:
   case ISD::VECREDUCE_AND:
   case ISD::VECREDUCE_OR:
   case ISD::VECREDUCE_XOR:
   case ISD::VECREDUCE_SMAX:
   case ISD::VECREDUCE_SMIN:
   case ISD::VECREDUCE_UMAX:

 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
   SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
   return SDValue(
       DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
 }
 
-SDValue DAGTypeLegalizer::PromoteIntOp_FP_TO_XINT_SAT(SDNode *N) {
-  SDValue Op1 = ZExtPromotedInteger(N->getOperand(1));
-  return SDValue(
-      DAG.UpdateNodeOperands(N, N->getOperand(0), Op1), 0);
-}
-
 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
   // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
   return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
   assert(OpNo > 1 && "Don't know how to promote this operand!");

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
   SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FIX(SDNode *N);
-  SDValue PromoteIntOp_FP_TO_XINT_SAT(SDNode *N);
   SDValue PromoteIntOp_FPOWI(SDNode *N);
   SDValue PromoteIntOp_VECREDUCE(SDNode *N);
   SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);
 
   void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
 
   //===--------------------------------------------------------------------===//
   // Integer Expansion Support: LegalizeIntegerTypes.cpp

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
         ConstantSDNode *C = cast<ConstantSDNode>(Op);
         APInt Val = C->getAPIntValue();
         Ops.push_back(SignExtendInReg(Val, OpVT));
       }
       return getBuildVector(VT, DL, Ops);
     }
     break;
   }
+  case ISD::FP_TO_SINT_SAT:
+  case ISD::FP_TO_UINT_SAT: {
+    assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
+           N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
+    assert(N1.getValueType().isVector() == VT.isVector() &&
+           "FP_TO_*INT_SAT type should be vector iff the operand type is "
+           "vector!");
+    assert((!VT.isVector() || VT.getVectorNumElements() ==
+                                  N1.getValueType().getVectorNumElements()) &&
+           "Vector element counts must match in FP_TO_*INT_SAT");
+    assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
+           "Type to saturate to must be a scalar.");
+    assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
+           "Not extending!");
+    break;
+  }
   case ISD::EXTRACT_VECTOR_ELT:
     assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
            "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
              element type of the vector.");
 
     // Extract from an undefined value or using an undefined index is undefined.
     if (N1.isUndef() || N2.isUndef())
       return getUNDEF(VT);

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

case Intrinsic::convert_to_fp16:
     return;
   case Intrinsic::convert_from_fp16:
     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
                                          getValue(I.getArgOperand(0)))));
     return;
   case Intrinsic::fptosi_sat: {
-    EVT Type = TLI.getValueType(DAG.getDataLayout(), I.getType());
-    SDValue SatW = DAG.getConstant(Type.getScalarSizeInBits(), sdl, MVT::i32);
-    setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, Type,
-                             getValue(I.getArgOperand(0)), SatW));
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+    setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
+                             getValue(I.getArgOperand(0)),
+                             DAG.getValueType(VT.getScalarType())));
     return;
   }
   case Intrinsic::fptoui_sat: {
-    EVT Type = TLI.getValueType(DAG.getDataLayout(), I.getType());
-    SDValue SatW = DAG.getConstant(Type.getScalarSizeInBits(), sdl, MVT::i32);
-    setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, Type,
-                             getValue(I.getArgOperand(0)), SatW));
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+    setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
+                             getValue(I.getArgOperand(0)),
+                             DAG.getValueType(VT.getScalarType())));
     return;
   }
   case Intrinsic::set_rounding:
     Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
                       {getRoot(), getValue(I.getArgOperand(0))});
     setValue(&I, Res);
     DAG.setRoot(Res.getValue(0));
     return;

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
   bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
   SDLoc dl(SDValue(Node, 0));
   SDValue Src = Node->getOperand(0);
 
   // DstVT is the result type, while SatVT is the size to which we saturate
   EVT SrcVT = Src.getValueType();
   EVT DstVT = Node->getValueType(0);
 
-  unsigned SatWidth = Node->getConstantOperandVal(1);
+  EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
+  unsigned SatWidth = SatVT.getScalarSizeInBits();
   unsigned DstWidth = DstVT.getScalarSizeInBits();
   assert(SatWidth <= DstWidth &&
          "Expected saturation width smaller than result width");
 
   // Determine minimum and maximum integer values and their corresponding
   // floating-point values.
   APInt MinInt, MaxInt;
   if (IsSigned) {

llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
 
   return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
 }
 
 SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
                                                       SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT ResT = Op.getValueType();
-  uint64_t Width = Op.getConstantOperandVal(1);
+  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
 
-  if ((ResT == MVT::i32 || ResT == MVT::i64) && (Width == 32 || Width == 64))
+  if ((ResT == MVT::i32 || ResT == MVT::i64) &&
+      (SatVT == MVT::i32 || SatVT == MVT::i64))
     return Op;
 
-  if (ResT == MVT::v4i32 && Width == 32)
+  if (ResT == MVT::v4i32 && SatVT == MVT::i32)
     return Op;
 
   return SDValue();
 }
 
 //===----------------------------------------------------------------------===//
 //   Custom DAG combine hooks
 //===----------------------------------------------------------------------===//
performVectorTruncSatLowCombine(SDNode *N,
   EVT ResVT = N->getValueType(0);
   if (ResVT != MVT::v4i32)
     return SDValue();
 
   auto FPToInt = N->getOperand(0);
   auto FPToIntOp = FPToInt.getOpcode();
   if (FPToIntOp != ISD::FP_TO_SINT_SAT && FPToIntOp != ISD::FP_TO_UINT_SAT)
     return SDValue();
-  if (FPToInt.getConstantOperandVal(1) != 32)
+  if (cast<VTSDNode>(FPToInt.getOperand(1))->getVT() != MVT::i32)
     return SDValue();
 
   auto Source = FPToInt.getOperand(0);
   if (Source.getValueType() != MVT::v2f64)
     return SDValue();
 
   auto *Splat = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
   APInt SplatValue, SplatUndef;

llvm/lib/Target/WebAssembly/WebAssemblyInstrConv.td

defm I64_TRUNC_S_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_U_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_uint F64:$src))],
                              "i64.trunc_sat_f64_u\t$dst, $src",
                              "i64.trunc_sat_f64_u", 0xfc07>,
                              Requires<[HasNontrappingFPToInt]>;
 
 // Support the explicitly saturating operations as well.
-def : Pat<(fp_to_sint_sat F32:$src, (i32 32)), (I32_TRUNC_S_SAT_F32 F32:$src)>;
-def : Pat<(fp_to_uint_sat F32:$src, (i32 32)), (I32_TRUNC_U_SAT_F32 F32:$src)>;
-def : Pat<(fp_to_sint_sat F64:$src, (i32 32)), (I32_TRUNC_S_SAT_F64 F64:$src)>;
-def : Pat<(fp_to_uint_sat F64:$src, (i32 32)), (I32_TRUNC_U_SAT_F64 F64:$src)>;
-def : Pat<(fp_to_sint_sat F32:$src, (i32 64)), (I64_TRUNC_S_SAT_F32 F32:$src)>;
-def : Pat<(fp_to_uint_sat F32:$src, (i32 64)), (I64_TRUNC_U_SAT_F32 F32:$src)>;
-def : Pat<(fp_to_sint_sat F64:$src, (i32 64)), (I64_TRUNC_S_SAT_F64 F64:$src)>;
-def : Pat<(fp_to_uint_sat F64:$src, (i32 64)), (I64_TRUNC_U_SAT_F64 F64:$src)>;
+def : Pat<(fp_to_sint_sat F32:$src, i32), (I32_TRUNC_S_SAT_F32 F32:$src)>;
+def : Pat<(fp_to_uint_sat F32:$src, i32), (I32_TRUNC_U_SAT_F32 F32:$src)>;
+def : Pat<(fp_to_sint_sat F64:$src, i32), (I32_TRUNC_S_SAT_F64 F64:$src)>;
+def : Pat<(fp_to_uint_sat F64:$src, i32), (I32_TRUNC_U_SAT_F64 F64:$src)>;
+def : Pat<(fp_to_sint_sat F32:$src, i64), (I64_TRUNC_S_SAT_F32 F32:$src)>;
+def : Pat<(fp_to_uint_sat F32:$src, i64), (I64_TRUNC_U_SAT_F32 F32:$src)>;
+def : Pat<(fp_to_sint_sat F64:$src, i64), (I64_TRUNC_S_SAT_F64 F64:$src)>;
+def : Pat<(fp_to_uint_sat F64:$src, i64), (I64_TRUNC_U_SAT_F64 F64:$src)>;
 
 // Conversion from floating point to integer pseudo-instructions which don't
 // trap on overflow or invalid.
 let usesCustomInserter = 1, isCodeGenOnly = 1 in {
 defm FP_TO_SINT_I32_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                             [(set I32:$dst, (fp_to_sint F32:$src))], "", "", 0>,
                             Requires<[NotHasNontrappingFPToInt]>;
 defm FP_TO_UINT_I32_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),

llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td

SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
            vec.prefix#"."#name#"\t$dst, $vec", vec.prefix#"."#name, simdop>;
 }
 
 // Floating point to integer with saturation: trunc_sat
 defm "" : SIMDConvert<I32x4, F32x4, fp_to_sint, "trunc_sat_f32x4_s", 248>;
 defm "" : SIMDConvert<I32x4, F32x4, fp_to_uint, "trunc_sat_f32x4_u", 249>;
 
 // Support the saturating variety as well.
-def trunc_s_sat32 : PatFrag<(ops node:$x), (fp_to_sint_sat $x, (i32 32))>;
-def trunc_u_sat32 : PatFrag<(ops node:$x), (fp_to_uint_sat $x, (i32 32))>;
+def trunc_s_sat32 : PatFrag<(ops node:$x), (fp_to_sint_sat $x, i32)>;
+def trunc_u_sat32 : PatFrag<(ops node:$x), (fp_to_uint_sat $x, i32)>;
 def : Pat<(v4i32 (trunc_s_sat32 (v4f32 V128:$src))), (fp_to_sint_I32x4 $src)>;
 def : Pat<(v4i32 (trunc_u_sat32 (v4f32 V128:$src))), (fp_to_uint_I32x4 $src)>;
 
 def trunc_sat_zero_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
 def trunc_sat_zero_s :
   SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_S", trunc_sat_zero_t>;
 def trunc_sat_zero_u :
   SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_U", trunc_sat_zero_t>;

llvm/lib/Target/X86/X86ISelLowering.cpp

X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
   EVT DstVT = Node->getValueType(0);
   EVT TmpVT = DstVT;
 
   // This code is only for floats and doubles. Fall back to generic code for
   // anything else.
   if (!isScalarFPTypeInSSEReg(SrcVT))
     return SDValue();
 
-  unsigned SatWidth = Node->getConstantOperandVal(1);
+  EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
+  unsigned SatWidth = SatVT.getScalarSizeInBits();
   unsigned DstWidth = DstVT.getScalarSizeInBits();
   unsigned TmpWidth = TmpVT.getScalarSizeInBits();
   assert(SatWidth <= DstWidth && SatWidth <= TmpWidth &&
          "Expected saturation width smaller than result width");
 
   // Promote result of FP_TO_*INT to at least 32 bits.
   if (TmpWidth < 32) {
     TmpVT = MVT::i32;