[AArch64][SVE] Add lowering for rounding operations

[Asif]

Summary

Add the functionality to lower SVE rounding operations for passthru variant.
Created a new test case file for all rounding operations.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

1425

So we can prove the DAGCombine for these intrinsics is the canonical route, can you please change this and the following sve_frint instances to null_frag. Once FRECPX and FSQRT follow a similar path we'll remove the parameter.

llvm/test/CodeGen/AArch64/sve-fp-rounding.ll

1

Since there's no expected variance for these tests can you use update_llc_test_checks.py to generate the check lines. It'll just make it easier in the future if somebody adds anything new.

[paulwalker-arm]

Other than adding the missing test before landing that patch, this LGTM.

llvm/test/CodeGen/AArch64/sve-fp-rounding.ll

118

It looks like you are missing a test for define <vscale x 2 x double> @frintm_nxv2f64.

[Asif]

Added the missing test.

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   SUB_PRED,
   UDIV_PRED,
   UMAX_PRED,
   UMIN_PRED,
 
   // Predicated instructions with the result of inactive lanes provided by the
   // last operand.
   FCEIL_MERGE_PASSTHRU,
+  FFLOOR_MERGE_PASSTHRU,
+  FNEARBYINT_MERGE_PASSTHRU,
   FNEG_MERGE_PASSTHRU,
+  FRINT_MERGE_PASSTHRU,
+  FROUND_MERGE_PASSTHRU,
+  FROUNDEVEN_MERGE_PASSTHRU,
+  FTRUNC_MERGE_PASSTHRU,
   SIGN_EXTEND_INREG_MERGE_PASSTHRU,
   ZERO_EXTEND_INREG_MERGE_PASSTHRU,
 
   SETCC_MERGE_ZERO,
 
   // Arithmetic instructions which write flags.
   ADDS,
   SUBS,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static bool isMergePassthruOpcode(unsigned Opc) {
   switch (Opc) {
   default:
     return false;
   case AArch64ISD::DUP_MERGE_PASSTHRU:
   case AArch64ISD::FNEG_MERGE_PASSTHRU:
   case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
   case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
   case AArch64ISD::FCEIL_MERGE_PASSTHRU:
+  case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
+  case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
+  case AArch64ISD::FRINT_MERGE_PASSTHRU:
+  case AArch64ISD::FROUND_MERGE_PASSTHRU:
+  case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
+  case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
     return true;
   }
 }
 
 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                              const AArch64Subtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
   // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
for (MVT VT : MVT::fp_scalable_vector_valuetypes()) {
         setOperationAction(ISD::SELECT, VT, Custom);
         setOperationAction(ISD::FADD, VT, Custom);
         setOperationAction(ISD::FDIV, VT, Custom);
         setOperationAction(ISD::FMA, VT, Custom);
         setOperationAction(ISD::FMUL, VT, Custom);
         setOperationAction(ISD::FNEG, VT, Custom);
         setOperationAction(ISD::FSUB, VT, Custom);
         setOperationAction(ISD::FCEIL, VT, Custom);
+        setOperationAction(ISD::FFLOOR, VT, Custom);
+        setOperationAction(ISD::FNEARBYINT, VT, Custom);
+        setOperationAction(ISD::FRINT, VT, Custom);
+        setOperationAction(ISD::FROUND, VT, Custom);
+        setOperationAction(ISD::FROUNDEVEN, VT, Custom);
+        setOperationAction(ISD::FTRUNC, VT, Custom);
       }
     }
 
     // NOTE: Currently this has to happen after computeRegisterProperties rather
     // than the preferred option of combining it with the addRegisterClass call.
     if (useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::SUB_PRED)
     MAKE_CASE(AArch64ISD::UDIV_PRED)
     MAKE_CASE(AArch64ISD::UMAX_PRED)
     MAKE_CASE(AArch64ISD::UMIN_PRED)
     MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::ADC)
     MAKE_CASE(AArch64ISD::SBC)
     MAKE_CASE(AArch64ISD::ADDS)
     MAKE_CASE(AArch64ISD::SUBS)
     MAKE_CASE(AArch64ISD::ADCS)
     MAKE_CASE(AArch64ISD::SBCS)
     MAKE_CASE(AArch64ISD::ANDS)
case Intrinsic::aarch64_sve_convert_from_svbool:
     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_fneg:
     return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintp:
     return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_frintm:
+    return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));                     
+  case Intrinsic::aarch64_sve_frinti:
+    return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_frintx:
+    return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_frinta:
+    return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_frintn:
+    return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_frintz:
+    return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_convert_to_svbool: {
     EVT OutVT = Op.getValueType();
     EVT InVT = Op.getOperand(1).getValueType();
     // Return the operand if the cast isn't changing type,
     // i.e. <n x 16 x i1> -> <n x 16 x i1>
     if (InVT == OutVT)
       return Op.getOperand(1);
     // Otherwise, zero the newly introduced lanes.
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::FDIV:
     if (Op.getValueType() == MVT::f128)
       return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
   case ISD::FNEG:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEG_MERGE_PASSTHRU);
   case ISD::FCEIL:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FCEIL_MERGE_PASSTHRU);
+  case ISD::FFLOOR:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FFLOOR_MERGE_PASSTHRU);
+  case ISD::FNEARBYINT:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEARBYINT_MERGE_PASSTHRU);
+  case ISD::FRINT:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FRINT_MERGE_PASSTHRU);
+  case ISD::FROUND:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUND_MERGE_PASSTHRU);
+  case ISD::FROUNDEVEN:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU);
+  case ISD::FTRUNC:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FTRUNC_MERGE_PASSTHRU);
   case ISD::FP_ROUND:
   case ISD::STRICT_FP_ROUND:
     return LowerFP_ROUND(Op, DAG);
   case ISD::FP_EXTEND:
     return LowerFP_EXTEND(Op, DAG);
   case ISD::FRAMEADDR:
     return LowerFRAMEADDR(Op, DAG);
   case ISD::SPONENTRY:

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

def SDT_AArch64IntExtend : SDTypeProfile<1, 4, [
   SDTCVecEltisVT<1,i1>, SDTCisSameAs<0,2>, SDTCisVTSmallerThanOp<3, 2>, SDTCisSameAs<0,4>
 ]>;
 
 // Predicated operations with the result of inactive lanes provided by the last operand.
 def AArch64fneg_mt : SDNode<"AArch64ISD::FNEG_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64sxt_mt  : SDNode<"AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU", SDT_AArch64IntExtend>;
 def AArch64uxt_mt  : SDNode<"AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU", SDT_AArch64IntExtend>;
 def AArch64frintp_mt : SDNode<"AArch64ISD::FCEIL_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frintm_mt : SDNode<"AArch64ISD::FFLOOR_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frinti_mt : SDNode<"AArch64ISD::FNEARBYINT_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frintx_mt : SDNode<"AArch64ISD::FRINT_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frinta_mt : SDNode<"AArch64ISD::FROUND_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frintn_mt : SDNode<"AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64frintz_mt : SDNode<"AArch64ISD::FTRUNC_MERGE_PASSTHRU", SDT_AArch64Arith>;
 
 def SDT_AArch64ReduceWithInit : SDTypeProfile<1, 3, [SDTCisVec<1>, SDTCisVec<3>]>;
 def AArch64clasta_n   : SDNode<"AArch64ISD::CLASTA_N",   SDT_AArch64ReduceWithInit>;
 def AArch64clastb_n   : SDNode<"AArch64ISD::CLASTB_N",   SDT_AArch64ReduceWithInit>;
 def AArch64fadda_p    : SDNode<"AArch64ISD::FADDA_PRED", SDT_AArch64ReduceWithInit>;
 
 def SDT_AArch64Rev   : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
 def AArch64rev       : SDNode<"AArch64ISD::REV", SDT_AArch64Rev>;
multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instruction RegImmInst, Instruction RegRegInst, int scale, ComplexPattern AddrCP> {
   defm FCVTZS_ZPmZ_HtoS : sve_fp_2op_p_zd<0b0111100, "fcvtzs", ZPR16, ZPR32, int_aarch64_sve_fcvtzs_i32f16,  nxv4i32, nxv4i1, nxv8f16, ElementSizeS>;
   defm FCVTZS_ZPmZ_HtoD : sve_fp_2op_p_zd<0b0111110, "fcvtzs", ZPR16, ZPR64, int_aarch64_sve_fcvtzs_i64f16,  nxv2i64, nxv2i1, nxv8f16, ElementSizeD>;
   defm FCVTZU_ZPmZ_HtoS : sve_fp_2op_p_zd<0b0111101, "fcvtzu", ZPR16, ZPR32, int_aarch64_sve_fcvtzu_i32f16,  nxv4i32, nxv4i1, nxv8f16, ElementSizeS>;
   defm FCVTZU_ZPmZ_HtoD : sve_fp_2op_p_zd<0b0111111, "fcvtzu", ZPR16, ZPR64, int_aarch64_sve_fcvtzu_i64f16,  nxv2i64, nxv2i1, nxv8f16, ElementSizeD>;
   defm FCVTZU_ZPmZ_StoD : sve_fp_2op_p_zd<0b1111101, "fcvtzu", ZPR32, ZPR64, int_aarch64_sve_fcvtzu_i64f32,  nxv2i64, nxv2i1, nxv4f32, ElementSizeD>;
   defm FCVTZS_ZPmZ_DtoD : sve_fp_2op_p_zd<0b1111110, "fcvtzs", ZPR64, ZPR64, int_aarch64_sve_fcvtzs,         nxv2i64, nxv2i1, nxv2f64, ElementSizeD>;
   defm FCVTZU_ZPmZ_DtoD : sve_fp_2op_p_zd<0b1111111, "fcvtzu", ZPR64, ZPR64, int_aarch64_sve_fcvtzu,         nxv2i64, nxv2i1, nxv2f64, ElementSizeD>;
 
-  defm FRINTN_ZPmZ : sve_fp_2op_p_zd_HSD<0b00000, "frintn", int_aarch64_sve_frintn>;
+  defm FRINTN_ZPmZ : sve_fp_2op_p_zd_HSD<0b00000, "frintn", int_aarch64_sve_frintn, AArch64frintn_mt>;
   defm FRINTP_ZPmZ : sve_fp_2op_p_zd_HSD<0b00001, "frintp", int_aarch64_sve_frintp, AArch64frintp_mt>;
-  defm FRINTM_ZPmZ : sve_fp_2op_p_zd_HSD<0b00010, "frintm", int_aarch64_sve_frintm>;
-  defm FRINTZ_ZPmZ : sve_fp_2op_p_zd_HSD<0b00011, "frintz", int_aarch64_sve_frintz>;
-  defm FRINTA_ZPmZ : sve_fp_2op_p_zd_HSD<0b00100, "frinta", int_aarch64_sve_frinta>;
-  defm FRINTX_ZPmZ : sve_fp_2op_p_zd_HSD<0b00110, "frintx", int_aarch64_sve_frintx>;
-  defm FRINTI_ZPmZ : sve_fp_2op_p_zd_HSD<0b00111, "frinti", int_aarch64_sve_frinti>;
+  defm FRINTM_ZPmZ : sve_fp_2op_p_zd_HSD<0b00010, "frintm", int_aarch64_sve_frintm, AArch64frintm_mt>;
+  defm FRINTZ_ZPmZ : sve_fp_2op_p_zd_HSD<0b00011, "frintz", int_aarch64_sve_frintz, AArch64frintz_mt>;
+  defm FRINTA_ZPmZ : sve_fp_2op_p_zd_HSD<0b00100, "frinta", int_aarch64_sve_frinta, AArch64frinta_mt>;
+  defm FRINTX_ZPmZ : sve_fp_2op_p_zd_HSD<0b00110, "frintx", int_aarch64_sve_frintx, AArch64frintx_mt>;
+  defm FRINTI_ZPmZ : sve_fp_2op_p_zd_HSD<0b00111, "frinti", int_aarch64_sve_frinti, AArch64frinti_mt>;
   defm FRECPX_ZPmZ : sve_fp_2op_p_zd_HSD<0b01100, "frecpx", int_aarch64_sve_frecpx>;
   defm FSQRT_ZPmZ  : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt",  int_aarch64_sve_fsqrt>;
 
   let Predicates = [HasBF16, HasSVE] in {
     defm BFDOT_ZZZ    : sve_bfloat_dot<"bfdot", int_aarch64_sve_bfdot>;
     defm BFDOT_ZZI    : sve_bfloat_dot_indexed<"bfdot", int_aarch64_sve_bfdot_lane>;
     defm BFMMLA_ZZZ   : sve_bfloat_matmul<"bfmmla", int_aarch64_sve_bfmmla>;
     defm BFMMLA_B_ZZZ : sve_bfloat_matmul_longvecl<0b0, "bfmlalb", int_aarch64_sve_bfmlalb>;

llvm/test/CodeGen/AArch64/sve-fp-rounding.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; FCEIL
+
+define <vscale x 8 x half> @frintp_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frintp_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frintp z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frintp_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frintp_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintp z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frintp_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frintp_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintp z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frintp_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frintp_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintp z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frintp_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frintp_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintp z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frintp_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frintp_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintp z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+; FFLOOR
+
+define <vscale x 8 x half> @frintm_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frintm_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frintm z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.floor.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frintm_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frintm_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintm z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.floor.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frintm_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frintm_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintm z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.floor.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frintm_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frintm_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintm z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frintm_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frintm_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintm z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+; FNEARBYINT
+
+define <vscale x 8 x half> @frinti_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frinti_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frinti z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frinti_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frinti_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frinti z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frinti_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frinti_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinti z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frinti_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frinti_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frinti z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frinti_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frinti_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinti z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frinti_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frinti_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinti z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+; FRINT
+
+define <vscale x 8 x half> @frintx_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frintx_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.rint.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frintx_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frintx_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.rint.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frintx_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frintx_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.rint.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frintx_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frintx_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintx z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frintx_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frintx_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintx z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frintx_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frintx_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintx z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+; ROUND
+
+define <vscale x 8 x half> @frinta_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frinta_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frinta z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.round.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frinta_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frinta_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frinta z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.round.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frinta_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frinta_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinta z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.round.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frinta_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frinta_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frinta z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frinta_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frinta_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinta z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frinta_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frinta_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frinta z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+; ROUNDEVEN
+
+define <vscale x 8 x half> @frintn_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frintn_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frintn z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.roundeven.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frintn_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frintn_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintn z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.roundeven.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frintn_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frintn_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintn z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.roundeven.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frintn_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frintn_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintn z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frintn_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frintn_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintn z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frintn_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frintn_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintn z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+; FTRUNC
+
+define <vscale x 8 x half> @frintz_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: frintz_nxv8f16:
+; CHECK: ptrue p0.h
+; CHECK-NEXT: frintz z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.trunc.nxv8f16(<vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %res  
+}
+
+define <vscale x 4 x half> @frintz_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: frintz_nxv4f16:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintz z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x half> @llvm.trunc.nxv4f16(<vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %res  
+}
+
+define <vscale x 2 x half> @frintz_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: frintz_nxv2f16:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintz z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x half> @llvm.trunc.nxv2f16(<vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %res  
+}
+
+define <vscale x 4 x float> @frintz_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: frintz_nxv4f32:
+; CHECK: ptrue p0.s
+; CHECK-NEXT: frintz z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %res  
+}
+
+define <vscale x 2 x float> @frintz_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: frintz_nxv2f32:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintz z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %res  
+}
+
+define <vscale x 2 x double> @frintz_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: frintz_nxv2f64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: frintz z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %res  
+}
+
+declare <vscale x 8 x half> @llvm.ceil.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.ceil.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.ceil.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.floor.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.floor.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.floor.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.nearbyint.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.nearbyint.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.nearbyint.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.rint.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.rint.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.rint.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.round.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.round.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.round.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.roundeven.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.roundeven.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.roundeven.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.trunc.nxv8f16( <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.trunc.nxv4f16( <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.trunc.nxv2f16( <vscale x 2 x half>)
+declare <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double>)

llvm/test/CodeGen/AArch64/sve-fp.ll

 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
   %A = load <vscale x 4 x float>, <vscale x 4 x float>* %P1, align 16
   store <vscale x 4 x float> %A, <vscale x 4 x float>* %P2, align 16
   ret void
 }
 
-; FCEIL
-
-define <vscale x 8 x half> @frintp_nxv8f16(<vscale x 8 x half> %a) {
-; CHECK-LABEL: frintp_nxv8f16:
-; CHECK: ptrue p0.h
-; CHECK-NEXT: frintp z0.h, p0/m, z0.h
-; CHECK-NEXT: ret
-  %res = call <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half> %a)
-  ret <vscale x 8 x half> %res  
-}
-
-define <vscale x 4 x half> @frintp_nxv4f16(<vscale x 4 x half> %a) {
-; CHECK-LABEL: frintp_nxv4f16:
-; CHECK: ptrue p0.s
-; CHECK-NEXT: frintp z0.h, p0/m, z0.h
-; CHECK-NEXT: ret
-  %res = call <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half> %a)
-  ret <vscale x 4 x half> %res  
-}
-
-define <vscale x 2 x half> @frintp_nxv2f16(<vscale x 2 x half> %a) {
-; CHECK-LABEL: frintp_nxv2f16:
-; CHECK: ptrue p0.d
-; CHECK-NEXT: frintp z0.h, p0/m, z0.h
-; CHECK-NEXT: ret
-  %res = call <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half> %a)
-  ret <vscale x 2 x half> %res  
-}
-
-define <vscale x 4 x float> @frintp_nxv4f32(<vscale x 4 x float> %a) {
-; CHECK-LABEL: frintp_nxv4f32:
-; CHECK: ptrue p0.s
-; CHECK-NEXT: frintp z0.s, p0/m, z0.s
-; CHECK-NEXT: ret
-  %res = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> %a)
-  ret <vscale x 4 x float> %res  
-}
-
-define <vscale x 2 x float> @frintp_nxv2f32(<vscale x 2 x float> %a) {
-; CHECK-LABEL: frintp_nxv2f32:
-; CHECK: ptrue p0.d
-; CHECK-NEXT: frintp z0.s, p0/m, z0.s
-; CHECK-NEXT: ret
-  %res = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> %a)
-  ret <vscale x 2 x float> %res  
-}
-
-define <vscale x 2 x double> @frintp_nxv2f64(<vscale x 2 x double> %a) {
-; CHECK-LABEL: frintp_nxv2f64:
-; CHECK: ptrue p0.d
-; CHECK-NEXT: frintp z0.d, p0/m, z0.d
-; CHECK-NEXT: ret
-  %res = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> %a)
-  ret <vscale x 2 x double> %res  
-}
-
 declare <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float>  @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> , <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
 
 declare <vscale x 8 x half> @llvm.aarch64.sve.frsqrts.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.frsqrts.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
 
 declare <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x float> @llvm.fma.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>)
 declare <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>, <vscale x 4 x half>)
 declare <vscale x 2 x half> @llvm.fma.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>, <vscale x 2 x half>)
 
-declare <vscale x 8 x half> @llvm.ceil.nxv8f16( <vscale x 8 x half>)
-declare <vscale x 4 x half> @llvm.ceil.nxv4f16( <vscale x 4 x half>)
-declare <vscale x 2 x half> @llvm.ceil.nxv2f16( <vscale x 2 x half>)
-declare <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float>)
-declare <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float>)
-declare <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double>)
-
 ; Function Attrs: nounwind readnone
 declare double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>) #2