[AArch64][SVE] Add FP unpredicated to predicated two-op codegen

[dancgr]

Summary

This patch changes the predicated fmin, fmax, fminnm, fmaxnm from intrinsics to custom ISD nodes for lowering. These custom ISD nodes are then used to lower unpredicated versions of those instructions to be lowered to predicated ones.

I initially created an unpredicated intrinsic for this, because I didn't find any other way of generating the unpredicated fmin, fmax, etc. So I would appreciate any suggestions as to how to generate those unpredicated instructions without having to add more intrinsics.

[cameron.mcinally]

I initially created an unpredicated intrinsic for this, because I didn't find any other way of generating the unpredicated fmin, fmax, etc. So I would appreciate any suggestions as to how to generate those unpredicated instructions without having to add more intrinsics.

I might be missing the motivation for the unpredicated intrinsic, but in the non-scalable world an unpredicated operation would be canonicalized as a predicated intrinsic with all-1s mask. That would then be lowered during ISel. Something like:

%mask = shufflevector <vscale x 4 x i1> 1, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
%res = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(
  <vscale x 4 x i1> %mask
  <vscale x 4 x float> %a,
  <vscale x 4 x float> %b)

X86 does something similar in X86ISelLowering.cpp:getVectorMaskingNode(...), although it's not really apples-to-apples since X86's masks are represented differently.

llvm/include/llvm/IR/IntrinsicsAArch64.td

1749

Unnecessary whitespace change?

There are a couple of these in this patch.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1579

Was this #undef intentionally removed?

11909

This looks good, but it may be table-worthy, if this is the way forward. X86 does something similar in llvm/lib/Target/X86/X86IntrinsicsInfo.h. Just a heads up.

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
 def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
 def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
 def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
 def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
+
+def int_aarch64_sve_fmax_unpred   : AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_fmin_unpred   : AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_fmaxnm_unpred : AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_fminnm_unpred : AdvSIMD_2VectorArg_Intrinsic;
+
 def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
+
+
+
 def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
 def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
 def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   SMAXV_PRED,
   UMAXV_PRED,
   SMINV_PRED,
   UMINV_PRED,
   ORV_PRED,
   EORV_PRED,
   ANDV_PRED,
 
+  FMAX,
+  FMIN,
+  FMAXNM,
+  FMINNM,
+
   // Vector bitwise negation
   NOT,
 
   // Vector bitwise insertion
   BIT,
 
   // Compare-and-branch
   CBZ,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
     MAKE_CASE(AArch64ISD::SSTNT1_PRED)
     MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
     MAKE_CASE(AArch64ISD::LDP)
     MAKE_CASE(AArch64ISD::STP)
     MAKE_CASE(AArch64ISD::STNP)
     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
+    MAKE_CASE(AArch64ISD::FMAX)
+    MAKE_CASE(AArch64ISD::FMIN)
+    MAKE_CASE(AArch64ISD::FMAXNM)
+    MAKE_CASE(AArch64ISD::FMINNM)
   }
-#undef MAKE_CASE
   return nullptr;
 }
 
 MachineBasicBlock *
 AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
                                     MachineBasicBlock *MBB) const {
   // We materialise the F128CSEL pseudo-instruction as some control flow and a
   // phi node:
 
SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
   SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
   SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
 
   // create the vector Val[idx64],Val[idx64+1],Val[idx64],Val[idx64+1],...
   SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
   return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
 }
 
+static SDValue combineSVEPredIntrinsic(unsigned Opc, SDNode *N, SelectionDAG &DAG) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  SDValue Vector1 = N->getOperand(1);
+  SDValue Pred = N->getOperand(2);
+  SDValue Vector2 = N->getOperand(3);
+
+  return DAG.getNode(Opc, DL, VT, Vector1, Pred, Vector2);
+}
+
+static SDValue combineToSVEPred(SDNode *N, SelectionDAG &DAG, unsigned NewOp) {
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  assert(N->getOperand(1).getValueType().isScalableVector() &&
+         N->getOperand(2).getValueType().isScalableVector() &&
+         "Only scalable vectors are supported");
+
+  auto PredTy =
+      VT.getVectorVT(*DAG.getContext(), MVT::i1, VT.getVectorElementCount());
+  SDValue Mask = getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
+
+  SmallVector<SDValue, 4> Operands = {Mask, N->getOperand(1), N->getOperand(2)};
+  return DAG.getNode(NewOp, DL, VT, Operands);
+}
 
 static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits,
                                APInt &UndefBits) {
   EVT VT = BVN->getValueType(0);
   APInt SplatBits, SplatUndef;
   unsigned SplatBitSize;
   bool HasAnyUndefs;
   if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
case Intrinsic::aarch64_sve_ptest_any:
     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
                     AArch64CC::ANY_ACTIVE);
   case Intrinsic::aarch64_sve_ptest_first:
     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
                     AArch64CC::FIRST_ACTIVE);
   case Intrinsic::aarch64_sve_ptest_last:
     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
                     AArch64CC::LAST_ACTIVE);
+  case Intrinsic::aarch64_sve_fmax:
+    return combineSVEPredIntrinsic(AArch64ISD::FMAX, N, DAG);
+  case Intrinsic::aarch64_sve_fmin:
+    return combineSVEPredIntrinsic(AArch64ISD::FMIN, N, DAG);
+  case Intrinsic::aarch64_sve_fmaxnm:
+    return combineSVEPredIntrinsic(AArch64ISD::FMAXNM, N, DAG);
+  case Intrinsic::aarch64_sve_fminnm:
+    return combineSVEPredIntrinsic(AArch64ISD::FMINNM, N, DAG);
+  case Intrinsic::aarch64_sve_fmax_unpred:
+    return combineToSVEPred(N, DAG, AArch64ISD::FMAX);
+  case Intrinsic::aarch64_sve_fmin_unpred:
+    return combineToSVEPred(N, DAG, AArch64ISD::FMIN);
+  case Intrinsic::aarch64_sve_fmaxnm_unpred:
+    return combineToSVEPred(N, DAG, AArch64ISD::FMAXNM);
+  case Intrinsic::aarch64_sve_fminnm_unpred:
+    return combineToSVEPred(N, DAG, AArch64ISD::FMINNM);
   }
+
   return SDValue();
 }
 
 static SDValue performExtendCombine(SDNode *N,
                                     TargetLowering::DAGCombinerInfo &DCI,
                                     SelectionDAG &DAG) {
   // If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
   // we can convert that DUP into another extract_high (of a bigger DUP), which

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 
 def AArch64saddv    : SDNode<"AArch64ISD::SADDV", SDT_AArch64UnaryVec>;
 def AArch64uaddv    : SDNode<"AArch64ISD::UADDV", SDT_AArch64UnaryVec>;
 def AArch64sminv    : SDNode<"AArch64ISD::SMINV", SDT_AArch64UnaryVec>;
 def AArch64uminv    : SDNode<"AArch64ISD::UMINV", SDT_AArch64UnaryVec>;
 def AArch64smaxv    : SDNode<"AArch64ISD::SMAXV", SDT_AArch64UnaryVec>;
 def AArch64umaxv    : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>;
 
+
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
 def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stzg : SDNode<"AArch64ISD::STZG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64st2g : SDNode<"AArch64ISD::ST2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stz2g : SDNode<"AArch64ISD::STZ2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
 def SDT_AArch64unpk : SDTypeProfile<1, 1, [
     SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<1, 0>

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 def SDT_AArch64DUP_PRED  : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 3>, SDTCisVec<1>, SDTCVecEltisVT<1,i1>]>;
 def AArch64dup_mt : SDNode<"AArch64ISD::DUP_MERGE_PASSTHRU", SDT_AArch64DUP_PRED>;
 
 def SDT_IndexVector : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<1, 2>, SDTCisInt<2>]>;
 def index_vector : SDNode<"AArch64ISD::INDEX_VECTOR", SDT_IndexVector, []>;
 
 def reinterpret_cast : SDNode<"AArch64ISD::REINTERPRET_CAST", SDTUnaryOp>;
 
+def SDT_AArch64PredBinFp : SDTypeProfile<1, 3, [SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>]>;
+def AArch64fmax     : SDNode<"AArch64ISD::FMAX", SDT_AArch64PredBinFp>;
+def AArch64fmin     : SDNode<"AArch64ISD::FMIN", SDT_AArch64PredBinFp>;
+def AArch64fmaxnm   : SDNode<"AArch64ISD::FMAXNM", SDT_AArch64PredBinFp>;
+def AArch64fminnm   : SDNode<"AArch64ISD::FMINNM", SDT_AArch64PredBinFp>;
+
 let Predicates = [HasSVE] in {
   defm RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
   def  RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
   defm RDFFR_P    : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>;
   def  SETFFR     : sve_int_setffr<"setffr", int_aarch64_sve_setffr>;
   def  WRFFR      : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>;
 
   defm ADD_ZZZ   : sve_int_bin_cons_arit_0<0b000, "add", add, null_frag>;
let Predicates = [HasSVE, HasBF16] in {
   defm FMINNM_ZPmI  : sve_fp_2op_i_p_zds<0b101, "fminnm", sve_fpimm_zero_one>;
   defm FMAX_ZPmI    : sve_fp_2op_i_p_zds<0b110, "fmax", sve_fpimm_zero_one>;
   defm FMIN_ZPmI    : sve_fp_2op_i_p_zds<0b111, "fmin", sve_fpimm_zero_one>;
 
   defm FADD_ZPmZ   : sve_fp_2op_p_zds<0b0000, "fadd", "FADD_ZPZZ", int_aarch64_sve_fadd, DestructiveBinaryComm>;
   defm FSUB_ZPmZ   : sve_fp_2op_p_zds<0b0001, "fsub", "FSUB_ZPZZ", int_aarch64_sve_fsub, DestructiveBinaryCommWithRev, "FSUBR_ZPmZ", 1>;
   defm FMUL_ZPmZ   : sve_fp_2op_p_zds<0b0010, "fmul", "FMUL_ZPZZ", int_aarch64_sve_fmul, DestructiveBinaryComm>;
   defm FSUBR_ZPmZ  : sve_fp_2op_p_zds<0b0011, "fsubr", "FSUBR_ZPZZ", int_aarch64_sve_fsubr, DestructiveBinaryCommWithRev, "FSUB_ZPmZ", 0>;
-  defm FMAXNM_ZPmZ : sve_fp_2op_p_zds<0b0100, "fmaxnm", "FMAXNM_ZPZZ", int_aarch64_sve_fmaxnm, DestructiveBinaryComm>;
-  defm FMINNM_ZPmZ : sve_fp_2op_p_zds<0b0101, "fminnm", "FMINNM_ZPZZ", int_aarch64_sve_fminnm, DestructiveBinaryComm>;
-  defm FMAX_ZPmZ   : sve_fp_2op_p_zds<0b0110, "fmax", "FMAX_ZPZZ", int_aarch64_sve_fmax, DestructiveBinaryComm>;
-  defm FMIN_ZPmZ   : sve_fp_2op_p_zds<0b0111, "fmin", "FMIN_ZPZZ", int_aarch64_sve_fmin, DestructiveBinaryComm>;
+  defm FMAXNM_ZPmZ : sve_fp_2op_p_zds<0b0100, "fmaxnm", "FMAXNM_ZPZZ", AArch64fmaxnm, DestructiveBinaryComm>;
+  defm FMINNM_ZPmZ : sve_fp_2op_p_zds<0b0101, "fminnm", "FMINNM_ZPZZ", AArch64fminnm, DestructiveBinaryComm>;
+  defm FMAX_ZPmZ   : sve_fp_2op_p_zds<0b0110, "fmax", "FMAX_ZPZZ", AArch64fmax, DestructiveBinaryComm>;
+  defm FMIN_ZPmZ   : sve_fp_2op_p_zds<0b0111, "fmin", "FMIN_ZPZZ", AArch64fmin, DestructiveBinaryComm>;
   defm FABD_ZPmZ   : sve_fp_2op_p_zds<0b1000, "fabd", "FABD_ZPZZ", int_aarch64_sve_fabd, DestructiveBinaryComm>;
   defm FSCALE_ZPmZ : sve_fp_2op_p_zds_fscale<0b1001, "fscale", int_aarch64_sve_fscale>;
   defm FMULX_ZPmZ  : sve_fp_2op_p_zds<0b1010, "fmulx", "FMULX_ZPZZ", int_aarch64_sve_fmulx, DestructiveBinaryComm>;
   defm FDIVR_ZPmZ  : sve_fp_2op_p_zds<0b1100, "fdivr", "FDIVR_ZPZZ", int_aarch64_sve_fdivr, DestructiveBinaryCommWithRev, "FDIV_ZPmZ", 0>;
   defm FDIV_ZPmZ   : sve_fp_2op_p_zds<0b1101, "fdiv", "FDIV_ZPZZ", int_aarch64_sve_fdiv, DestructiveBinaryCommWithRev, "FDIVR_ZPmZ", 1>;
 
   defm FADD_ZPZZ   : sve_fp_bin_pred_hfd<AArch64fadd_p>;
 
   defm FADD_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fadd>;
   defm FSUB_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fsub>;
   defm FMUL_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmul>;
   defm FSUBR_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fsubr>;
-  defm FMAXNM_ZPZZ : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmaxnm>;
-  defm FMINNM_ZPZZ : sve_fp_2op_p_zds_zx<int_aarch64_sve_fminnm>;
-  defm FMAX_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmax>;
-  defm FMIN_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmin>;
+  defm FMAXNM_ZPZZ : sve_fp_2op_p_zds_zx<AArch64fmaxnm>;
+  defm FMINNM_ZPZZ : sve_fp_2op_p_zds_zx<AArch64fminnm>;
+  defm FMAX_ZPZZ   : sve_fp_2op_p_zds_zx<AArch64fmax>;
+  defm FMIN_ZPZZ   : sve_fp_2op_p_zds_zx<AArch64fmin>;
   defm FABD_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fabd>;
   defm FMULX_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fmulx>;
   defm FDIVR_ZPZZ  : sve_fp_2op_p_zds_zx<int_aarch64_sve_fdivr>;
   defm FDIV_ZPZZ   : sve_fp_2op_p_zds_zx<int_aarch64_sve_fdiv>;
 
   defm FADD_ZZZ    : sve_fp_3op_u_zd<0b000, "fadd",    fadd>;
   defm FSUB_ZZZ    : sve_fp_3op_u_zd<0b001, "fsub",    fsub>;
   defm FMUL_ZZZ    : sve_fp_3op_u_zd<0b010, "fmul",    fmul>;

llvm/test/CodeGen/AArch64/sve-fp-unpred-to-pred.ll

+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; FMAX
+
+define <vscale x 8 x half> @fmax_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmax_half:
+; CHECK: fmax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.unpred.nxv8f16(
+    <vscale x 8 x half> %a,
+    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fmax_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmax_float:
+; CHECK: fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.unpred.nxv4f32(
+    <vscale x 4 x float> %a,
+    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fmax_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmax_double:
+; CHECK: fmax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.unpred.nxv2f64(
+    <vscale x 2 x double> %a,
+    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+; FMINNM
+
+define <vscale x 8 x half> @fmin_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmin_half:
+; CHECK: fmin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.unpred.nxv8f16(
+    <vscale x 8 x half> %a,
+    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fmin_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmin_float:
+; CHECK: fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.unpred.nxv4f32(
+    <vscale x 4 x float> %a,
+    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fmin_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmin_double:
+; CHECK: fmin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.unpred.nxv2f64(
+    <vscale x 2 x double> %a,
+    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+; FMAXNM
+
+define <vscale x 8 x half> @fmaxnm_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmaxnm_half:
+; CHECK: fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.unpred.nxv8f16(
+    <vscale x 8 x half> %a,
+    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fmaxnm_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmaxnm_float:
+; CHECK: fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.unpred.nxv4f32(
+    <vscale x 4 x float> %a,
+    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fmaxnm_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmaxnm_double:
+; CHECK: fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.unpred.nxv2f64(
+    <vscale x 2 x double> %a,
+    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+; FMINNM
+
+define <vscale x 8 x half> @fminnm_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fminnm_half:
+; CHECK: fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.unpred.nxv8f16(
+    <vscale x 8 x half> %a,
+    <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fminnm_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fminnm_float:
+; CHECK: fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.unpred.nxv4f32(
+    <vscale x 4 x float> %a,
+    <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fminnm_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fminnm_double:
+; CHECK: fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.unpred.nxv2f64(
+    <vscale x 2 x double> %a,
+    <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmax.unpred.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmax.unpred.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmax.unpred.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmin.unpred.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmin.unpred.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmin.unpred.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.unpred.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.unpred.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.unpred.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fminnm.unpred.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fminnm.unpred.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fminnm.unpred.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)