[AArch64][SVE] Add patterns for signed and unsigned min/max instructions

[dancgr]

Summary

Add pattern matching for the following instructions:

• smax, smin, umax, umin

[efriedma]

Do we actually need an intrinsic for this, as opposed to adding patterns for ISD::SMAX etc.?

[dancgr]

As far as I know the ISD::SMAX only takes one input (https://llvm.org/docs/LangRef.html#llvm-experimental-vector-reduce-smax-intrinsic), in this case we need two inputs so we can do the max between the input vector and the immediate value.

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

2943

This shouldn't be here.

[efriedma]

Wong SMAX; you're referring to VECREDUCE_SMAX, which isn't relevant here. There is no IR intrinsic for ISD::SMAX; it's pattern-matched from select instructions.

[dancgr]

Update to use default ISD:SMAX variety.

[dancgr]

In D71779#1795054, @efriedma wrote:

Wong SMAX; you're referring to VECREDUCE_SMAX, which isn't relevant here. There is no IR intrinsic for ISD::SMAX; it's pattern-matched from select instructions.

That makes way more sense, I have updated the patch to reflect that.

[c-rhodes]

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

2930

Should this be zero extended (getZExtValue)?

llvm/lib/Target/AArch64/SVEInstrFormats.td

215

SVEUArithImmPat?

llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll

9

the BB check can be removed

[dancgr]

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

2930

If we use getZExtValue this returns an incorrect results when we have i8 values like 129.

llvm/lib/Target/AArch64/SVEInstrFormats.td

215

Will update this

llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll

9

Will be removing that.

[dancgr]

Updated some minor changes.

[dancgr]

@efriedma Any other remarks on this patch?

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

private:
 
   bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
 
   bool SelectCMP_SWAP(SDNode *N);
 
   bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
 
   bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm);
+
+  bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
+
+  bool SelectSVEArithImm(SDValue N, SDValue &Imm);
 };
 } // end anonymous namespace
 
 /// isIntImmediate - This method tests to see if the node is a constant
 /// operand. If so Imm will receive the 32-bit value.
 static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
   if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(N)) {
     Imm = C->getZExtValue();
if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
     default:
       break;
     }
   }
 
   return false;
 }
 
+bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
+  if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
+    int64_t ImmVal = CNode->getSExtValue();
+    SDLoc DL(N);
+    if (ImmVal >= -127 && ImmVal < 127) {
+      Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
+      return true;
+    }
+  }
+  return false;
+}
+
+bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, SDValue &Imm) {
+  if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
+    uint64_t ImmVal = CNode->getSExtValue();
+    SDLoc DL(N);
+    ImmVal = ImmVal & 0xFF;
+    if (ImmVal < 256) {
+      Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
+      return true;
+    }
+  }
+  return false;
+}
+
 bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm) {
   if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
     uint64_t ImmVal = CNode->getZExtValue();
     SDLoc DL(N);
 
     // Shift mask depending on type size.
     switch (VT.SimpleTy) {
       case MVT::i8:
         ImmVal &= 0xFF;
         ImmVal |= ImmVal << 8;
         ImmVal |= ImmVal << 16;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasSVE()) {
     addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
     addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
 
     for (auto VT : { MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64 }) {
       setOperationAction(ISD::SADDSAT, VT, Legal);
       setOperationAction(ISD::UADDSAT, VT, Legal);
       setOperationAction(ISD::SSUBSAT, VT, Legal);
       setOperationAction(ISD::USUBSAT, VT, Legal);
+      setOperationAction(ISD::SMAX, VT, Legal);
+      setOperationAction(ISD::UMAX, VT, Legal);
+      setOperationAction(ISD::SMIN, VT, Legal);
+      setOperationAction(ISD::UMIN, VT, Legal);
     }
 
     for (auto VT :
          { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
            MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
   }
 

llvm/lib/Target/AArch64/AArch64InstrFormats.td

 
 def SImm9Operand : SImmOperand<9>;
 def simm9 : Operand<i64>, ImmLeaf<i64, [{ return Imm >= -256 && Imm < 256; }]> {
   let ParserMatchClass = SImm9Operand;
   let DecoderMethod = "DecodeSImm<9>";
 }
 
 def SImm8Operand : SImmOperand<8>;
-def simm8 : Operand<i64>, ImmLeaf<i64, [{ return Imm >= -128 && Imm < 127; }]> {
+def simm8 : Operand<i32>, ImmLeaf<i32, [{ return Imm >= -128 && Imm < 127; }]> {
   let ParserMatchClass = SImm8Operand;
   let DecoderMethod = "DecodeSImm<8>";
 }
 
 def SImm6Operand : SImmOperand<6>;
 def simm6_32b : Operand<i32>, ImmLeaf<i32, [{ return Imm >= -32 && Imm < 32; }]> {
   let ParserMatchClass = SImm6Operand;
   let DecoderMethod = "DecodeSImm<6>";

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm ORV_VPZ   : sve_int_reduce_2<0b000, "orv", AArch64orv_pred>;
   defm EORV_VPZ  : sve_int_reduce_2<0b001, "eorv", AArch64eorv_pred>;
   defm ANDV_VPZ  : sve_int_reduce_2<0b010, "andv", AArch64andv_pred>;
 
   defm ORR_ZI : sve_int_log_imm<0b00, "orr", "orn", or>;
   defm EOR_ZI : sve_int_log_imm<0b01, "eor", "eon", xor>;
   defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
 
-  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", simm8>;
-  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", simm8>;
-  defm UMAX_ZI   : sve_int_arith_imm1<0b01, "umax", imm0_255>;
-  defm UMIN_ZI   : sve_int_arith_imm1<0b11, "umin", imm0_255>;
+  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", smax>;
+  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", smin>;
+  defm UMAX_ZI   : sve_int_arith_imm1_unsigned<0b01, "umax", umax>;
+  defm UMIN_ZI   : sve_int_arith_imm1_unsigned<0b11, "umin", umin>;
 
   defm MUL_ZI    : sve_int_arith_imm2<"mul">;
   defm MUL_ZPmZ   : sve_int_bin_pred_arit_2<0b000, "mul", int_aarch64_sve_mul>;
   defm SMULH_ZPmZ : sve_int_bin_pred_arit_2<0b010, "smulh", int_aarch64_sve_smulh>;
   defm UMULH_ZPmZ : sve_int_bin_pred_arit_2<0b011, "umulh", int_aarch64_sve_umulh>;
 
   defm SDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b100, "sdiv", int_aarch64_sve_sdiv>;
   defm UDIV_ZPmZ  : sve_int_bin_pred_arit_2_div<0b101, "udiv", int_aarch64_sve_udiv>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

 def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>;
 def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
 
 def SVELogicalImm8Pat  : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>;
 def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>;
 def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>;
 def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>;
 
+def SVEArithUImmPat  : ComplexPattern<i32, 1, "SelectSVEArithImm", []>;
+def SVEArithSImmPat  : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;
 
 class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass {
   let Name = "SVEExactFPImmOperand" # Suffix;
   let DiagnosticType = "Invalid" # Name;
   let ParserMethod = "tryParseFPImm<false>";
   let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">";
   let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">";
 }
class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
   : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))),
         (inst $Op1, i32:$imm, i32:$shift)>;
 
 class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
                               ValueType it, ComplexPattern cpx, Instruction inst>
   : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))),
         (inst $Op1, i32:$imm, i32:$shift)>;
 
+class SVE_1_Op_Imm_Arith_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
+                             ValueType it, ComplexPattern cpx, Instruction inst>
+  : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))),
+        (inst $Op1, i32:$imm)>;
+
 class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
                            ValueType it, ComplexPattern cpx, Instruction inst>
   : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))),
         (inst $Op1, i64:$imm)>;
 
 class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    ValueType vt2, Instruction inst>
 : Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
   let Inst{12-5} = imm;
   let Inst{4-0} = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = Destructive;
   let ElementSize = ElementSizeNone;
 }
 
-multiclass sve_int_arith_imm1<bits<2> opc, string asm, Operand immtype> {
-  def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, immtype>;
-  def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, immtype>;
-  def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, immtype>;
-  def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, immtype>;
+multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
+  def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>;
+  def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>;
+  def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>;
+  def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>;
+
+  def : SVE_1_Op_Imm_Arith_Pat<nxv16i8, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv8i16, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv4i32, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv2i64, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>;
+}
+
+multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {
+  def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>;
+  def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>;
+  def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>;
+  def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>;
+
+  def : SVE_1_Op_Imm_Arith_Pat<nxv16i8, op, ZPR8, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv8i16, op, ZPR16, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv4i32, op, ZPR32, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_Arith_Pat<nxv2i64, op, ZPR64, i64, SVEArithUImmPat, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_arith_imm2<string asm> {
   def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8,  simm8>;
   def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>;
   def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>;
   def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>;
 }

llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; SMAX
+;
+define <vscale x 16 x i8> @smax_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smax_i8_pos
+; CHECK: smax z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @smax_i8_neg(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smax_i8_neg
+; CHECK: smax z0.b, z0.b, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @smax_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smax_i16_pos
+; CHECK: smax z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @smax_i16_neg(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smax_i16_neg
+; CHECK: smax z0.h, z0.h, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @smax_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smax_i32_pos
+; CHECK: smax z0.s, z0.s, #27
+; CHECK: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @smax_i32_neg(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smax_i32_neg
+; CHECK: smax z0.s, z0.s, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @smax_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smax_i64_pos
+; CHECK: smax z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @smax_i64_neg(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smax_i64_neg
+; CHECK: smax z0.d, z0.d, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; SMIN
+;
+define <vscale x 16 x i8> @smin_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smin_i8_pos
+; CHECK: smin z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @smin_i8_neg(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: smin_i8_neg
+; CHECK: smin z0.b, z0.b, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @smin_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smin_i16_pos
+; CHECK: smin z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @smin_i16_neg(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: smin_i16_neg
+; CHECK: smin z0.h, z0.h, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @smin_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smin_i32_pos
+; CHECK: smin z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @smin_i32_neg(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: smin_i32_neg
+; CHECK: smin z0.s, z0.s, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @smin_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smin_i64_pos
+; CHECK: smin z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @smin_i64_neg(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: smin_i64_neg
+; CHECK: smin z0.d, z0.d, #-58
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; UMAX
+;
+define <vscale x 16 x i8> @umax_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umax_i8_pos
+; CHECK: umax z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @umax_i8_large(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umax_i8_large
+; CHECK: umax z0.b, z0.b, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @umax_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umax_i16_pos
+; CHECK: umax z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @umax_i16_large(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umax_i16_large
+; CHECK: umax z0.h, z0.h, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 129, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @umax_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umax_i32_pos
+; CHECK: umax z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @umax_i32_large(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umax_i32_large
+; CHECK: umax z0.s, z0.s, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 129, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @umax_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umax_i64_pos
+; CHECK: umax z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @umax_i64_large(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umax_i64_large
+; CHECK: umax z0.d, z0.d, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 129, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; UMIN
+;
+define <vscale x 16 x i8> @umin_i8_pos(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umin_i8_pos
+; CHECK: umin z0.b, z0.b, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @umin_i8_large(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: umin_i8_large
+; CHECK: umin z0.b, z0.b, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @umin_i16_pos(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umin_i16_pos
+; CHECK: umin z0.h, z0.h, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @umin_i16_large(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: umin_i16_large
+; CHECK: umin z0.h, z0.h, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 129, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @umin_i32_pos(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umin_i32_pos
+; CHECK: umin z0.s, z0.s, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @umin_i32_large(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: umin_i32_large
+; CHECK: umin z0.s, z0.s, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 129, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @umin_i64_pos(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umin_i64_pos
+; CHECK: umin z0.d, z0.d, #27
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @umin_i64_large(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: umin_i64_large
+; CHECK: umin z0.d, z0.d, #129
+; CHECK-NEXT: ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 129, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> %res
+}