[AArch64][SVE] Implement integer compare intrinsics

[c-rhodes]

Summary

Adds intrinsics for the following:

• cmphs, cmphi
• cmpge, cmpgt
• cmpeq, cmpne
• cmplt, cmple
• cmplo, cmpls

Includes a minor change to TLI.getMemValueType that fixes a crash due to the
scalable flag being dropped.

[efriedma]

No testcases for the "IR" (setcc) patterns?

[c-rhodes]

In D70889#1769773, @efriedma wrote:

No testcases for the "IR" (setcc) patterns?

Good spot, so for the signed and unsigned immediate variants (defined with sve_int_scmp_vi, sve_int_ucmp_vi) there are IR tests in sve-intrinsics-int-compares-with-imm.ll but it seems I missed a separate test we have downstream covering the non-immediate forms (defined with sve_int_cmp_0). I tried pulling that test in but it looks like it's failing for a number of reasons I don't entirely understand just yet. I think for this patch I'll remove the IR patterns for the non-immediate forms and in the meantime see what's required there for a follow-up patch.

[c-rhodes]

Remove IR patterns for non-immediate forms.

[efriedma]

LGTM

llvm/include/llvm/CodeGen/TargetLowering.h

if (PointerType *PTy = dyn_cast<PointerType>(Ty))
       return getPointerMemTy(DL, PTy->getAddressSpace());
     else if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
       Type *Elm = VTy->getElementType();
       if (PointerType *PT = dyn_cast<PointerType>(Elm)) {
         EVT PointerTy(getPointerMemTy(DL, PT->getAddressSpace()));
         Elm = PointerTy.getTypeForEVT(Ty->getContext());
       }
       return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false),
-                       VTy->getNumElements());
+                              VTy->getElementCount());
     }
 
     return getValueType(DL, Ty, AllowUnknown);
   }
 
 
   /// Return the MVT corresponding to this LLVM type. See getValueType.
   MVT getSimpleValueType(const DataLayout &DL, Type *Ty,

llvm/include/llvm/IR/IntrinsicsAArch64.td

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 
   class AdvSIMD_SVE_Compare_Intrinsic
     : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  llvm_anyvector_ty,
                  LLVMMatchType<0>],
                 [IntrNoMem]>;
 
+  class AdvSIMD_SVE_CompareWide_Intrinsic
+    : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
+                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                 llvm_anyvector_ty,
+                 llvm_nxv2i64_ty],
+                [IntrNoMem]>;
+
   class AdvSIMD_SVE_CNT_Intrinsic
     : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
                 [LLVMVectorOfBitcastsToInt<0>,
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  llvm_anyvector_ty],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_Reduce_Intrinsic

 def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
 def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
 def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
 def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
 
 //
+// Integer comparisons
+//
+
+def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
+def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
+def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
+def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
+def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
+def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
+
+def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
+
+//
 // Counting bits
 //
 
 def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
 def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
 
 //

llvm/lib/Target/AArch64/AArch64InstrFormats.td

 def Imm0_127Operand : AsmImmRange<0, 127>;
 def imm0_127 : Operand<i32>, ImmLeaf<i32, [{
   return ((uint32_t)Imm) < 128;
 }]> {
   let ParserMatchClass = Imm0_127Operand;
   let PrintMethod = "printImm";
 }
 
+def imm0_127_64b : Operand<i64>, ImmLeaf<i64, [{
+  return ((uint64_t)Imm) < 128;
+}]> {
+  let ParserMatchClass = Imm0_127Operand;
+  let PrintMethod = "printImm";
+}
+
 // NOTE: These imm0_N operands have to be of type i64 because i64 is the size
 // for all shift-amounts.
 
 // imm0_63 predicate - True if the immediate is in the range [0,63]
 def imm0_63 : Operand<i64>, ImmLeaf<i64, [{
   return ((uint64_t)Imm) < 64;
 }]> {
   let ParserMatchClass = Imm0_63Operand;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
 
   defm ZIP1_PPP : sve_int_perm_bin_perm_pp<0b000, "zip1">;
   defm ZIP2_PPP : sve_int_perm_bin_perm_pp<0b001, "zip2">;
   defm UZP1_PPP : sve_int_perm_bin_perm_pp<0b010, "uzp1">;
   defm UZP2_PPP : sve_int_perm_bin_perm_pp<0b011, "uzp2">;
   defm TRN1_PPP : sve_int_perm_bin_perm_pp<0b100, "trn1">;
   defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2">;
 
-  defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs">;
-  defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi">;
-  defm CMPGE_PPzZZ : sve_int_cmp_0<0b100, "cmpge">;
-  defm CMPGT_PPzZZ : sve_int_cmp_0<0b101, "cmpgt">;
-  defm CMPEQ_PPzZZ : sve_int_cmp_0<0b110, "cmpeq">;
-  defm CMPNE_PPzZZ : sve_int_cmp_0<0b111, "cmpne">;
-
-  defm CMPEQ_WIDE_PPzZZ : sve_int_cmp_0_wide<0b010, "cmpeq">;
-  defm CMPNE_WIDE_PPzZZ : sve_int_cmp_0_wide<0b011, "cmpne">;
-  defm CMPGE_WIDE_PPzZZ : sve_int_cmp_1_wide<0b000, "cmpge">;
-  defm CMPGT_WIDE_PPzZZ : sve_int_cmp_1_wide<0b001, "cmpgt">;
-  defm CMPLT_WIDE_PPzZZ : sve_int_cmp_1_wide<0b010, "cmplt">;
-  defm CMPLE_WIDE_PPzZZ : sve_int_cmp_1_wide<0b011, "cmple">;
-  defm CMPHS_WIDE_PPzZZ : sve_int_cmp_1_wide<0b100, "cmphs">;
-  defm CMPHI_WIDE_PPzZZ : sve_int_cmp_1_wide<0b101, "cmphi">;
-  defm CMPLO_WIDE_PPzZZ : sve_int_cmp_1_wide<0b110, "cmplo">;
-  defm CMPLS_WIDE_PPzZZ : sve_int_cmp_1_wide<0b111, "cmpls">;
-
-  defm CMPGE_PPzZI : sve_int_scmp_vi<0b000, "cmpge">;
-  defm CMPGT_PPzZI : sve_int_scmp_vi<0b001, "cmpgt">;
-  defm CMPLT_PPzZI : sve_int_scmp_vi<0b010, "cmplt">;
-  defm CMPLE_PPzZI : sve_int_scmp_vi<0b011, "cmple">;
-  defm CMPEQ_PPzZI : sve_int_scmp_vi<0b100, "cmpeq">;
-  defm CMPNE_PPzZI : sve_int_scmp_vi<0b101, "cmpne">;
-  defm CMPHS_PPzZI : sve_int_ucmp_vi<0b00, "cmphs">;
-  defm CMPHI_PPzZI : sve_int_ucmp_vi<0b01, "cmphi">;
-  defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo">;
-  defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls">;
+  defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", int_aarch64_sve_cmphs, SETUGE>;
+  defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", int_aarch64_sve_cmphi, SETUGT>;
+  defm CMPGE_PPzZZ : sve_int_cmp_0<0b100, "cmpge", int_aarch64_sve_cmpge, SETGE>;
+  defm CMPGT_PPzZZ : sve_int_cmp_0<0b101, "cmpgt", int_aarch64_sve_cmpgt, SETGT>;
+  defm CMPEQ_PPzZZ : sve_int_cmp_0<0b110, "cmpeq", int_aarch64_sve_cmpeq, SETEQ>;
+  defm CMPNE_PPzZZ : sve_int_cmp_0<0b111, "cmpne", int_aarch64_sve_cmpne, SETNE>;
+
+  defm CMPEQ_WIDE_PPzZZ : sve_int_cmp_0_wide<0b010, "cmpeq", int_aarch64_sve_cmpeq_wide>;
+  defm CMPNE_WIDE_PPzZZ : sve_int_cmp_0_wide<0b011, "cmpne", int_aarch64_sve_cmpne_wide>;
+  defm CMPGE_WIDE_PPzZZ : sve_int_cmp_1_wide<0b000, "cmpge", int_aarch64_sve_cmpge_wide>;
+  defm CMPGT_WIDE_PPzZZ : sve_int_cmp_1_wide<0b001, "cmpgt", int_aarch64_sve_cmpgt_wide>;
+  defm CMPLT_WIDE_PPzZZ : sve_int_cmp_1_wide<0b010, "cmplt", int_aarch64_sve_cmplt_wide>;
+  defm CMPLE_WIDE_PPzZZ : sve_int_cmp_1_wide<0b011, "cmple", int_aarch64_sve_cmple_wide>;
+  defm CMPHS_WIDE_PPzZZ : sve_int_cmp_1_wide<0b100, "cmphs", int_aarch64_sve_cmphs_wide>;
+  defm CMPHI_WIDE_PPzZZ : sve_int_cmp_1_wide<0b101, "cmphi", int_aarch64_sve_cmphi_wide>;
+  defm CMPLO_WIDE_PPzZZ : sve_int_cmp_1_wide<0b110, "cmplo", int_aarch64_sve_cmplo_wide>;
+  defm CMPLS_WIDE_PPzZZ : sve_int_cmp_1_wide<0b111, "cmpls", int_aarch64_sve_cmpls_wide>;
+
+  defm CMPGE_PPzZI : sve_int_scmp_vi<0b000, "cmpge", SETGE, int_aarch64_sve_cmpge>;
+  defm CMPGT_PPzZI : sve_int_scmp_vi<0b001, "cmpgt", SETGT, int_aarch64_sve_cmpgt>;
+  defm CMPLT_PPzZI : sve_int_scmp_vi<0b010, "cmplt", SETLT, null_frag, int_aarch64_sve_cmpgt>;
+  defm CMPLE_PPzZI : sve_int_scmp_vi<0b011, "cmple", SETLE, null_frag, int_aarch64_sve_cmpge>;
+  defm CMPEQ_PPzZI : sve_int_scmp_vi<0b100, "cmpeq", SETEQ, int_aarch64_sve_cmpeq>;
+  defm CMPNE_PPzZI : sve_int_scmp_vi<0b101, "cmpne", SETNE, int_aarch64_sve_cmpne>;
+  defm CMPHS_PPzZI : sve_int_ucmp_vi<0b00, "cmphs", SETUGE, int_aarch64_sve_cmphs>;
+  defm CMPHI_PPzZI : sve_int_ucmp_vi<0b01, "cmphi", SETUGT, int_aarch64_sve_cmphi>;
+  defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, null_frag, int_aarch64_sve_cmphi>;
+  defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, null_frag, int_aarch64_sve_cmphs>;
 
   defm FCMGE_PPzZZ : sve_fp_3op_p_pd<0b000, "fcmge", int_aarch64_sve_fcmpge>;
   defm FCMGT_PPzZZ : sve_fp_3op_p_pd<0b001, "fcmgt", int_aarch64_sve_fcmpgt>;
   defm FCMEQ_PPzZZ : sve_fp_3op_p_pd<0b010, "fcmeq", int_aarch64_sve_fcmpeq>;
   defm FCMNE_PPzZZ : sve_fp_3op_p_pd<0b011, "fcmne", int_aarch64_sve_fcmpne>;
   defm FCMUO_PPzZZ : sve_fp_3op_p_pd<0b100, "fcmuo", int_aarch64_sve_fcmpuo>;
   defm FACGE_PPzZZ : sve_fp_3op_p_pd<0b101, "facge", int_aarch64_sve_facge>;
   defm FACGT_PPzZZ : sve_fp_3op_p_pd<0b111, "facgt", int_aarch64_sve_facgt>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm),
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
 }
 
-multiclass sve_int_cmp_0<bits<3> opc, string asm> {
+multiclass sve_int_cmp_0<bits<3> opc, string asm, SDPatternOperator op,
+                         CondCode cc> {
   def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>;
   def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>;
   def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>;
   def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>;
+
+  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4i1,  op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_3_Op_Pat<nxv2i1,  op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_cmp_0_wide<bits<3> opc, string asm> {
+multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;
   def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;
   def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;
+
+  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
+  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4i1,  op, nxv4i1,  nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
 }
 
-multiclass sve_int_cmp_1_wide<bits<3> opc, string asm> {
+multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;
   def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;
   def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;
+
+  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
+  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4i1,  op, nxv4i1,  nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
 }
 
 
 //===----------------------------------------------------------------------===//
 // SVE Integer Compare - Signed Immediate Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5),
   let Inst{14}    = 0b0;
   let Inst{13}    = opc{1};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
+  let ElementSize = pprty.ElementSize;
 }
 
-multiclass sve_int_scmp_vi<bits<3> opc, string asm> {
+multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc,
+                           SDPatternOperator op = null_frag,
+                           SDPatternOperator inv_op = null_frag> {
   def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>;
   def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>;
   def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>;
   def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>;
+
+  // IR version
+  def : Pat<(nxv16i1 (setcc (nxv16i8 ZPR:$Zs1),
+                             (nxv16i8 (AArch64dup (simm5_32b:$imm))),
+                             cc)),
+             (!cast<Instruction>(NAME # "_B") (PTRUE_B 31), ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv8i1 (setcc (nxv8i16 ZPR:$Zs1),
+                            (nxv8i16 (AArch64dup (simm5_32b:$imm))),
+                            cc)),
+             (!cast<Instruction>(NAME # "_H") (PTRUE_H 31), ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv4i1 (setcc (nxv4i32 ZPR:$Zs1),
+                            (nxv4i32 (AArch64dup (simm5_32b:$imm))),
+                            cc)),
+             (!cast<Instruction>(NAME # "_S") (PTRUE_S 31), ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv2i1 (setcc (nxv2i64 ZPR:$Zs1),
+                           (nxv2i64 (AArch64dup (simm5_64b:$imm))),
+                           cc)),
+            (!cast<Instruction>(NAME # "_D") (PTRUE_D 31), ZPR:$Zs1, simm5_64b:$imm)>;
+
+  // Intrinsic version
+  def : Pat<(nxv16i1 (op (nxv16i1 PPR_3b:$Pg),
+                         (nxv16i8 ZPR:$Zs1),
+                         (nxv16i8 (AArch64dup (simm5_32b:$imm))))),
+            (!cast<Instruction>(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv8i1 (op (nxv8i1 PPR_3b:$Pg),
+                        (nxv8i16 ZPR:$Zs1),
+                        (nxv8i16 (AArch64dup (simm5_32b:$imm))))),
+            (!cast<Instruction>(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv4i1 (op (nxv4i1 PPR_3b:$Pg),
+                        (nxv4i32 ZPR:$Zs1),
+                        (nxv4i32 (AArch64dup (simm5_32b:$imm))))),
+            (!cast<Instruction>(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv2i1 (op (nxv2i1 PPR_3b:$Pg),
+                        (nxv2i64 ZPR:$Zs1),
+                        (nxv2i64 (AArch64dup (simm5_64b:$imm))))),
+            (!cast<Instruction>(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, simm5_64b:$imm)>;
+
+  // Inverted intrinsic version
+  def : Pat<(nxv16i1 (inv_op (nxv16i1 PPR_3b:$Pg),
+                             (nxv16i8 (AArch64dup (simm5_32b:$imm))),
+                             (nxv16i8 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv8i1 (inv_op (nxv8i1 PPR_3b:$Pg),
+                            (nxv8i16 (AArch64dup (simm5_32b:$imm))),
+                            (nxv8i16 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv4i1 (inv_op (nxv4i1 PPR_3b:$Pg),
+                            (nxv4i32 (AArch64dup (simm5_32b:$imm))),
+                            (nxv4i32 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>;
+  def : Pat<(nxv2i1 (inv_op (nxv2i1 PPR_3b:$Pg),
+                            (nxv2i64 (AArch64dup (simm5_64b:$imm))),
+                            (nxv2i64 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, simm5_64b:$imm)>;
 }
 
 
 //===----------------------------------------------------------------------===//
 // SVE Integer Compare - Unsigned Immediate Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty,
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7),
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
 }
 
-multiclass sve_int_ucmp_vi<bits<2> opc, string asm> {
+multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc,
+                           SDPatternOperator op = null_frag,
+                           SDPatternOperator inv_op = null_frag> {
   def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>;
   def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>;
   def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>;
-  def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127>;
+  def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>;
+
+  // IR version
+  def : Pat<(nxv16i1 (setcc (nxv16i8 ZPR:$Zs1),
+                            (nxv16i8 (AArch64dup (imm0_127:$imm))),
+                            cc)),
+            (!cast<Instruction>(NAME # "_B") (PTRUE_B 31), ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv8i1 (setcc (nxv8i16 ZPR:$Zs1),
+                           (nxv8i16 (AArch64dup (imm0_127:$imm))),
+                           cc)),
+            (!cast<Instruction>(NAME # "_H") (PTRUE_H 31), ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv4i1 (setcc (nxv4i32 ZPR:$Zs1),
+                           (nxv4i32 (AArch64dup (imm0_127:$imm))),
+                           cc)),
+            (!cast<Instruction>(NAME # "_S") (PTRUE_S 31), ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv2i1 (setcc (nxv2i64 ZPR:$Zs1),
+                           (nxv2i64 (AArch64dup (imm0_127_64b:$imm))),
+                           cc)),
+            (!cast<Instruction>(NAME # "_D") (PTRUE_D 31), ZPR:$Zs1, imm0_127_64b:$imm)>;
+
+  // Intrinsic version
+  def : Pat<(nxv16i1 (op (nxv16i1 PPR_3b:$Pg),
+                         (nxv16i8 ZPR:$Zs1),
+                         (nxv16i8 (AArch64dup (imm0_127:$imm))))),
+            (!cast<Instruction>(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv8i1 (op (nxv8i1 PPR_3b:$Pg),
+                        (nxv8i16 ZPR:$Zs1),
+                        (nxv8i16 (AArch64dup (imm0_127:$imm))))),
+            (!cast<Instruction>(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv4i1 (op (nxv4i1 PPR_3b:$Pg),
+                        (nxv4i32 ZPR:$Zs1),
+                        (nxv4i32 (AArch64dup (imm0_127:$imm))))),
+            (!cast<Instruction>(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv2i1 (op (nxv2i1 PPR_3b:$Pg),
+                        (nxv2i64 ZPR:$Zs1),
+                        (nxv2i64 (AArch64dup (imm0_127_64b:$imm))))),
+            (!cast<Instruction>(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, imm0_127_64b:$imm)>;
+
+  // Inverted intrinsic version
+  def : Pat<(nxv16i1 (inv_op (nxv16i1 PPR_3b:$Pg),
+                             (nxv16i8 (AArch64dup (imm0_127:$imm))),
+                             (nxv16i8 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv8i1 (inv_op (nxv8i1 PPR_3b:$Pg),
+                            (nxv8i16 (AArch64dup (imm0_127:$imm))),
+                            (nxv8i16 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv4i1 (inv_op (nxv4i1 PPR_3b:$Pg),
+                            (nxv4i32 (AArch64dup (imm0_127:$imm))),
+                            (nxv4i32 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>;
+  def : Pat<(nxv2i1 (inv_op (nxv2i1 PPR_3b:$Pg),
+                            (nxv2i64 (AArch64dup (imm0_127_64b:$imm))),
+                            (nxv2i64 ZPR:$Zs1))),
+            (!cast<Instruction>(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, imm0_127_64b:$imm)>;
 }
 
 
 //===----------------------------------------------------------------------===//
 // SVE Integer Compare - Scalars Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt>

llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;                             Signed Comparisons                             ;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;
+; CMPEQ
+;
+
+define <vscale x 16 x i1> @ir_cmpeq_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpeq_b
+; CHECK: cmpeq p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp eq <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpeq_b
+; CHECK: cmpeq p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpeq_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpeq_h
+; CHECK: cmpeq p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp eq <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpeq_h
+; CHECK: cmpeq p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpeq_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpeq_s
+; CHECK: cmpeq p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp eq <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpeq_s
+; CHECK: cmpeq p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpeq_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpeq_d
+; CHECK: cmpeq p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp eq <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpeq_d
+; CHECK: cmpeq p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPGE
+;
+
+define <vscale x 16 x i1> @ir_cmpge_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpge_b
+; CHECK: cmpge p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sge <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpge_b
+; CHECK: cmpge p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpge_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpge_h
+; CHECK: cmpge p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sge <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpge_h
+; CHECK: cmpge p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpge_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpge_s
+; CHECK: cmpge p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sge <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpge_s
+; CHECK: cmpge p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpge_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpge_d
+; CHECK: cmpge p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sge <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpge_d
+; CHECK: cmpge p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPGT
+;
+
+define <vscale x 16 x i1> @ir_cmpgt_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpgt_b
+; CHECK: cmpgt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpgt_b
+; CHECK: cmpgt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpgt_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpgt_h
+; CHECK: cmpgt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpgt_h
+; CHECK: cmpgt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpgt_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpgt_s
+; CHECK: cmpgt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpgt_s
+; CHECK: cmpgt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpgt_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpgt_d
+; CHECK: cmpgt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sgt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpgt_d
+; CHECK: cmpgt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLE
+;
+
+define <vscale x 16 x i1> @ir_cmple_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmple_b
+; CHECK: cmple p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp sle <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmple_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmple_b
+; CHECK: cmple p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmple_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmple_h
+; CHECK: cmple p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp sle <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmple_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmple_h
+; CHECK: cmple p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmple_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmple_s
+; CHECK: cmple p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp sle <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmple_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmple_s
+; CHECK: cmple p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmple_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmple_d
+; CHECK: cmple p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp sle <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmple_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmple_d
+; CHECK: cmple p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLT
+;
+
+define <vscale x 16 x i1> @ir_cmplt_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmplt_b
+; CHECK: cmplt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp slt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmplt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmplt_b
+; CHECK: cmplt p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmplt_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmplt_h
+; CHECK: cmplt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp slt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmplt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmplt_h
+; CHECK: cmplt p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmplt_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmplt_s
+; CHECK: cmplt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp slt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmplt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmplt_s
+; CHECK: cmplt p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmplt_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmplt_d
+; CHECK: cmplt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp slt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmplt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmplt_d
+; CHECK: cmplt p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPNE
+;
+
+define <vscale x 16 x i1> @ir_cmpne_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpne_b
+; CHECK: cmpne p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ne <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpne_b
+; CHECK: cmpne p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpne_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpne_h
+; CHECK: cmpne p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ne <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpne_h
+; CHECK: cmpne p0.h, p0/z, z0.h, #-16
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpne_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpne_s
+; CHECK: cmpne p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ne <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpne_s
+; CHECK: cmpne p0.s, p0/z, z0.s, #15
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpne_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpne_d
+; CHECK: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ne <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpne_d
+; CHECK: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;                            Unsigned Comparisons                            ;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;
+; CMPHI
+;
+
+define <vscale x 16 x i1> @ir_cmphi_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmphi_b
+; CHECK: cmphi p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmphi_b
+; CHECK: cmphi p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmphi_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmphi_h
+; CHECK: cmphi p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmphi_h
+; CHECK: cmphi p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmphi_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmphi_s
+; CHECK: cmphi p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmphi_s
+; CHECK: cmphi p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmphi_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmphi_d
+; CHECK: cmphi p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ugt <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmphi_d
+; CHECK: cmphi p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPHS
+;
+
+define <vscale x 16 x i1> @ir_cmphs_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmphs_b
+; CHECK: cmphs p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp uge <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmphs_b
+; CHECK: cmphs p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmphs_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmphs_h
+; CHECK: cmphs p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp uge <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmphs_h
+; CHECK: cmphs p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmphs_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmphs_s
+; CHECK: cmphs p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp uge <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmphs_s
+; CHECK: cmphs p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmphs_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmphs_d
+; CHECK: cmphs p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp uge <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmphs_d
+; CHECK: cmphs p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLO
+;
+
+define <vscale x 16 x i1> @ir_cmplo_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmplo_b
+; CHECK: cmplo p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ult <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmplo_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmplo_b
+; CHECK: cmplo p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmplo_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmplo_h
+; CHECK: cmplo p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmplo_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmplo_s
+; CHECK: cmplo p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ult <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmplo_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmplo_s
+; CHECK: cmplo p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmplo_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmplo_d
+; CHECK: cmplo p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ult <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmplo_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmplo_d
+; CHECK: cmplo p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+;
+; CMPLS
+;
+
+define <vscale x 16 x i1> @ir_cmpls_b(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: ir_cmpls_b
+; CHECK: cmpls p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = icmp ule <vscale x 16 x i8> %a, %splat
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @int_cmpls_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: int_cmpls_b
+; CHECK: cmpls p0.b, p0/z, z0.b, #4
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ir_cmpls_h(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: ir_cmpls_h
+; CHECK: cmpls p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = icmp ule <vscale x 8 x i16> %a, %splat
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @int_cmpls_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: int_cmpls_h
+; CHECK: cmpls p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ir_cmpls_s(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: ir_cmpls_s
+; CHECK: cmpls p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = icmp ule <vscale x 4 x i32> %a, %splat
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @int_cmpls_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: int_cmpls_s
+; CHECK: cmpls p0.s, p0/z, z0.s, #68
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ir_cmpls_d(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: ir_cmpls_d
+; CHECK: cmpls p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = icmp ule <vscale x 2 x i64> %a, %splat
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @int_cmpls_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: int_cmpls_d
+; CHECK: cmpls p0.d, p0/z, z0.d, #127
+; CHECK-NEXT: ret
+  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)

llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CMPEQ
+;
+
+define <vscale x 16 x i1> @cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpeq_b:
+; CHECK: cmpeq p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpeq_h:
+; CHECK: cmpeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpeq_s:
+; CHECK: cmpeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_d:
+; CHECK: cmpeq p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpeq_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_b:
+; CHECK: cmpeq p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpeq_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_h:
+; CHECK: cmpeq p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpeq_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpeq_wide_s:
+; CHECK: cmpeq p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPGE
+;
+
+define <vscale x 16 x i1> @cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpge_b:
+; CHECK: cmpge p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpge_h:
+; CHECK: cmpge p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpge_s:
+; CHECK: cmpge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_d:
+; CHECK: cmpge p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpge_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_b:
+; CHECK: cmpge p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpge_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_h:
+; CHECK: cmpge p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpge_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpge_wide_s:
+; CHECK: cmpge p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPGT
+;
+
+define <vscale x 16 x i1> @cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpgt_b:
+; CHECK: cmpgt p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpgt_h:
+; CHECK: cmpgt p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpgt_s:
+; CHECK: cmpgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_d:
+; CHECK: cmpgt p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpgt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_b:
+; CHECK: cmpgt p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpgt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_h:
+; CHECK: cmpgt p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpgt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpgt_wide_s:
+; CHECK: cmpgt p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPHI
+;
+
+define <vscale x 16 x i1> @cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmphi_b:
+; CHECK: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmphi_h:
+; CHECK: cmphi p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmphi_s:
+; CHECK: cmphi p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_d:
+; CHECK: cmphi p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmphi_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_b:
+; CHECK: cmphi p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphi_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_h:
+; CHECK: cmphi p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphi_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphi_wide_s:
+; CHECK: cmphi p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPHS
+;
+
+define <vscale x 16 x i1> @cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmphs_b:
+; CHECK: cmphs p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmphs_h:
+; CHECK: cmphs p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmphs_s:
+; CHECK: cmphs p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_d:
+; CHECK: cmphs p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmphs_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_b:
+; CHECK: cmphs p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmphs_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_h:
+; CHECK: cmphs p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmphs_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmphs_wide_s:
+; CHECK: cmphs p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLE
+;
+
+define <vscale x 16 x i1> @cmple_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_b:
+; CHECK: cmple p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmple_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_h:
+; CHECK: cmple p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmple_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmple_wide_s:
+; CHECK: cmple p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLO
+;
+
+define <vscale x 16 x i1> @cmplo_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_b:
+; CHECK: cmplo p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmplo_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_h:
+; CHECK: cmplo p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmplo_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplo_wide_s:
+; CHECK: cmplo p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLS
+;
+
+define <vscale x 16 x i1> @cmpls_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_b:
+; CHECK: cmpls p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpls_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_h:
+; CHECK: cmpls p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpls_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpls_wide_s:
+; CHECK: cmpls p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPLT
+;
+
+define <vscale x 16 x i1> @cmplt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_b:
+; CHECK: cmplt p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmplt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_h:
+; CHECK: cmplt p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmplt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmplt_wide_s:
+; CHECK: cmplt p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+;
+; CMPNE
+;
+
+define <vscale x 16 x i1> @cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cmpne_b:
+; CHECK: cmpne p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                 <vscale x 16 x i8> %a,
+                                                                 <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cmpne_h:
+; CHECK: cmpne p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cmpne_s:
+; CHECK: cmpne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_d:
+; CHECK: cmpne p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                <vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 16 x i1> @cmpne_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_b:
+; CHECK: cmpne p0.b, p0/z, z0.b, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                      <vscale x 16 x i8> %a,
+                                                                      <vscale x 2 x i64> %b)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @cmpne_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_h:
+; CHECK: cmpne p0.h, p0/z, z0.h, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                     <vscale x 8 x i16> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @cmpne_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cmpne_wide_s:
+; CHECK: cmpne p0.s, p0/z, z0.s, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x i32> %a,
+                                                                     <vscale x 2 x i64> %b)
+  ret <vscale x 4 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)