[AArch64][SME2/SVE2p1] Add predicate-as-counter intrinsics for while*

[sdesmalen]

Summary

These intrinsics are used to implement the while intrinsics that result
in a predicate-as-counter value, e.g.

__attribute__((arm_streaming))
svcount_t svwhilelt_c8(int64_t rn, int64_t rm, uint64_t vl)

As described in https://github.com/ARM-software/acle/pull/217

llvm/include/llvm/IR/IntrinsicsAArch64.td

def int_aarch64_sve_cntp_c16
                               [IntrNoMem, ImmArg<ArgIndex<1>>]>;
   def int_aarch64_sve_cntp_c32
       : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty],
                               [IntrNoMem, ImmArg<ArgIndex<1>>]>;
   def int_aarch64_sve_cntp_c64
       : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty],
                               [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
+  // While (predicate-as-counter) intrinsics
+  foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in {
+    foreach ty = ["c8", "c16", "c32", "c64"] in {
+      def int_aarch64_sve_while # cmp # _ # ty
+          : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty],
+                                  [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty],
+                                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+    }
+  }
+
   //
   // SME2 Intrinsics
   //
 
   class SME2_Matrix_ArrayVector_Single_Single_Intrinsic
     : DefaultAttrsIntrinsic<[],
                 [llvm_i32_ty,
                  llvm_anyvector_ty, LLVMMatchType<0>],

llvm/lib/Target/AArch64/SVEInstrFormats.td

: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))),
       (inst $Op1, ImmTy:$Op2)>;
 
 multiclass SVE2p1_Cntp_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                            Instruction inst> {
   def : Pat<(vtd (op vt1:$Op1, (i32 2))), (inst $Op1, 0)>;
   def : Pat<(vtd (op vt1:$Op1, (i32 4))), (inst $Op1, 1)>;
 }
 
+multiclass SVE2p1_While_PN_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
+                               Instruction inst> {
+  def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 2))), (inst $Op1, $Op2, 0)>;
+  def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 4))), (inst $Op1, $Op2, 1)>;
+}
+
 class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                        ValueType vt2, ValueType vt3, Operand ImmTy,
                        Instruction inst>
 : Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))),
       (inst $Op1, $Op2, ImmTy:$Op3)>;
 
 class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                        ValueType vt2, ValueType vt3, ValueType vt4,

 }
 
 
 multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> {
  def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, PNR8_p8to15>;
  def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>;
  def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>;
  def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>;
+
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c8"),
+                            i64, !cast<Instruction>(NAME # _B)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c16"),
+                            i64, !cast<Instruction>(NAME # _H)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c32"),
+                            i64, !cast<Instruction>(NAME # _S)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c64"),
+                            i64, !cast<Instruction>(NAME # _D)>;
 }
 
 
 // SVE integer compare scalar count and limit (predicate pair)
 class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc,
                              RegisterOperand ppr_ty>
     : I<(outs ppr_ty:$Pd), (ins GPR64:$Rn, GPR64:$Rm),
         mnemonic, "\t$Pd, $Rn, $Rm",

llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 < %s | FileCheck %s
+
+
+;
+; WHILEGE
+;
+
+define target("aarch64.svcount") @whilege_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEGT
+;
+
+define target("aarch64.svcount") @whilegt_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEHI
+;
+
+define target("aarch64.svcount") @whilehi_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEHS
+;
+
+define target("aarch64.svcount") @whilehs_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELE
+;
+
+define target("aarch64.svcount") @whilele_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELO
+;
+
+define target("aarch64.svcount") @whilelo_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELS
+;
+
+define target("aarch64.svcount") @whilels_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELT
+;
+
+define target("aarch64.svcount") @whilelt_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64, i64, i32)