[SVE] Lower vector CTLZ, CTPOP and CTTZ operations.

[paulwalker-arm]

Summary

CTLZ and CTPOP are lowered to CLZ and CNT instructions respectively.

CTTZ is not a native SVE operation but is instead lowered to:

CTTZ(V) => CTLZ(BITREVERSE(V))

In the case of fixed-length support using SVE we also lower CTTZ
operating on NEON sized vectors because of its reliance on
BITREVERSE which is also lowered to SVE intructions at these lengths.

Depends on D93606

[paulwalker-arm]

Fixed incorrect variable naming within ctlz_v64i8, ctpop_v64i8, cttz_v64i8.

[cameron.mcinally]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

227

The NOT patterns could use some tests. Or are they required for this patch and I missed it?

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

227

This patch doesn't really change anything for NOT, which is still relying directly on the SVE intrinsics for isel. However, the operand order used by the SVE unary intrinsics is different to that used for _MERGE_PASSTHRU nodes:

SVE unary intrinsic = OP <PASSTHRU> <PG> <OPERAND>
Unary_MERGE_PATHRU = OP <PG> <OPERAND> <PATHTHRU>

The above PatFrags is just shuffling the intrinsic operands so that sve_int_un_pred_arit_1 can take a single "op" parameter and have a single set of patterns. The existing tests within sve-intrinsics-logical.ll should ensure that I've not broken isel for the affected intrinsics cls, cnot and not.

I didn't realise this operand shuffling was possibly so I've likely added unnecessary int_op and ir_op parameters to some of the classes under past patches.

[cameron.mcinally]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   FMINNMV_PRED,
 
   INSR,
   PTEST,
   PTRUE,
 
   BITREVERSE_MERGE_PASSTHRU,
   BSWAP_MERGE_PASSTHRU,
+  CTLZ_MERGE_PASSTHRU,
+  CTPOP_MERGE_PASSTHRU,
   DUP_MERGE_PASSTHRU,
   INDEX_VECTOR,
 
   REINTERPRET_CAST,
 
   LD1_MERGE_ZERO,
   LD1S_MERGE_ZERO,
   LDNF1_MERGE_ZERO,
private:
   SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 // Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
 // predicate and end with a passthru value matching the result type.
 static bool isMergePassthruOpcode(unsigned Opc) {
   switch (Opc) {
   default:
     return false;
   case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
   case AArch64ISD::BSWAP_MERGE_PASSTHRU:
+  case AArch64ISD::CTLZ_MERGE_PASSTHRU:
+  case AArch64ISD::CTPOP_MERGE_PASSTHRU:
   case AArch64ISD::DUP_MERGE_PASSTHRU:
   case AArch64ISD::FNEG_MERGE_PASSTHRU:
   case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
   case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
   case AArch64ISD::FCEIL_MERGE_PASSTHRU:
   case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
   case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
   case AArch64ISD::FRINT_MERGE_PASSTHRU:
#undef LCALLNAME5
 
   if (Subtarget->hasSVE()) {
     // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
     // splat of 0 or undef) once vector selects supported in SVE codegen. See
     // D68877 for more details.
     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
       setOperationAction(ISD::BITREVERSE, VT, Custom);
       setOperationAction(ISD::BSWAP, VT, Custom);
+      setOperationAction(ISD::CTLZ, VT, Custom);
+      setOperationAction(ISD::CTPOP, VT, Custom);
+      setOperationAction(ISD::CTTZ, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
       setOperationAction(ISD::MGATHER, VT, Custom);
       setOperationAction(ISD::MSCATTER, VT, Custom);
       setOperationAction(ISD::MUL, VT, Custom);
if (Subtarget->useSVEForFixedLengthVectors()) {
       // 128bit results imply a bigger than NEON input.
       for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
         setOperationAction(ISD::TRUNCATE, VT, Custom);
       for (auto VT : {MVT::v8f16, MVT::v4f32})
         setOperationAction(ISD::FP_ROUND, VT, Expand);
 
       // These operations are not supported on NEON but SVE can do them.
       setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
+      setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
+      setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
+      setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
       setOperationAction(ISD::MUL, MVT::v1i64, Custom);
       setOperationAction(ISD::MUL, MVT::v2i64, Custom);
       setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
       setOperationAction(ISD::SDIV, MVT::v16i8, Custom);
       setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
       setOperationAction(ISD::SDIV, MVT::v8i16, Custom);
       setOperationAction(ISD::SDIV, MVT::v2i32, Custom);
       setOperationAction(ISD::SDIV, MVT::v4i32, Custom);
if (Subtarget->useSVEForFixedLengthVectors()) {
       setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
       setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
       setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
 
       // Int operations with no NEON support.
       for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
                       MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
         setOperationAction(ISD::BITREVERSE, VT, Custom);
+        setOperationAction(ISD::CTTZ, VT, Custom);
         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
       }
 
       // FP operations with no NEON support.
       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32,
                       MVT::v1f64, MVT::v2f64})
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
   // Lower fixed length vector operations to scalable equivalents.
   setOperationAction(ISD::ADD, VT, Custom);
   setOperationAction(ISD::AND, VT, Custom);
   setOperationAction(ISD::ANY_EXTEND, VT, Custom);
   setOperationAction(ISD::BITREVERSE, VT, Custom);
   setOperationAction(ISD::BSWAP, VT, Custom);
+  setOperationAction(ISD::CTLZ, VT, Custom);
+  setOperationAction(ISD::CTPOP, VT, Custom);
+  setOperationAction(ISD::CTTZ, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
   setOperationAction(ISD::FCEIL, VT, Custom);
   setOperationAction(ISD::FDIV, VT, Custom);
   setOperationAction(ISD::FFLOOR, VT, Custom);
   setOperationAction(ISD::FMA, VT, Custom);
   setOperationAction(ISD::FMAXNUM, VT, Custom);
   setOperationAction(ISD::FMINNUM, VT, Custom);
   setOperationAction(ISD::FMUL, VT, Custom);
case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
     MAKE_CASE(AArch64ISD::SSTNT1_PRED)
     MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
     MAKE_CASE(AArch64ISD::LDP)
     MAKE_CASE(AArch64ISD::STP)
     MAKE_CASE(AArch64ISD::STNP)
     MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
     MAKE_CASE(AArch64ISD::UABD)
     MAKE_CASE(AArch64ISD::SABD)
     MAKE_CASE(AArch64ISD::CALL_RVMARKER)
   }
 #undef MAKE_CASE
   return nullptr;
case Intrinsic::aarch64_sve_zip1:
     return DAG.getNode(AArch64ISD::ZIP1, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_zip2:
     return DAG.getNode(AArch64ISD::ZIP2, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_ptrue:
     return DAG.getNode(AArch64ISD::PTRUE, dl, Op.getValueType(),
                        Op.getOperand(1));
+  case Intrinsic::aarch64_sve_clz:
+    return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_cnt: {
+    SDValue Data = Op.getOperand(3);
+    // CTPOP only supports integer operands.
+    if (Data.getValueType().isFloatingPoint())
+      Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data);
+    return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Data, Op.getOperand(1));
+  }
   case Intrinsic::aarch64_sve_dupq_lane:
     return LowerDUPQLane(Op, DAG);
   case Intrinsic::aarch64_sve_convert_from_svbool:
     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_fneg:
     return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
case ISD::VSELECT:
     return LowerFixedLengthVectorSelectToSVE(Op, DAG);
   case ISD::ABS:
     return LowerABS(Op, DAG);
   case ISD::BITREVERSE:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BITREVERSE_MERGE_PASSTHRU,
                                /*OverrideNEON=*/true);
   case ISD::BSWAP:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BSWAP_MERGE_PASSTHRU);
+  case ISD::CTLZ:
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTLZ_MERGE_PASSTHRU,
+                               /*OverrideNEON=*/true);
+  case ISD::CTTZ:
+    return LowerCTTZ(Op, DAG);
   }
 }
 
 bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const {
   return !Subtarget->useSVEForFixedLengthVectors();
 }
 
 bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
if (VT == MVT::i32 || VT == MVT::i64) {
     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
     SDValue UaddLV = DAG.getNode(
         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
         DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
 
     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
   }
 
+  if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
+
   assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
           VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
          "Unexpected type for custom ctpop lowering");
 
   EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
   Val = DAG.getBitcast(VT8Bit, Val);
   Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
 
   // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
   unsigned EltSize = 8;
   unsigned NumElts = VT.is64BitVector() ? 8 : 16;
   while (EltSize != VT.getScalarSizeInBits()) {
     EltSize *= 2;
     NumElts /= 2;
     MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
     Val = DAG.getNode(
         ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
         DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
   }
 
   return Val;
 }
 
+SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isScalableVector() ||
+         useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true));
+
+  SDLoc DL(Op);
+  SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
+  return DAG.getNode(ISD::CTLZ, DL, VT, RBIT);
+}
+
 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
 
   if (Op.getValueType().isVector())
     return LowerVSETCC(Op, DAG);
 
   bool IsStrict = Op->isStrictFPOpcode();
   bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
   unsigned OpNo = IsStrict ? 1 : 0;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 def AArch64umin_p : SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
 
 def SDT_AArch64IntExtend : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVT<3, OtherVT>, SDTCisVec<4>,
   SDTCVecEltisVT<1,i1>, SDTCisSameAs<0,2>, SDTCisVTSmallerThanOp<3, 2>, SDTCisSameAs<0,4>
 ]>;
 
 // Predicated operations with the result of inactive lanes provided by the last operand.
+def AArch64clz_mt    : SDNode<"AArch64ISD::CTLZ_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64cnt_mt    : SDNode<"AArch64ISD::CTPOP_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64fneg_mt   : SDNode<"AArch64ISD::FNEG_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64fabs_mt   : SDNode<"AArch64ISD::FABS_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64sxt_mt    : SDNode<"AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU", SDT_AArch64IntExtend>;
 def AArch64uxt_mt    : SDNode<"AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU", SDT_AArch64IntExtend>;
 def AArch64frintp_mt : SDNode<"AArch64ISD::FCEIL_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frintm_mt : SDNode<"AArch64ISD::FFLOOR_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frinti_mt : SDNode<"AArch64ISD::FNEARBYINT_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frintx_mt : SDNode<"AArch64ISD::FRINT_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frinta_mt : SDNode<"AArch64ISD::FROUND_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frintn_mt : SDNode<"AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frintz_mt : SDNode<"AArch64ISD::FTRUNC_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64fsqrt_mt  : SDNode<"AArch64ISD::FSQRT_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64frecpx_mt : SDNode<"AArch64ISD::FRECPX_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64rbit_mt   : SDNode<"AArch64ISD::BITREVERSE_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64revb_mt   : SDNode<"AArch64ISD::BSWAP_MERGE_PASSTHRU", SDT_AArch64Arith>;
 
+// These are like the above but we don't yet have need for ISD nodes. They allow
+// a single pattern to match intrinsic and ISD operand layouts.
+def AArch64cls_mt  : PatFrags<(ops node:$pg, node:$op, node:$pt), [(int_aarch64_sve_cls  node:$pt, node:$pg, node:$op)]>;
+def AArch64cnot_mt : PatFrags<(ops node:$pg, node:$op, node:$pt), [(int_aarch64_sve_cnot node:$pt, node:$pg, node:$op)]>;
+def AArch64not_mt  : PatFrags<(ops node:$pg, node:$op, node:$pt), [(int_aarch64_sve_not  node:$pt, node:$pg, node:$op)]>;
+
 def SDT_AArch64FCVT : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,
   SDTCVecEltisVT<1,i1>
 ]>;
 
 def SDT_AArch64FCVTR : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisInt<3>, SDTCisVec<4>,
   SDTCVecEltisVT<1,i1>
let Predicates = [HasSVE] in {
   defm UXTB_ZPmZ : sve_int_un_pred_arit_0_h<0b001, "uxtb", AArch64uxt_mt>;
   defm SXTH_ZPmZ : sve_int_un_pred_arit_0_w<0b010, "sxth", AArch64sxt_mt>;
   defm UXTH_ZPmZ : sve_int_un_pred_arit_0_w<0b011, "uxth", AArch64uxt_mt>;
   defm SXTW_ZPmZ : sve_int_un_pred_arit_0_d<0b100, "sxtw", AArch64sxt_mt>;
   defm UXTW_ZPmZ : sve_int_un_pred_arit_0_d<0b101, "uxtw", AArch64uxt_mt>;
   defm ABS_ZPmZ  : sve_int_un_pred_arit_0<  0b110, "abs",  int_aarch64_sve_abs>;
   defm NEG_ZPmZ  : sve_int_un_pred_arit_0<  0b111, "neg",  int_aarch64_sve_neg>;
 
-  defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls",  int_aarch64_sve_cls>;
-  defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz",  int_aarch64_sve_clz>;
-  defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt",  int_aarch64_sve_cnt>;
-
-  defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot", int_aarch64_sve_cnot>;
-  defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not",  int_aarch64_sve_not>;
+  defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls",  AArch64cls_mt>;
+  defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz",  AArch64clz_mt>;
+  defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt",  AArch64cnt_mt>;
+  defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot", AArch64cnot_mt>;
+  defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not",  AArch64not_mt>;
   defm FABS_ZPmZ : sve_int_un_pred_arit_1_fp<0b100, "fabs", AArch64fabs_mt>;
   defm FNEG_ZPmZ : sve_int_un_pred_arit_1_fp<0b101, "fneg", AArch64fneg_mt>;
 
   defm SMAX_ZPmZ : sve_int_bin_pred_arit_1<0b000, "smax", "SMAX_ZPZZ", int_aarch64_sve_smax, DestructiveBinaryComm>;
   defm UMAX_ZPmZ : sve_int_bin_pred_arit_1<0b001, "umax", "UMAX_ZPZZ", int_aarch64_sve_umax, DestructiveBinaryComm>;
   defm SMIN_ZPmZ : sve_int_bin_pred_arit_1<0b010, "smin", "SMIN_ZPZZ", int_aarch64_sve_smin, DestructiveBinaryComm>;
   defm UMIN_ZPmZ : sve_int_bin_pred_arit_1<0b011, "umin", "UMIN_ZPZZ", int_aarch64_sve_umin, DestructiveBinaryComm>;
   defm SABD_ZPmZ : sve_int_bin_pred_arit_1<0b100, "sabd", "SABD_ZPZZ", int_aarch64_sve_sabd, DestructiveBinaryComm>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

 
 multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm,
                                   SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>;
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>;
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>;
 
-  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
-  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
-
-  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
-
-  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>;
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>;
 
   def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;

llvm/test/CodeGen/AArch64/sve-bit-counting.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; CLZ
+;
+
+define <vscale x 16 x i8> @ctlz_b(<vscale x 16 x i8> %a) #0 {
+; CHECK-LABEL: ctlz_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @ctlz_h(<vscale x 8 x i16> %a) #0 {
+; CHECK-LABEL: ctlz_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @ctlz_s(<vscale x 4 x i32> %a) #0 {
+; CHECK-LABEL: ctlz_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ctlz_d(<vscale x 2 x i64> %a) #0 {
+; CHECK-LABEL: ctlz_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    clz z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; CNT
+;
+
+define <vscale x 16 x i8> @ctpop_b(<vscale x 16 x i8> %a) #0 {
+; CHECK-LABEL: ctpop_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @ctpop_h(<vscale x 8 x i16> %a) #0 {
+; CHECK-LABEL: ctpop_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @ctpop_s(<vscale x 4 x i32> %a) #0 {
+; CHECK-LABEL: ctpop_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ctpop_d(<vscale x 2 x i64> %a) #0 {
+; CHECK-LABEL: ctpop_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; Count trailing zeros
+;
+
+define <vscale x 16 x i8> @cttz_b(<vscale x 16 x i8> %a) #0 {
+; CHECK-LABEL: cttz_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @cttz_h(<vscale x 8 x i16> %a) #0 {
+; CHECK-LABEL: cttz_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @cttz_s(<vscale x 4 x i32> %a) #0 {
+; CHECK-LABEL: cttz_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @cttz_d(<vscale x 2 x i64> %a) #0 {
+; CHECK-LABEL: cttz_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
+; CHECK-NEXT:    clz z0.d, p0/m, z0.d
+; CHECK-NEXT:    ret
+
+  %res = call <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %res
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+declare <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64>)

llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+;
+; CLZ
+;
+
+; Don't use SVE for 64-bit vectors.
+define <8 x i8> @ctlz_v8i8(<8 x i8> %op) #0 {
+; CHECK-LABEL: ctlz_v8i8:
+; CHECK: clz v0.8b, v0.8b
+; CHECK-NEXT: ret
+  %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)
+  ret <8 x i8> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <16 x i8> @ctlz_v16i8(<16 x i8> %op) #0 {
+; CHECK-LABEL: ctlz_v16i8:
+; CHECK: clz v0.16b, v0.16b
+; CHECK-NEXT: ret
+  %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)
+  ret <16 x i8> %res
+}
+
+define void @ctlz_v32i8(<32 x i8>* %a) #0 {
+; CHECK-LABEL: ctlz_v32i8:
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl32
+; CHECK-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; CHECK-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <32 x i8>, <32 x i8>* %a
+  %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)
+  store <32 x i8> %res, <32 x i8>* %a
+  ret void
+}
+
+define void @ctlz_v64i8(<64 x i8>* %a) #0 {
+; CHECK-LABEL: ctlz_v64i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].b, vl64
+; VBITS_GE_512-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_512-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+;
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].b, vl32
+; VBITS_EQ_256-DAG: mov w[[A:[0-9]+]], #32
+; VBITS_EQ_256-DAG: ld1b { [[OP_LO:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1b { [[OP_HI:z[0-9]+]].b }, [[PG]]/z, [x0, x[[A]]]
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].b, [[PG]]/m, [[OP_LO]].b
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].b, [[PG]]/m, [[OP_HI]].b
+; VBITS_EQ_256-DAG: st1b { [[RES_LO]].b }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1b { [[RES_HI]].b }, [[PG]], [x0, x[[A]]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <64 x i8>, <64 x i8>* %a
+  %res = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %op)
+  store <64 x i8> %res, <64 x i8>* %a
+  ret void
+}
+
+define void @ctlz_v128i8(<128 x i8>* %a) #0 {
+; CHECK-LABEL: ctlz_v128i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].b, vl128
+; VBITS_GE_1024-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_1024-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <128 x i8>, <128 x i8>* %a
+  %res = call <128 x i8> @llvm.ctlz.v128i8(<128 x i8> %op)
+  store <128 x i8> %res, <128 x i8>* %a
+  ret void
+}
+
+define void @ctlz_v256i8(<256 x i8>* %a) #0 {
+; CHECK-LABEL: ctlz_v256i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].b, vl256
+; VBITS_GE_2048-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_2048-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <256 x i8>, <256 x i8>* %a
+  %res = call <256 x i8> @llvm.ctlz.v256i8(<256 x i8> %op)
+  store <256 x i8> %res, <256 x i8>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <4 x i16> @ctlz_v4i16(<4 x i16> %op) #0 {
+; CHECK-LABEL: ctlz_v4i16:
+; CHECK: clz v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)
+  ret <4 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x i16> @ctlz_v8i16(<8 x i16> %op) #0 {
+; CHECK-LABEL: ctlz_v8i16:
+; CHECK: clz v0.8h, v0.8h
+; CHECK-NEXT: ret
+  %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)
+  ret <8 x i16> %res
+}
+
+define void @ctlz_v16i16(<16 x i16>* %a) #0 {
+; CHECK-LABEL: ctlz_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <16 x i16>, <16 x i16>* %a
+  %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)
+  store <16 x i16> %res, <16 x i16>* %a
+  ret void
+}
+
+define void @ctlz_v32i16(<32 x i16>* %a) #0 {
+; CHECK-LABEL: ctlz_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].h, vl16
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1h { [[OP_LO:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1h { [[OP_HI:z[0-9]+]].h }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].h, [[PG]]/m, [[OP_LO]].h
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].h, [[PG]]/m, [[OP_HI]].h
+; VBITS_EQ_256-DAG: st1h { [[RES_LO]].h }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1h { [[RES_HI]].h }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <32 x i16>, <32 x i16>* %a
+  %res = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %op)
+  store <32 x i16> %res, <32 x i16>* %a
+  ret void
+}
+
+define void @ctlz_v64i16(<64 x i16>* %a) #0 {
+; CHECK-LABEL: ctlz_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <64 x i16>, <64 x i16>* %a
+  %res = call <64 x i16> @llvm.ctlz.v64i16(<64 x i16> %op)
+  store <64 x i16> %res, <64 x i16>* %a
+  ret void
+}
+
+define void @ctlz_v128i16(<128 x i16>* %a) #0 {
+; CHECK-LABEL: ctlz_v128i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <128 x i16>, <128 x i16>* %a
+  %res = call <128 x i16> @llvm.ctlz.v128i16(<128 x i16> %op)
+  store <128 x i16> %res, <128 x i16>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @ctlz_v2i32(<2 x i32> %op) #0 {
+; CHECK-LABEL: ctlz_v2i32:
+; CHECK: clz v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @ctlz_v4i32(<4 x i32> %op) #0 {
+; CHECK-LABEL: ctlz_v4i32:
+; CHECK: clz v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)
+  ret <4 x i32> %res
+}
+
+define void @ctlz_v8i32(<8 x i32>* %a) #0 {
+; CHECK-LABEL: ctlz_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <8 x i32>, <8 x i32>* %a
+  %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)
+  store <8 x i32> %res, <8 x i32>* %a
+  ret void
+}
+
+define void @ctlz_v16i32(<16 x i32>* %a) #0 {
+; CHECK-LABEL: ctlz_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1w { [[OP_LO:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1w { [[OP_HI:z[0-9]+]].s }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].s, [[PG]]/m, [[OP_LO]].s
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].s, [[PG]]/m, [[OP_HI]].s
+; VBITS_EQ_256-DAG: st1w { [[RES_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[RES_HI]].s }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <16 x i32>, <16 x i32>* %a
+  %res = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %op)
+  store <16 x i32> %res, <16 x i32>* %a
+  ret void
+}
+
+define void @ctlz_v32i32(<32 x i32>* %a) #0 {
+; CHECK-LABEL: ctlz_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <32 x i32>, <32 x i32>* %a
+  %res = call <32 x i32> @llvm.ctlz.v32i32(<32 x i32> %op)
+  store <32 x i32> %res, <32 x i32>* %a
+  ret void
+}
+
+define void @ctlz_v64i32(<64 x i32>* %a) #0 {
+; CHECK-LABEL: ctlz_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <64 x i32>, <64 x i32>* %a
+  %res = call <64 x i32> @llvm.ctlz.v64i32(<64 x i32> %op)
+  store <64 x i32> %res, <64 x i32>* %a
+  ret void
+}
+
+define <1 x i64> @ctlz_v1i64(<1 x i64> %op) #0 {
+; CHECK-LABEL: ctlz_v1i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl1
+; CHECK-NEXT: clz z0.d, [[PG]]/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %op)
+  ret <1 x i64> %res
+}
+
+define <2 x i64> @ctlz_v2i64(<2 x i64> %op) #0 {
+; CHECK-LABEL: ctlz_v2i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl2
+; CHECK-NEXT: clz z0.d, [[PG]]/m, z0.d
+; CHECK-NEXT: ret
+  %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %op)
+  ret <2 x i64> %res
+}
+
+define void @ctlz_v4i64(<4 x i64>* %a) #0 {
+; CHECK-LABEL: ctlz_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <4 x i64>, <4 x i64>* %a
+  %res = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %op)
+  store <4 x i64> %res, <4 x i64>* %a
+  ret void
+}
+
+define void @ctlz_v8i64(<8 x i64>* %a) #0 {
+; CHECK-LABEL: ctlz_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1d { [[OP_LO:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1d { [[OP_HI:z[0-9]+]].d }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].d, [[PG]]/m, [[OP_LO]].d
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].d, [[PG]]/m, [[OP_HI]].d
+; VBITS_EQ_256-DAG: st1d { [[RES_LO]].d }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1d { [[RES_HI]].d }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <8 x i64>, <8 x i64>* %a
+  %res = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %op)
+  store <8 x i64> %res, <8 x i64>* %a
+  ret void
+}
+
+define void @ctlz_v16i64(<16 x i64>* %a) #0 {
+; CHECK-LABEL: ctlz_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <16 x i64>, <16 x i64>* %a
+  %res = call <16 x i64> @llvm.ctlz.v16i64(<16 x i64> %op)
+  store <16 x i64> %res, <16 x i64>* %a
+  ret void
+}
+
+define void @ctlz_v32i64(<32 x i64>* %a) #0 {
+; CHECK-LABEL: ctlz_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <32 x i64>, <32 x i64>* %a
+  %res = call <32 x i64> @llvm.ctlz.v32i64(<32 x i64> %op)
+  store <32 x i64> %res, <32 x i64>* %a
+  ret void
+}
+
+;
+; CNT
+;
+
+; Don't use SVE for 64-bit vectors.
+define <8 x i8> @ctpop_v8i8(<8 x i8> %op) #0 {
+; CHECK-LABEL: ctpop_v8i8:
+; CHECK: cnt v0.8b, v0.8b
+; CHECK-NEXT: ret
+  %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)
+  ret <8 x i8> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <16 x i8> @ctpop_v16i8(<16 x i8> %op) #0 {
+; CHECK-LABEL: ctpop_v16i8:
+; CHECK: cnt v0.16b, v0.16b
+; CHECK-NEXT: ret
+  %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)
+  ret <16 x i8> %res
+}
+
+define void @ctpop_v32i8(<32 x i8>* %a) #0 {
+; CHECK-LABEL: ctpop_v32i8:
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl32
+; CHECK-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-NEXT: cnt [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; CHECK-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <32 x i8>, <32 x i8>* %a
+  %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
+  store <32 x i8> %res, <32 x i8>* %a
+  ret void
+}
+
+define void @ctpop_v64i8(<64 x i8>* %a) #0 {
+; CHECK-LABEL: ctpop_v64i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].b, vl64
+; VBITS_GE_512-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: cnt [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_512-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+;
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].b, vl32
+; VBITS_EQ_256-DAG: mov w[[A:[0-9]+]], #32
+; VBITS_EQ_256-DAG: ld1b { [[OP_LO:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1b { [[OP_HI:z[0-9]+]].b }, [[PG]]/z, [x0, x[[A]]]
+; VBITS_EQ_256-DAG: cnt [[RES_LO:z[0-9]+]].b, [[PG]]/m, [[OP_LO]].b
+; VBITS_EQ_256-DAG: cnt [[RES_HI:z[0-9]+]].b, [[PG]]/m, [[OP_HI]].b
+; VBITS_EQ_256-DAG: st1b { [[RES_LO]].b }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1b { [[RES_HI]].b }, [[PG]], [x0, x[[A]]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <64 x i8>, <64 x i8>* %a
+  %res = call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %op)
+  store <64 x i8> %res, <64 x i8>* %a
+  ret void
+}
+
+define void @ctpop_v128i8(<128 x i8>* %a) #0 {
+; CHECK-LABEL: ctpop_v128i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].b, vl128
+; VBITS_GE_1024-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: cnt [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_1024-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <128 x i8>, <128 x i8>* %a
+  %res = call <128 x i8> @llvm.ctpop.v128i8(<128 x i8> %op)
+  store <128 x i8> %res, <128 x i8>* %a
+  ret void
+}
+
+define void @ctpop_v256i8(<256 x i8>* %a) #0 {
+; CHECK-LABEL: ctpop_v256i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].b, vl256
+; VBITS_GE_2048-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: cnt [[RES:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_2048-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <256 x i8>, <256 x i8>* %a
+  %res = call <256 x i8> @llvm.ctpop.v256i8(<256 x i8> %op)
+  store <256 x i8> %res, <256 x i8>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <4 x i16> @ctpop_v4i16(<4 x i16> %op) #0 {
+; CHECK-LABEL: ctpop_v4i16:
+; CHECK: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: ret
+  %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
+  ret <4 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x i16> @ctpop_v8i16(<8 x i16> %op) #0 {
+; CHECK-LABEL: ctpop_v8i16:
+; CHECK: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+  %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
+  ret <8 x i16> %res
+}
+
+define void @ctpop_v16i16(<16 x i16>* %a) #0 {
+; CHECK-LABEL: ctpop_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: cnt [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <16 x i16>, <16 x i16>* %a
+  %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)
+  store <16 x i16> %res, <16 x i16>* %a
+  ret void
+}
+
+define void @ctpop_v32i16(<32 x i16>* %a) #0 {
+; CHECK-LABEL: ctpop_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: cnt [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].h, vl16
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1h { [[OP_LO:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1h { [[OP_HI:z[0-9]+]].h }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: cnt [[RES_LO:z[0-9]+]].h, [[PG]]/m, [[OP_LO]].h
+; VBITS_EQ_256-DAG: cnt [[RES_HI:z[0-9]+]].h, [[PG]]/m, [[OP_HI]].h
+; VBITS_EQ_256-DAG: st1h { [[RES_LO]].h }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1h { [[RES_HI]].h }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <32 x i16>, <32 x i16>* %a
+  %res = call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %op)
+  store <32 x i16> %res, <32 x i16>* %a
+  ret void
+}
+
+define void @ctpop_v64i16(<64 x i16>* %a) #0 {
+; CHECK-LABEL: ctpop_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: cnt [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <64 x i16>, <64 x i16>* %a
+  %res = call <64 x i16> @llvm.ctpop.v64i16(<64 x i16> %op)
+  store <64 x i16> %res, <64 x i16>* %a
+  ret void
+}
+
+define void @ctpop_v128i16(<128 x i16>* %a) #0 {
+; CHECK-LABEL: ctpop_v128i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: cnt [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <128 x i16>, <128 x i16>* %a
+  %res = call <128 x i16> @llvm.ctpop.v128i16(<128 x i16> %op)
+  store <128 x i16> %res, <128 x i16>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @ctpop_v2i32(<2 x i32> %op) #0 {
+; CHECK-LABEL: ctpop_v2i32:
+; CHECK: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: uaddlp v0.2s, v0.4h
+; CHECK-NEXT: ret
+  %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @ctpop_v4i32(<4 x i32> %op) #0 {
+; CHECK-LABEL: ctpop_v4i32:
+; CHECK: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: uaddlp v0.4s, v0.8h
+; CHECK-NEXT: ret
+  %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
+  ret <4 x i32> %res
+}
+
+define void @ctpop_v8i32(<8 x i32>* %a) #0 {
+; CHECK-LABEL: ctpop_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: cnt [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <8 x i32>, <8 x i32>* %a
+  %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)
+  store <8 x i32> %res, <8 x i32>* %a
+  ret void
+}
+
+define void @ctpop_v16i32(<16 x i32>* %a) #0 {
+; CHECK-LABEL: ctpop_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: cnt [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1w { [[OP_LO:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1w { [[OP_HI:z[0-9]+]].s }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: cnt [[RES_LO:z[0-9]+]].s, [[PG]]/m, [[OP_LO]].s
+; VBITS_EQ_256-DAG: cnt [[RES_HI:z[0-9]+]].s, [[PG]]/m, [[OP_HI]].s
+; VBITS_EQ_256-DAG: st1w { [[RES_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[RES_HI]].s }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <16 x i32>, <16 x i32>* %a
+  %res = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %op)
+  store <16 x i32> %res, <16 x i32>* %a
+  ret void
+}
+
+define void @ctpop_v32i32(<32 x i32>* %a) #0 {
+; CHECK-LABEL: ctpop_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: cnt [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <32 x i32>, <32 x i32>* %a
+  %res = call <32 x i32> @llvm.ctpop.v32i32(<32 x i32> %op)
+  store <32 x i32> %res, <32 x i32>* %a
+  ret void
+}
+
+define void @ctpop_v64i32(<64 x i32>* %a) #0 {
+; CHECK-LABEL: ctpop_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: cnt [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <64 x i32>, <64 x i32>* %a
+  %res = call <64 x i32> @llvm.ctpop.v64i32(<64 x i32> %op)
+  store <64 x i32> %res, <64 x i32>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @ctpop_v1i64(<1 x i64> %op) #0 {
+; CHECK-LABEL: ctpop_v1i64:
+; CHECK: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: uaddlp v0.2s, v0.4h
+; CHECK-NEXT: uaddlp v0.1d, v0.2s
+; CHECK-NEXT: ret
+  %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @ctpop_v2i64(<2 x i64> %op) #0 {
+; CHECK-LABEL: ctpop_v2i64:
+; CHECK: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: uaddlp v0.4s, v0.8h
+; CHECK-NEXT: uaddlp v0.2d, v0.4s
+; CHECK-NEXT: ret
+  %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
+  ret <2 x i64> %res
+}
+
+define void @ctpop_v4i64(<4 x i64>* %a) #0 {
+; CHECK-LABEL: ctpop_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: cnt [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <4 x i64>, <4 x i64>* %a
+  %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)
+  store <4 x i64> %res, <4 x i64>* %a
+  ret void
+}
+
+define void @ctpop_v8i64(<8 x i64>* %a) #0 {
+; CHECK-LABEL: ctpop_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: cnt [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1d { [[OP_LO:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1d { [[OP_HI:z[0-9]+]].d }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: cnt [[RES_LO:z[0-9]+]].d, [[PG]]/m, [[OP_LO]].d
+; VBITS_EQ_256-DAG: cnt [[RES_HI:z[0-9]+]].d, [[PG]]/m, [[OP_HI]].d
+; VBITS_EQ_256-DAG: st1d { [[RES_LO]].d }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1d { [[RES_HI]].d }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <8 x i64>, <8 x i64>* %a
+  %res = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %op)
+  store <8 x i64> %res, <8 x i64>* %a
+  ret void
+}
+
+define void @ctpop_v16i64(<16 x i64>* %a) #0 {
+; CHECK-LABEL: ctpop_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: cnt [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <16 x i64>, <16 x i64>* %a
+  %res = call <16 x i64> @llvm.ctpop.v16i64(<16 x i64> %op)
+  store <16 x i64> %res, <16 x i64>* %a
+  ret void
+}
+
+define void @ctpop_v32i64(<32 x i64>* %a) #0 {
+; CHECK-LABEL: ctpop_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: cnt [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <32 x i64>, <32 x i64>* %a
+  %res = call <32 x i64> @llvm.ctpop.v32i64(<32 x i64> %op)
+  store <32 x i64> %res, <32 x i64>* %a
+  ret void
+}
+
+;
+; Count trailing zeros
+;
+
+define <8 x i8> @cttz_v8i8(<8 x i8> %op) #0 {
+; CHECK-LABEL: cttz_v8i8:
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl8
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].b, p0/m, z0.b
+; CHECK-NEXT: clz v0.8b, v[[RBIT]].8b
+; CHECK-NEXT: ret
+  %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @cttz_v16i8(<16 x i8> %op) #0 {
+; CHECK-LABEL: cttz_v16i8:
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl16
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].b, p0/m, z0.b
+; CHECK-NEXT: clz v0.16b, v[[RBIT]].16b
+; CHECK-NEXT: ret
+  %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)
+  ret <16 x i8> %res
+}
+
+define void @cttz_v32i8(<32 x i8>* %a) #0 {
+; CHECK-LABEL: cttz_v32i8:
+; CHECK: ptrue [[PG:p[0-9]+]].b, vl32
+; CHECK-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; CHECK-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[RBIT]].b
+; CHECK-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <32 x i8>, <32 x i8>* %a
+  %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)
+  store <32 x i8> %res, <32 x i8>* %a
+  ret void
+}
+
+define void @cttz_v64i8(<64 x i8>* %a) #0 {
+; CHECK-LABEL: cttz_v64i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].b, vl64
+; VBITS_GE_512-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: rbit [[RBIT:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[RBIT]].b
+; VBITS_GE_512-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+;
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].b, vl32
+; VBITS_EQ_256-DAG: mov w[[A:[0-9]+]], #32
+; VBITS_EQ_256-DAG: ld1b { [[OP_LO:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1b { [[OP_HI:z[0-9]+]].b }, [[PG]]/z, [x0, x[[A]]]
+; VBITS_EQ_256-DAG: rbit [[RBIT_LO:z[0-9]+]].b, [[PG]]/m, [[OP_LO]].b
+; VBITS_EQ_256-DAG: rbit [[RBIT_HI:z[0-9]+]].b, [[PG]]/m, [[OP_HI]].b
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].b, [[PG]]/m, [[RBIT_LO]].b
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].b, [[PG]]/m, [[RBIT_HI]].b
+; VBITS_EQ_256-DAG: st1b { [[RES_LO]].b }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1b { [[RES_HI]].b }, [[PG]], [x0, x[[A]]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <64 x i8>, <64 x i8>* %a
+  %res = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %op)
+  store <64 x i8> %res, <64 x i8>* %a
+  ret void
+}
+
+define void @cttz_v128i8(<128 x i8>* %a) #0 {
+; CHECK-LABEL: cttz_v128i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].b, vl128
+; VBITS_GE_1024-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: rbit [[RBIT:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[RBIT]].b
+; VBITS_GE_1024-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <128 x i8>, <128 x i8>* %a
+  %res = call <128 x i8> @llvm.cttz.v128i8(<128 x i8> %op)
+  store <128 x i8> %res, <128 x i8>* %a
+  ret void
+}
+
+define void @cttz_v256i8(<256 x i8>* %a) #0 {
+; CHECK-LABEL: cttz_v256i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].b, vl256
+; VBITS_GE_2048-NEXT: ld1b { [[OP:z[0-9]+]].b }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: rbit [[RBIT:z[0-9]+]].b, [[PG]]/m, [[OP]].b
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].b, [[PG]]/m, [[RBIT]].b
+; VBITS_GE_2048-NEXT: st1b { [[RES]].b }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <256 x i8>, <256 x i8>* %a
+  %res = call <256 x i8> @llvm.cttz.v256i8(<256 x i8> %op)
+  store <256 x i8> %res, <256 x i8>* %a
+  ret void
+}
+
+define <4 x i16> @cttz_v4i16(<4 x i16> %op) #0 {
+; CHECK-LABEL: cttz_v4i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl4
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].h, p0/m, z0.h
+; CHECK-NEXT: clz v0.4h, v[[RBIT]].4h
+; CHECK-NEXT: ret
+  %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @cttz_v8i16(<8 x i16> %op) #0 {
+; CHECK-LABEL: cttz_v8i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl8
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].h, p0/m, z0.h
+; CHECK-NEXT: clz v0.8h, v[[RBIT]].8h
+; CHECK-NEXT: ret
+  %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)
+  ret <8 x i16> %res
+}
+
+define void @cttz_v16i16(<16 x i16>* %a) #0 {
+; CHECK-LABEL: cttz_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[RBIT]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <16 x i16>, <16 x i16>* %a
+  %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)
+  store <16 x i16> %res, <16 x i16>* %a
+  ret void
+}
+
+define void @cttz_v32i16(<32 x i16>* %a) #0 {
+; CHECK-LABEL: cttz_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: rbit [[RBIT:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[RBIT]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].h, vl16
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1h { [[OP_LO:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1h { [[OP_HI:z[0-9]+]].h }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: rbit [[RBIT_LO:z[0-9]+]].h, [[PG]]/m, [[OP_LO]].h
+; VBITS_EQ_256-DAG: rbit [[RBIT_HI:z[0-9]+]].h, [[PG]]/m, [[OP_HI]].h
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].h, [[PG]]/m, [[RBIT_LO]].h
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].h, [[PG]]/m, [[RBIT_HI]].h
+; VBITS_EQ_256-DAG: st1h { [[RES_LO]].h }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1h { [[RES_HI]].h }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <32 x i16>, <32 x i16>* %a
+  %res = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %op)
+  store <32 x i16> %res, <32 x i16>* %a
+  ret void
+}
+
+define void @cttz_v64i16(<64 x i16>* %a) #0 {
+; CHECK-LABEL: cttz_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: rbit [[RBIT:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[RBIT]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <64 x i16>, <64 x i16>* %a
+  %res = call <64 x i16> @llvm.cttz.v64i16(<64 x i16> %op)
+  store <64 x i16> %res, <64 x i16>* %a
+  ret void
+}
+
+define void @cttz_v128i16(<128 x i16>* %a) #0 {
+; CHECK-LABEL: cttz_v128i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: rbit [[RBIT:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].h, [[PG]]/m, [[RBIT]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <128 x i16>, <128 x i16>* %a
+  %res = call <128 x i16> @llvm.cttz.v128i16(<128 x i16> %op)
+  store <128 x i16> %res, <128 x i16>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @cttz_v2i32(<2 x i32> %op) #0 {
+; CHECK-LABEL: cttz_v2i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl2
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].s, p0/m, z0.s
+; CHECK-NEXT: clz v0.2s, v[[RBIT]].2s
+; CHECK-NEXT: ret
+  %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @cttz_v4i32(<4 x i32> %op) #0 {
+; CHECK-LABEL: cttz_v4i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl4
+; CHECK-NEXT: rbit z[[RBIT:[0-9]+]].s, p0/m, z0.s
+; CHECK-NEXT: clz v0.4s, v[[RBIT]].4s
+; CHECK-NEXT: ret
+  %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)
+  ret <4 x i32> %res
+}
+
+define void @cttz_v8i32(<8 x i32>* %a) #0 {
+; CHECK-LABEL: cttz_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[RBIT]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <8 x i32>, <8 x i32>* %a
+  %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)
+  store <8 x i32> %res, <8 x i32>* %a
+  ret void
+}
+
+define void @cttz_v16i32(<16 x i32>* %a) #0 {
+; CHECK-LABEL: cttz_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: rbit [[RBIT:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[RBIT]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1w { [[OP_LO:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1w { [[OP_HI:z[0-9]+]].s }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: rbit [[RBIT_LO:z[0-9]+]].s, [[PG]]/m, [[OP_LO]].s
+; VBITS_EQ_256-DAG: rbit [[RBIT_HI:z[0-9]+]].s, [[PG]]/m, [[OP_HI]].s
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].s, [[PG]]/m, [[RBIT_LO]].s
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].s, [[PG]]/m, [[RBIT_HI]].s
+; VBITS_EQ_256-DAG: st1w { [[RES_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[RES_HI]].s }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <16 x i32>, <16 x i32>* %a
+  %res = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %op)
+  store <16 x i32> %res, <16 x i32>* %a
+  ret void
+}
+
+define void @cttz_v32i32(<32 x i32>* %a) #0 {
+; CHECK-LABEL: cttz_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: rbit [[RBIT:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[RBIT]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <32 x i32>, <32 x i32>* %a
+  %res = call <32 x i32> @llvm.cttz.v32i32(<32 x i32> %op)
+  store <32 x i32> %res, <32 x i32>* %a
+  ret void
+}
+
+define void @cttz_v64i32(<64 x i32>* %a) #0 {
+; CHECK-LABEL: cttz_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: rbit [[RBIT:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].s, [[PG]]/m, [[RBIT]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <64 x i32>, <64 x i32>* %a
+  %res = call <64 x i32> @llvm.cttz.v64i32(<64 x i32> %op)
+  store <64 x i32> %res, <64 x i32>* %a
+  ret void
+}
+
+define <1 x i64> @cttz_v1i64(<1 x i64> %op) #0 {
+; CHECK-LABEL: cttz_v1i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl1
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, z0.d
+; CHECK-NEXT: clz z0.d, [[PG]]/m, [[RBIT]].d
+; CHECK-NEXT: ret
+  %res = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %op)
+  ret <1 x i64> %res
+}
+
+define <2 x i64> @cttz_v2i64(<2 x i64> %op) #0 {
+; CHECK-LABEL: cttz_v2i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl2
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, z0.d
+; CHECK-NEXT: clz z0.d, [[PG]]/m, [[RBIT]].d
+; CHECK-NEXT: ret
+  %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %op)
+  ret <2 x i64> %res
+}
+
+define void @cttz_v4i64(<4 x i64>* %a) #0 {
+; CHECK-LABEL: cttz_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[RBIT]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <4 x i64>, <4 x i64>* %a
+  %res = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %op)
+  store <4 x i64> %res, <4 x i64>* %a
+  ret void
+}
+
+define void @cttz_v8i64(<8 x i64>* %a) #0 {
+; CHECK-LABEL: cttz_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[RBIT]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1d { [[OP_LO:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1d { [[OP_HI:z[0-9]+]].d }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: rbit [[RBIT_LO:z[0-9]+]].d, [[PG]]/m, [[OP_LO]].d
+; VBITS_EQ_256-DAG: rbit [[RBIT_HI:z[0-9]+]].d, [[PG]]/m, [[OP_HI]].d
+; VBITS_EQ_256-DAG: clz [[RES_LO:z[0-9]+]].d, [[PG]]/m, [[RBIT_LO]].d
+; VBITS_EQ_256-DAG: clz [[RES_HI:z[0-9]+]].d, [[PG]]/m, [[RBIT_HI]].d
+; VBITS_EQ_256-DAG: st1d { [[RES_LO]].d }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1d { [[RES_HI]].d }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <8 x i64>, <8 x i64>* %a
+  %res = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %op)
+  store <8 x i64> %res, <8 x i64>* %a
+  ret void
+}
+
+define void @cttz_v16i64(<16 x i64>* %a) #0 {
+; CHECK-LABEL: cttz_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[RBIT]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <16 x i64>, <16 x i64>* %a
+  %res = call <16 x i64> @llvm.cttz.v16i64(<16 x i64> %op)
+  store <16 x i64> %res, <16 x i64>* %a
+  ret void
+}
+
+define void @cttz_v32i64(<32 x i64>* %a) #0 {
+; CHECK-LABEL: cttz_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: rbit [[RBIT:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: clz [[RES:z[0-9]+]].d, [[PG]]/m, [[RBIT]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <32 x i64>, <32 x i64>* %a
+  %res = call <32 x i64> @llvm.cttz.v32i64(<32 x i64> %op)
+  store <32 x i64> %res, <32 x i64>* %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>)
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>)
+declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>)
+declare <64 x i8> @llvm.ctlz.v64i8(<64 x i8>)
+declare <128 x i8> @llvm.ctlz.v128i8(<128 x i8>)
+declare <256 x i8> @llvm.ctlz.v256i8(<256 x i8>)
+declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>)
+declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>)
+declare <32 x i16> @llvm.ctlz.v32i16(<32 x i16>)
+declare <64 x i16> @llvm.ctlz.v64i16(<64 x i16>)
+declare <128 x i16> @llvm.ctlz.v128i16(<128 x i16>)
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>)
+declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>)
+declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>)
+declare <32 x i32> @llvm.ctlz.v32i32(<32 x i32>)
+declare <64 x i32> @llvm.ctlz.v64i32(<64 x i32>)
+declare <1 x i64> @llvm.ctlz.v1i64(<1 x i64>)
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>)
+declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>)
+declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>)
+declare <16 x i64> @llvm.ctlz.v16i64(<16 x i64>)
+declare <32 x i64> @llvm.ctlz.v32i64(<32 x i64>)
+
+declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>)
+declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
+declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)
+declare <64 x i8> @llvm.ctpop.v64i8(<64 x i8>)
+declare <128 x i8> @llvm.ctpop.v128i8(<128 x i8>)
+declare <256 x i8> @llvm.ctpop.v256i8(<256 x i8>)
+declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
+declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
+declare <32 x i16> @llvm.ctpop.v32i16(<32 x i16>)
+declare <64 x i16> @llvm.ctpop.v64i16(<64 x i16>)
+declare <128 x i16> @llvm.ctpop.v128i16(<128 x i16>)
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
+declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
+declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>)
+declare <32 x i32> @llvm.ctpop.v32i32(<32 x i32>)
+declare <64 x i32> @llvm.ctpop.v64i32(<64 x i32>)
+declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
+declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>)
+declare <16 x i64> @llvm.ctpop.v16i64(<16 x i64>)
+declare <32 x i64> @llvm.ctpop.v32i64(<32 x i64>)
+
+declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>)
+declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>)
+declare <128 x i8> @llvm.cttz.v128i8(<128 x i8>)
+declare <256 x i8> @llvm.cttz.v256i8(<256 x i8>)
+declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>)
+declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>)
+declare <64 x i16> @llvm.cttz.v64i16(<64 x i16>)
+declare <128 x i16> @llvm.cttz.v128i16(<128 x i16>)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>)
+declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>)
+declare <32 x i32> @llvm.cttz.v32i32(<32 x i32>)
+declare <64 x i32> @llvm.cttz.v64i32(<64 x i32>)
+declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>)
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>)
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>)
+declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>)
+declare <16 x i64> @llvm.cttz.v16i64(<16 x i64>)
+declare <32 x i64> @llvm.cttz.v32i64(<32 x i64>)