[AArch64][SME]: Generate streaming-compatible code for bit counting/select

[hassnaa-arm]

Summary

To generate code compatible to streaming mode:

• enable custom-lowering ISD::CTLZ and ISD::CTPOP.
• disable combining OR into BSL.

• Testing files:
  • bit-counting.ll
  • bitselect.ll

[hassnaa-arm]

fix format

[david-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

I think you need brackets around the first part, i.e.

if ((!VT.is64BitVector() && !VT.is128BitVector()) ||

[hassnaa-arm]

Add brackets for the && check.

[david-arm]

LGTM!

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

8479

Odd indentation, have you used clang-format?

15691–15692

This comment seems out of date?

15694

Odd indentation, have you used clang-format?

Also, is this the same as (and should this be):

if (useSVEForFixedLengthVectorVT(VT, Subtarget->forceStreamingCompatibleSVE()))

?

[hassnaa-arm]

code format

[mgabka]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

I think this code is introducing a bug, before the combine had an early exit if the VT wasn't 64 or 128 bit vector, while now we allow scalable vectors, and the combine does not give correct results for scalable vectors, it is just not triggered because the "ISD::isBuildVectorAllZeros" and "ISD::isBuildVectorAllOnes" reject vector splat, but I think it would be better to have a clear early exit for scalable VT.
What do you think?

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

From what I can see, useSVEForFixedLengthVectorVT returns false if VT is a scalable vector, so I don't think this would be a problem.

[mgabka]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

exactly it returns false, so the branch is not taken and there is no early exit, while before we would have an early exit.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

Okay I see what you mean now. Yes, that does seem like a functional change. Did you find this by looking at the code, or did you come across a regression somewhere?

[mgabka]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

15693–15694

I had a downstream regression.

Looks like in upstream LLVM there is no tests for it, but the code below relies on VT.getScalarSizeInBits() being 64 or 128, check line 15720, so although for scalable vector luckily we are safe (thanks to the fact that ISD::isBuildVectorAllZeros does not accept spats), I think we might have issues for fixed width vectors where SVE should not be used, and where VT.getScalarSizeInBits() isn't 64 nor 128.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

void AArch64TargetLowering::addTypeForStreamingSVE(MVT VT) {
   setOperationAction(ISD::FP_ROUND, VT, Custom);
   setOperationAction(ISD::FCEIL, VT, Custom);
   setOperationAction(ISD::FFLOOR, VT, Custom);
   setOperationAction(ISD::FNEARBYINT, VT, Custom);
   setOperationAction(ISD::FRINT, VT, Custom);
   setOperationAction(ISD::FROUND, VT, Custom);
   setOperationAction(ISD::FROUNDEVEN, VT, Custom);
   setOperationAction(ISD::FTRUNC, VT, Custom);
+  setOperationAction(ISD::CTLZ, VT, Custom);
+  setOperationAction(ISD::CTPOP, VT, Custom);
   if (VT.isFloatingPoint()) {
     setCondCodeAction(ISD::SETO, VT, Expand);
     setCondCodeAction(ISD::SETOLT, VT, Expand);
     setCondCodeAction(ISD::SETOLE, VT, Expand);
     setCondCodeAction(ISD::SETULT, VT, Expand);
     setCondCodeAction(ISD::SETULE, VT, Expand);
     setCondCodeAction(ISD::SETUGE, VT, Expand);
     setCondCodeAction(ISD::SETUGT, VT, Expand);
if (IsParity)
       UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV,
                            DAG.getConstant(1, DL, MVT::i32));
 
     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
   }
 
   assert(!IsParity && "ISD::PARITY of vector types not supported");
 
-  if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
+  if (VT.isScalableVector() ||
+    useSVEForFixedLengthVectorVT(VT, Subtarget->forceStreamingCompatibleSVE()))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
 
   assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
           VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
          "Unexpected type for custom ctpop lowering");
 
   EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
   Val = DAG.getBitcast(VT8Bit, Val);
static SDValue tryCombineToBSL(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI) {
   EVT VT = N->getValueType(0);
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   if (!VT.isVector())
     return SDValue();
 
   // The combining code currently only works for NEON vectors. In particular,
   // it does not work for SVE when dealing with vectors wider than 128 bits.
-  if (!VT.is64BitVector() && !VT.is128BitVector())
+  if (!VT.is64BitVector() && !VT.is128BitVector() ||
+    DAG.getSubtarget<AArch64Subtarget>().forceStreamingCompatibleSVE())
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   if (N0.getOpcode() != ISD::AND)
     return SDValue();
 
   SDValue N1 = N->getOperand(1);
   if (N1.getOpcode() != ISD::AND)

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ;
 ; CLZ
 ;
 
 define <4 x i8> @ctlz_v4i8(<4 x i8> %op) #0 {
 ; CHECK-LABEL: ctlz_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    adrp x9, .LCPI0_1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT:    adrp x8, .LCPI0_1
+; CHECK-NEXT:    ldr d2, [x9, :lo12:.LCPI0_1]
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
-; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI0_1]
-; CHECK-NEXT:    clz v0.4h, v0.4h
-; CHECK-NEXT:    sub z0.h, z0.h, z1.h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    sub z0.h, z0.h, z2.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i8> @llvm.ctlz.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
 
 define <8 x i8> @ctlz_v8i8(<8 x i8> %op) #0 {
 ; CHECK-LABEL: ctlz_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.8b, v0.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.b, vl8
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
 
 define <16 x i8> @ctlz_v16i8(<16 x i8> %op) #0 {
 ; CHECK-LABEL: ctlz_v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.16b, v0.16b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
 
 define void @ctlz_v32i8(ptr %a) #0 {
 ; CHECK-LABEL: ctlz_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    clz v0.16b, v0.16b
-; CHECK-NEXT:    clz v1.16b, v1.16b
+; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    clz z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
   ret void
 }
 
 define <2 x i16> @ctlz_v2i16(<2 x i16> %op) #0 {
 ; CHECK-LABEL: ctlz_v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    adrp x9, .LCPI4_1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI4_0]
-; CHECK-NEXT:    adrp x8, .LCPI4_1
+; CHECK-NEXT:    ldr d2, [x9, :lo12:.LCPI4_1]
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
-; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI4_1]
-; CHECK-NEXT:    clz v0.2s, v0.2s
-; CHECK-NEXT:    sub z0.s, z0.s, z1.s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    sub z0.s, z0.s, z2.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
 
 define <4 x i16> @ctlz_v4i16(<4 x i16> %op) #0 {
 ; CHECK-LABEL: ctlz_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
 
 define <8 x i16> @ctlz_v8i16(<8 x i16> %op) #0 {
 ; CHECK-LABEL: ctlz_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
 
 define void @ctlz_v16i16(ptr %a) #0 {
 ; CHECK-LABEL: ctlz_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    clz v0.8h, v0.8h
-; CHECK-NEXT:    clz v1.8h, v1.8h
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    clz z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
   ret void
 }
 
 define <2 x i32> @ctlz_v2i32(<2 x i32> %op) #0 {
 ; CHECK-LABEL: ctlz_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
 
 define <4 x i32> @ctlz_v4i32(<4 x i32> %op) #0 {
 ; CHECK-LABEL: ctlz_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
 
 define void @ctlz_v8i32(ptr %a) #0 {
 ; CHECK-LABEL: ctlz_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    clz v0.4s, v0.4s
-; CHECK-NEXT:    clz v1.4s, v1.4s
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    clz z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
   ret void
 }
 

 ; CNT
 ;
 
 define <4 x i8> @ctpop_v4i8(<4 x i8> %op) #0 {
 ; CHECK-LABEL: ctpop_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI14_0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI14_0]
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
 
 define <8 x i8> @ctpop_v8i8(<8 x i8> %op) #0 {
 ; CHECK-LABEL: ctpop_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.b, vl8
+; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
 
 define <16 x i8> @ctpop_v16i8(<16 x i8> %op) #0 {
 ; CHECK-LABEL: ctpop_v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
 
 define void @ctpop_v32i8(ptr %a) #0 {
 ; CHECK-LABEL: ctpop_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
+; CHECK-NEXT:    cnt z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
   ret void
 }
 
 define <2 x i16> @ctpop_v2i16(<2 x i16> %op) #0 {
 ; CHECK-LABEL: ctpop_v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI18_0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI18_0]
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
 
 define <4 x i16> @ctpop_v4i16(<4 x i16> %op) #0 {
 ; CHECK-LABEL: ctpop_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
 
 define <8 x i16> @ctpop_v8i16(<8 x i16> %op) #0 {
 ; CHECK-LABEL: ctpop_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
 
 define void @ctpop_v16i16(ptr %a) #0 {
 ; CHECK-LABEL: ctpop_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    cnt z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
   ret void
 }
 
 define <2 x i32> @ctpop_v2i32(<2 x i32> %op) #0 {
 ; CHECK-LABEL: ctpop_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
 
 define <4 x i32> @ctpop_v4i32(<4 x i32> %op) #0 {
 ; CHECK-LABEL: ctpop_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
 
 define void @ctpop_v8i32(ptr %a) #0 {
 ; CHECK-LABEL: ctpop_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    cnt z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
   ret void
 }
 
 define <1 x i64> @ctpop_v1i64(<1 x i64> %op) #0 {
 ; CHECK-LABEL: ctpop_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
 
 define <2 x i64> @ctpop_v2i64(<2 x i64> %op) #0 {
 ; CHECK-LABEL: ctpop_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
 
 define void @ctpop_v4i64(ptr %a) #0 {
 ; CHECK-LABEL: ctpop_v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v1.4s, v1.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    cnt z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
   ret void
 }
 
 ;
 ; Count trailing zeros
 ;
 
 define <4 x i8> @cttz_v4i8(<4 x i8> %op) #0 {
 ; CHECK-LABEL: cttz_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI28_0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI28_0]
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
-; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
 
 define <8 x i8> @cttz_v8i8(<8 x i8> %op) #0 {
 ; CHECK-LABEL: cttz_v8i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.b, vl8
 ; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
-; CHECK-NEXT:    clz v0.8b, v0.8b
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
 
 define <16 x i8> @cttz_v16i8(<16 x i8> %op) #0 {
 ; CHECK-LABEL: cttz_v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
-; CHECK-NEXT:    clz v0.16b, v0.16b
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
 
 define void @cttz_v32i8(ptr %a) #0 {
 ; CHECK-LABEL: cttz_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
-; CHECK-NEXT:    clz v0.16b, v0.16b
+; CHECK-NEXT:    clz z0.b, p0/m, z0.b
 ; CHECK-NEXT:    rbit z1.b, p0/m, z1.b
-; CHECK-NEXT:    clz v1.16b, v1.16b
+; CHECK-NEXT:    clz z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
   ret void
 }
 
 define <2 x i16> @cttz_v2i16(<2 x i16> %op) #0 {
 ; CHECK-LABEL: cttz_v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI32_0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI32_0]
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
-; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
 
 define <4 x i16> @cttz_v4i16(<4 x i16> %op) #0 {
 ; CHECK-LABEL: cttz_v4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
-; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
 
 define <8 x i16> @cttz_v8i16(<8 x i16> %op) #0 {
 ; CHECK-LABEL: cttz_v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
-; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
 
 define void @cttz_v16i16(ptr %a) #0 {
 ; CHECK-LABEL: cttz_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
-; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    rbit z1.h, p0/m, z1.h
-; CHECK-NEXT:    clz v1.8h, v1.8h
+; CHECK-NEXT:    clz z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
   ret void
 }
 
 define <2 x i32> @cttz_v2i32(<2 x i32> %op) #0 {
 ; CHECK-LABEL: cttz_v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
-; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
 
 define <4 x i32> @cttz_v4i32(<4 x i32> %op) #0 {
 ; CHECK-LABEL: cttz_v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ptrue p0.s, vl4
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
-; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
 
 define void @cttz_v8i32(ptr %a) #0 {
 ; CHECK-LABEL: cttz_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
-; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    rbit z1.s, p0/m, z1.s
-; CHECK-NEXT:    clz v1.4s, v1.4s
+; CHECK-NEXT:    clz z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
   ret void
 }
 

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitselect.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s
 
 target triple = "aarch64"
 
 ;
 ; NOTE: SVE lowering for the BSP pseudoinst is not currently implemented, so we
 ;       don't currently expect the code below to lower to BSL/BIT/BIF. Once
 ;       this is implemented, this test will be fleshed out.
 ;
 
 define <8 x i32> @fixed_bitselect_v8i32(ptr %pre_cond_ptr, ptr %left_ptr, ptr %right_ptr) #0 {
 ; CHECK-LABEL: fixed_bitselect_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI0_0
-; CHECK-NEXT:    ldp q0, q2, [x0]
-; CHECK-NEXT:    ldp q3, q1, [x1]
-; CHECK-NEXT:    ldp q5, q6, [x2]
-; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT:    sub z2.s, z4.s, z2.s
-; CHECK-NEXT:    sub z0.s, z4.s, z0.s
-; CHECK-NEXT:    bsl v0.16b, v3.16b, v5.16b
-; CHECK-NEXT:    bif v1.16b, v6.16b, v2.16b
+; CHECK-NEXT:    ldp q1, q0, [x0]
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    adrp x8, .LCPI0_1
+; CHECK-NEXT:    ldp q3, q4, [x1]
+; CHECK-NEXT:    sub z6.s, z2.s, z1.s
+; CHECK-NEXT:    sub z2.s, z2.s, z0.s
+; CHECK-NEXT:    and z3.d, z6.d, z3.d
+; CHECK-NEXT:    ldp q7, q16, [x2]
+; CHECK-NEXT:    and z2.d, z2.d, z4.d
+; CHECK-NEXT:    ldr q5, [x8, :lo12:.LCPI0_1]
+; CHECK-NEXT:    add z1.s, z1.s, z5.s
+; CHECK-NEXT:    add z0.s, z0.s, z5.s
+; CHECK-NEXT:    and z4.d, z0.d, z16.d
+; CHECK-NEXT:    and z0.d, z1.d, z7.d
+; CHECK-NEXT:    orr z0.d, z0.d, z3.d
+; CHECK-NEXT:    orr z1.d, z4.d, z2.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
   %pre_cond = load <8 x i32>, ptr %pre_cond_ptr
   %left = load <8 x i32>, ptr %left_ptr
   %right = load <8 x i32>, ptr %right_ptr
 
   %neg_cond = sub <8 x i32> zeroinitializer, %pre_cond
   %min_cond = add <8 x i32> %pre_cond, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
   %left_bits_0 = and <8 x i32> %neg_cond, %left
   %right_bits_0 = and <8 x i32> %min_cond, %right
   %bsl0000 = or <8 x i32> %right_bits_0, %left_bits_0
   ret <8 x i32> %bsl0000
 }
 
 attributes #0 = { "target-features"="+sve" }