[RISCV] Lower unary zvbb ops for fixed vectors

[luke]

Summary

This reuses the same strategy for fixed vectors as other ops, i.e. custom lower
to a scalable *_vl SD node.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

5111

The Subtarget.hasStdExtZvbb() check could be an assert.

[luke]

Assert zvbb extension

[craig.topper]

LGTM

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (Subtarget.useRVVForFixedLengthVectors()) {
         // nodes' operands.
         setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
                             ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
                             ISD::VECREDUCE_UMIN},
                            VT, Custom);
 
         setOperationAction(IntegerVPOps, VT, Custom);
 
+        if (Subtarget.hasStdExtZvbb()) {
+          setOperationAction({ISD::BITREVERSE, ISD::BSWAP, ISD::CTLZ,
+                              ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ,
+                              ISD::CTTZ_ZERO_UNDEF, ISD::CTPOP},
+                             VT, Custom);
+        } else {
           // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
           // range of f32.
           EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
           if (isTypeLegal(FloatVT))
             setOperationAction(
                 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
                 Custom);
         }
+      }
 
       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
         // There are no extending loads or truncating stores.
         for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
           setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
           setTruncStoreAction(VT, InnerVT, Expand);
         }
 
#define VP_CASE(NODE)                                                          \
   OP_CASE(MULHU)
   OP_CASE(SDIV)
   OP_CASE(SREM)
   OP_CASE(UDIV)
   OP_CASE(UREM)
   OP_CASE(SHL)
   OP_CASE(SRA)
   OP_CASE(SRL)
+  OP_CASE(BSWAP)
+  OP_CASE(CTTZ)
+  OP_CASE(CTLZ)
+  OP_CASE(CTPOP)
+  OP_CASE(BITREVERSE)
   OP_CASE(SADDSAT)
   OP_CASE(UADDSAT)
   OP_CASE(SSUBSAT)
   OP_CASE(USUBSAT)
   OP_CASE(FADD)
   OP_CASE(FSUB)
   OP_CASE(FMUL)
   OP_CASE(FDIV)
#define VP_CASE(NODE)                                                          \
   VP_CASE(SETCC)      // VP_SETCC
   VP_CASE(SINT_TO_FP) // VP_SINT_TO_FP
   VP_CASE(UINT_TO_FP) // VP_UINT_TO_FP
   VP_CASE(BITREVERSE) // VP_BITREVERSE
   VP_CASE(BSWAP)      // VP_BSWAP
   VP_CASE(CTLZ)       // VP_CTLZ
   VP_CASE(CTTZ)       // VP_CTTZ
   VP_CASE(CTPOP)      // VP_CTPOP
+  case ISD::CTLZ_ZERO_UNDEF:
   case ISD::VP_CTLZ_ZERO_UNDEF:
     return RISCVISD::CTLZ_VL;
+  case ISD::CTTZ_ZERO_UNDEF:
   case ISD::VP_CTTZ_ZERO_UNDEF:
     return RISCVISD::CTTZ_VL;
   case ISD::FMA:
   case ISD::VP_FMA:
     return RISCVISD::VFMADD_VL;
   case ISD::STRICT_FMA:
     return RISCVISD::STRICT_VFMADD_VL;
   case ISD::AND:
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::INTRINSIC_W_CHAIN:
     return LowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
   case ISD::IS_FPCLASS:
     return LowerIS_FPCLASS(Op, DAG);
   case ISD::BITREVERSE: {
     MVT VT = Op.getSimpleValueType();
+    if (VT.isFixedLengthVector()) {
+      assert(Subtarget.hasStdExtZvbb());
+      return lowerToScalableOp(Op, DAG);
+    }
     SDLoc DL(Op);
     assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
     assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
     // Expand bitreverse to a bswap(rev8) followed by brev8.
     SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
     return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
   }
   case ISD::TRUNCATE:
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::MULHU:
   case ISD::AND:
   case ISD::OR:
   case ISD::XOR:
   case ISD::SDIV:
   case ISD::SREM:
   case ISD::UDIV:
   case ISD::UREM:
+  case ISD::BSWAP:
+  case ISD::CTPOP:
     return lowerToScalableOp(Op, DAG);
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:
     if (Op.getSimpleValueType().isFixedLengthVector())
       return lowerToScalableOp(Op, DAG);
     // This can be called for an i32 shift amount that needs to be promoted.
     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
     return lowerToScalableOp(Op, DAG);
   case ISD::ABS:
   case ISD::VP_ABS:
     return lowerABS(Op, DAG);
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF:
+  case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF:
+    if (Subtarget.hasStdExtZvbb())
+      return lowerToScalableOp(Op, DAG);
+    assert(Op.getOpcode() != ISD::CTTZ);
     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
   case ISD::VSELECT:
     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
   case ISD::FCOPYSIGN:
     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
   case ISD::STRICT_FADD:
   case ISD::STRICT_FSUB:
   case ISD::STRICT_FMUL:

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32,LMULMAX2-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64,LMULMAX2-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64,LMULMAX1-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
 
 define void @bitreverse_v8i16(ptr %x, ptr %y) {
 ; RV32-LABEL: bitreverse_v8i16:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; RV32-NEXT:    vle16.v v8, (a0)
 ; RV32-NEXT:    vsrl.vi v9, v8, 8
 ; RV32-NEXT:    vsll.vi v8, v8, 8

 ; RV64-NEXT:    lui a1, 5
 ; RV64-NEXT:    addiw a1, a1, 1365
 ; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vand.vx v8, v8, a1
 ; RV64-NEXT:    vadd.vv v8, v8, v8
 ; RV64-NEXT:    vor.vv v8, v9, v8
 ; RV64-NEXT:    vse16.v v8, (a0)
 ; RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a)
   store <8 x i16> %c, ptr %x
   ret void
 }
 declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
 

 ; RV64-NEXT:    lui a1, 349525
 ; RV64-NEXT:    addiw a1, a1, 1365
 ; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vand.vx v8, v8, a1
 ; RV64-NEXT:    vadd.vv v8, v8, v8
 ; RV64-NEXT:    vor.vv v8, v9, v8
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a)
   store <4 x i32> %c, ptr %x
   ret void
 }
 declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
 

 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vand.vx v8, v8, a1
 ; RV64-NEXT:    vadd.vv v8, v8, v8
 ; RV64-NEXT:    vor.vv v8, v9, v8
 ; RV64-NEXT:    vse64.v v8, (a0)
 ; RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a)
   store <2 x i64> %c, ptr %x
   ret void
 }
 declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 1
 ; LMULMAX1-RV64-NEXT:    vand.vx v10, v10, a4
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a4
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v9
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v10, v9
 ; LMULMAX1-RV64-NEXT:    vse16.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse16.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a)
   store <16 x i16> %c, ptr %x
   ret void
 }
 declare <16 x i16> @llvm.bitreverse.v16i16(<16 x i16>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 1
 ; LMULMAX1-RV64-NEXT:    vand.vx v10, v10, a5
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a5
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v9
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v10, v9
 ; LMULMAX1-RV64-NEXT:    vse32.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse32.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a)
   store <8 x i32> %c, ptr %x
   ret void
 }
 declare <8 x i32> @llvm.bitreverse.v8i32(<8 x i32>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v8, 1
 ; LMULMAX1-RV64-NEXT:    vand.vx v10, v10, t1
 ; LMULMAX1-RV64-NEXT:    vand.vx v8, v8, t1
 ; LMULMAX1-RV64-NEXT:    vadd.vv v8, v8, v8
 ; LMULMAX1-RV64-NEXT:    vor.vv v8, v10, v8
 ; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX1-RV64-NEXT:    vse64.v v9, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bitreverse_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vbrev.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a)
   store <4 x i64> %c, ptr %x
   ret void
 }
 declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>)

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,LMULMAX2-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,LMULMAX2-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,LMULMAX1-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
 
 define void @bswap_v8i16(ptr %x, ptr %y) {
 ; CHECK-LABEL: bswap_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    vsrl.vi v9, v8, 8
 ; CHECK-NEXT:    vsll.vi v8, v8, 8
 ; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %a)
   store <8 x i16> %c, ptr %x
   ret void
 }
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 

 ; RV64-NEXT:    vor.vv v9, v9, v10
 ; RV64-NEXT:    vsll.vi v10, v8, 24
 ; RV64-NEXT:    vand.vx v8, v8, a1
 ; RV64-NEXT:    vsll.vi v8, v8, 8
 ; RV64-NEXT:    vor.vv v8, v10, v8
 ; RV64-NEXT:    vor.vv v8, v8, v9
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a)
   store <4 x i32> %c, ptr %x
   ret void
 }
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
 

 ; RV64-NEXT:    vsll.vx v11, v8, a1
 ; RV64-NEXT:    vand.vx v8, v8, a3
 ; RV64-NEXT:    vsll.vx v8, v8, a2
 ; RV64-NEXT:    vor.vv v8, v11, v8
 ; RV64-NEXT:    vor.vv v8, v8, v10
 ; RV64-NEXT:    vor.vv v8, v8, v9
 ; RV64-NEXT:    vse64.v v8, (a0)
 ; RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %a)
   store <2 x i64> %c, ptr %x
   ret void
 }
 declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 

 ; LMULMAX1-RV64-NEXT:    vsll.vi v8, v8, 8
 ; LMULMAX1-RV64-NEXT:    vor.vv v8, v8, v10
 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 8
 ; LMULMAX1-RV64-NEXT:    vsll.vi v9, v9, 8
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vse16.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse16.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %a)
   store <16 x i16> %c, ptr %x
   ret void
 }
 declare <16 x i16> @llvm.bswap.v16i16(<16 x i16>)
 

 ; LMULMAX1-RV64-NEXT:    vsll.vi v11, v9, 24
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a2
 ; LMULMAX1-RV64-NEXT:    vsll.vi v9, v9, 8
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v11, v9
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vse32.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse32.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %a)
   store <8 x i32> %c, ptr %x
   ret void
 }
 declare <8 x i32> @llvm.bswap.v8i32(<8 x i32>)
 

 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a4
 ; LMULMAX1-RV64-NEXT:    vsll.vx v9, v9, a3
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v12, v9
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v9, v11
 ; LMULMAX1-RV64-NEXT:    vor.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vse64.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse64.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: bswap_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vrev8.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %a)
   store <4 x i64> %c, ptr %x
   ret void
 }
 declare <4 x i64> @llvm.bswap.v4i64(<4 x i64>)

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32I
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64I
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64f,+f -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32F
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64f,+f -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64F
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32D
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64D
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=LMULMAX8
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=LMULMAX8
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
 
 define void @ctlz_v16i8(ptr %x, ptr %y) nounwind {
 ; CHECK-LABEL: ctlz_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vor.vv v8, v8, v9

 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v10, v8, 0
 ; LMULMAX8-NEXT:    li a1, 134
 ; LMULMAX8-NEXT:    vrsub.vx v8, v10, a1
 ; LMULMAX8-NEXT:    li a1, 8
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = load <16 x i8>, ptr %y
   %c = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
   store <16 x i8> %c, ptr %x
   ret void
 }
 declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
 

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v10, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v10, 23
 ; LMULMAX8-NEXT:    li a1, 142
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 16
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
   store <8 x i16> %c, ptr %x
   ret void
 }
 declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
 

 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v10, a1
 ; LMULMAX8-NEXT:    li a1, 1054
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 32
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
   store <4 x i32> %c, ptr %x
   ret void
 }
 declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
 

 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 1086
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 64
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
   store <2 x i64> %c, ptr %x
   ret void
 }
 declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
 

 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v12, v8, 0
 ; LMULMAX8-NEXT:    li a1, 134
 ; LMULMAX8-NEXT:    vrsub.vx v8, v12, a1
 ; LMULMAX8-NEXT:    li a1, 8
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v32i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    li a1, 32
+; ZVBB-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = load <32 x i8>, ptr %y
   %c = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %a, i1 false)
   store <32 x i8> %c, ptr %x
   ret void
 }
 declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>, i1)
 

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v12, 23
 ; LMULMAX8-NEXT:    li a1, 142
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 16
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %a, i1 false)
   store <16 x i16> %c, ptr %x
   ret void
 }
 declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>, i1)
 

 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v12, a1
 ; LMULMAX8-NEXT:    li a1, 1054
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 32
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %a, i1 false)
   store <8 x i32> %c, ptr %x
   ret void
 }
 declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)
 

 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 1086
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 64
 ; LMULMAX8-NEXT:    vminu.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %a, i1 false)
   store <4 x i64> %c, ptr %x
   ret void
 }
 declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)
 

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v10
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v12, 23
 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v10, v8, 0
 ; LMULMAX8-NEXT:    li a1, 134
 ; LMULMAX8-NEXT:    vrsub.vx v8, v10, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = load <16 x i8>, ptr %y
   %c = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 true)
   store <16 x i8> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v8i16(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; LMULMAX8-NEXT:    vle16.v v8, (a0)
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v10, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v10, 23
 ; LMULMAX8-NEXT:    li a1, 142
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 true)
   store <8 x i16> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v4i32(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vle32.v v8, (a0)
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v10, v8
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v10, a1
 ; LMULMAX8-NEXT:    li a1, 1054
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 true)
   store <4 x i32> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vfcvt.f.xu.v v8, v8
 ; LMULMAX8-NEXT:    fsrm a1
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 1086
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 true)
   store <2 x i64> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v32i8(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v16, v12
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v16, 23
 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v12, v8, 0
 ; LMULMAX8-NEXT:    li a1, 134
 ; LMULMAX8-NEXT:    vrsub.vx v8, v12, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v32i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    li a1, 32
+; ZVBB-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = load <32 x i8>, ptr %y
   %c = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %a, i1 true)
   store <32 x i8> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v16i16(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; LMULMAX8-NEXT:    vle16.v v8, (a0)
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v12, 23
 ; LMULMAX8-NEXT:    li a1, 142
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %a, i1 true)
   store <16 x i16> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v8i32(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vle32.v v8, (a0)
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v8
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v12, a1
 ; LMULMAX8-NEXT:    li a1, 1054
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %a, i1 true)
   store <8 x i32> %c, ptr %x
   ret void
 }
 
 define void @ctlz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vfcvt.f.xu.v v8, v8
 ; LMULMAX8-NEXT:    fsrm a1
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-NEXT:    li a1, 1086
 ; LMULMAX8-NEXT:    vrsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: ctlz_zero_undef_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vclz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %a, i1 true)
   store <4 x i64> %c, ptr %x
   ret void
 }

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
 
 define void @ctpop_v16i8(ptr %x, ptr %y) {
 ; CHECK-LABEL: ctpop_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    li a1, 85
 ; CHECK-NEXT:    vand.vx v9, v9, a1
 ; CHECK-NEXT:    vsub.vv v8, v8, v9
 ; CHECK-NEXT:    li a1, 51
 ; CHECK-NEXT:    vand.vx v9, v8, a1
 ; CHECK-NEXT:    vsrl.vi v8, v8, 2
 ; CHECK-NEXT:    vand.vx v8, v8, a1
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 4
 ; CHECK-NEXT:    vadd.vv v8, v8, v9
 ; CHECK-NEXT:    vand.vi v8, v8, 15
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = load <16 x i8>, ptr %y
   %c = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %a)
   store <16 x i8> %c, ptr %x
   ret void
 }
 declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
 

 ; LMULMAX1-RV64-NEXT:    lui a1, 1
 ; LMULMAX1-RV64-NEXT:    addiw a1, a1, -241
 ; LMULMAX1-RV64-NEXT:    vand.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    li a1, 257
 ; LMULMAX1-RV64-NEXT:    vmul.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    vsrl.vi v8, v8, 8
 ; LMULMAX1-RV64-NEXT:    vse16.v v8, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a)
   store <8 x i16> %c, ptr %x
   ret void
 }
 declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
 

 ; LMULMAX1-RV64-NEXT:    addiw a1, a1, -241
 ; LMULMAX1-RV64-NEXT:    vand.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    lui a1, 4112
 ; LMULMAX1-RV64-NEXT:    addiw a1, a1, 257
 ; LMULMAX1-RV64-NEXT:    vmul.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    vsrl.vi v8, v8, 24
 ; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a)
   store <4 x i32> %c, ptr %x
   ret void
 }
 declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
 

 ; LMULMAX1-RV64-NEXT:    addiw a1, a1, 257
 ; LMULMAX1-RV64-NEXT:    slli a2, a1, 32
 ; LMULMAX1-RV64-NEXT:    add a1, a1, a2
 ; LMULMAX1-RV64-NEXT:    vmul.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    li a1, 56
 ; LMULMAX1-RV64-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
   store <2 x i64> %c, ptr %x
   ret void
 }
 declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
 

 ; LMULMAX1-NEXT:    vand.vx v9, v9, a3
 ; LMULMAX1-NEXT:    vadd.vv v9, v10, v9
 ; LMULMAX1-NEXT:    vsrl.vi v10, v9, 4
 ; LMULMAX1-NEXT:    vadd.vv v9, v9, v10
 ; LMULMAX1-NEXT:    vand.vi v9, v9, 15
 ; LMULMAX1-NEXT:    vse8.v v9, (a0)
 ; LMULMAX1-NEXT:    vse8.v v8, (a1)
 ; LMULMAX1-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v32i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    li a1, 32
+; ZVBB-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = load <32 x i8>, ptr %y
   %c = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %a)
   store <32 x i8> %c, ptr %x
   ret void
 }
 declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 4
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a4
 ; LMULMAX1-RV64-NEXT:    vmul.vx v9, v9, a5
 ; LMULMAX1-RV64-NEXT:    vsrl.vi v9, v9, 8
 ; LMULMAX1-RV64-NEXT:    vse16.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse16.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %a)
   store <16 x i16> %c, ptr %x
   ret void
 }
 declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 4
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a4
 ; LMULMAX1-RV64-NEXT:    vmul.vx v9, v9, a5
 ; LMULMAX1-RV64-NEXT:    vsrl.vi v9, v9, 24
 ; LMULMAX1-RV64-NEXT:    vse32.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse32.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %a)
   store <8 x i32> %c, ptr %x
   ret void
 }
 declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
 

 ; LMULMAX1-RV64-NEXT:    vsrl.vi v10, v9, 4
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vand.vx v9, v9, a4
 ; LMULMAX1-RV64-NEXT:    vmul.vx v9, v9, a5
 ; LMULMAX1-RV64-NEXT:    vsrl.vx v9, v9, a6
 ; LMULMAX1-RV64-NEXT:    vse64.v v9, (a0)
 ; LMULMAX1-RV64-NEXT:    vse64.v v8, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: ctpop_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vcpop.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a)
   store <4 x i64> %c, ptr %x
   ret void
 }
 declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32I
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64I
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64x -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+zve64f,+f -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32F
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+zve64f,+f -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64F
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV32,LMULMAX2-RV32D
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2,LMULMAX2-RV64,LMULMAX2-RV64D
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=LMULMAX8,LMULMAX8-RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=LMULMAX8,LMULMAX8-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
+; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvbb -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=ZVBB
 
 define void @cttz_v16i8(ptr %x, ptr %y) nounwind {
 ; CHECK-LABEL: cttz_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsub.vx v9, v8, a1

 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v9, v10, 0
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    vsub.vx v8, v9, a1
 ; LMULMAX8-NEXT:    vmerge.vim v8, v8, 8, v0
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = load <16 x i8>, ptr %y
   %c = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
   store <16 x i8> %c, ptr %x
   ret void
 }
 declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
 

 ; LMULMAX8-NEXT:    vnsrl.wi v9, v10, 23
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v9, v9, a1
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    li a1, 16
 ; LMULMAX8-NEXT:    vmerge.vxm v8, v9, a1, v0
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
   store <8 x i16> %c, ptr %x
   ret void
 }
 declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
 

 ; LMULMAX8-NEXT:    vnsrl.wx v9, v10, a1
 ; LMULMAX8-NEXT:    li a1, 1023
 ; LMULMAX8-NEXT:    vsub.vx v9, v9, a1
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    li a1, 32
 ; LMULMAX8-NEXT:    vmerge.vxm v8, v9, a1, v0
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
   store <4 x i32> %c, ptr %x
   ret void
 }
 declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
 

 ; LMULMAX8-RV64-NEXT:    vsrl.vx v9, v9, a1
 ; LMULMAX8-RV64-NEXT:    li a1, 1023
 ; LMULMAX8-RV64-NEXT:    vsub.vx v9, v9, a1
 ; LMULMAX8-RV64-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-RV64-NEXT:    li a1, 64
 ; LMULMAX8-RV64-NEXT:    vmerge.vxm v8, v9, a1, v0
 ; LMULMAX8-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
   store <2 x i64> %c, ptr %x
   ret void
 }
 declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
 

 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v10, v12, 0
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    vsub.vx v8, v10, a1
 ; LMULMAX8-NEXT:    vmerge.vim v8, v8, 8, v0
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v32i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    li a1, 32
+; ZVBB-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = load <32 x i8>, ptr %y
   %c = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
   store <32 x i8> %c, ptr %x
   ret void
 }
 declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
 

 ; LMULMAX8-NEXT:    vnsrl.wi v10, v12, 23
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v10, v10, a1
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    li a1, 16
 ; LMULMAX8-NEXT:    vmerge.vxm v8, v10, a1, v0
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
   store <16 x i16> %c, ptr %x
   ret void
 }
 declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
 

 ; LMULMAX8-NEXT:    vnsrl.wx v10, v12, a1
 ; LMULMAX8-NEXT:    li a1, 1023
 ; LMULMAX8-NEXT:    vsub.vx v10, v10, a1
 ; LMULMAX8-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-NEXT:    li a1, 32
 ; LMULMAX8-NEXT:    vmerge.vxm v8, v10, a1, v0
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
   store <8 x i32> %c, ptr %x
   ret void
 }
 declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
 

 ; LMULMAX8-RV64-NEXT:    vsrl.vx v10, v10, a1
 ; LMULMAX8-RV64-NEXT:    li a1, 1023
 ; LMULMAX8-RV64-NEXT:    vsub.vx v10, v10, a1
 ; LMULMAX8-RV64-NEXT:    vmseq.vi v0, v8, 0
 ; LMULMAX8-RV64-NEXT:    li a1, 64
 ; LMULMAX8-RV64-NEXT:    vmerge.vxm v8, v10, a1, v0
 ; LMULMAX8-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
   store <4 x i64> %c, ptr %x
   ret void
 }
 declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
 

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v10
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v12, 23
 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v10, v8, 0
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v8, v10, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v16i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = load <16 x i8>, ptr %y
   %c = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
   store <16 x i8> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v8i16(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vrsub.vi v9, v8, 0
 ; LMULMAX8-NEXT:    vand.vv v8, v8, v9
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v10, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v10, 23
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v8i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i16>, ptr %x
   %b = load <8 x i16>, ptr %y
   %c = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
   store <8 x i16> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v4i32(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vand.vv v8, v8, v9
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v10, v8
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v10, a1
 ; LMULMAX8-NEXT:    li a1, 1023
 ; LMULMAX8-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v4i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i32>, ptr %x
   %b = load <4 x i32>, ptr %y
   %c = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
   store <4 x i32> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-RV64-NEXT:    vfcvt.f.xu.v v8, v8
 ; LMULMAX8-RV64-NEXT:    fsrm a1
 ; LMULMAX8-RV64-NEXT:    li a1, 52
 ; LMULMAX8-RV64-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-RV64-NEXT:    li a1, 1023
 ; LMULMAX8-RV64-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v2i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
   %c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
   store <2 x i64> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v32i8(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v16, v12
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v16, 23
 ; LMULMAX8-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
 ; LMULMAX8-NEXT:    vnsrl.wi v12, v8, 0
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v8, v12, a1
 ; LMULMAX8-NEXT:    vse8.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v32i8:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    li a1, 32
+; ZVBB-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; ZVBB-NEXT:    vle8.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse8.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = load <32 x i8>, ptr %y
   %c = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
   store <32 x i8> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v16i16(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vrsub.vi v10, v8, 0
 ; LMULMAX8-NEXT:    vand.vv v8, v8, v10
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v8
 ; LMULMAX8-NEXT:    vnsrl.wi v8, v12, 23
 ; LMULMAX8-NEXT:    li a1, 127
 ; LMULMAX8-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v16i16:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZVBB-NEXT:    vle16.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse16.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <16 x i16>, ptr %x
   %b = load <16 x i16>, ptr %y
   %c = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
   store <16 x i16> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v8i32(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-NEXT:    vand.vv v8, v8, v10
 ; LMULMAX8-NEXT:    vfwcvt.f.xu.v v12, v8
 ; LMULMAX8-NEXT:    li a1, 52
 ; LMULMAX8-NEXT:    vnsrl.wx v8, v12, a1
 ; LMULMAX8-NEXT:    li a1, 1023
 ; LMULMAX8-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v8i32:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZVBB-NEXT:    vle32.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse32.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <8 x i32>, ptr %x
   %b = load <8 x i32>, ptr %y
   %c = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
   store <8 x i32> %c, ptr %x
   ret void
 }
 
 define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {

 ; LMULMAX8-RV64-NEXT:    vfcvt.f.xu.v v8, v8
 ; LMULMAX8-RV64-NEXT:    fsrm a1
 ; LMULMAX8-RV64-NEXT:    li a1, 52
 ; LMULMAX8-RV64-NEXT:    vsrl.vx v8, v8, a1
 ; LMULMAX8-RV64-NEXT:    li a1, 1023
 ; LMULMAX8-RV64-NEXT:    vsub.vx v8, v8, a1
 ; LMULMAX8-RV64-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-RV64-NEXT:    ret
+;
+; ZVBB-LABEL: cttz_zero_undef_v4i64:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZVBB-NEXT:    vle64.v v8, (a0)
+; ZVBB-NEXT:    vctz.v v8, v8
+; ZVBB-NEXT:    vse64.v v8, (a0)
+; ZVBB-NEXT:    ret
   %a = load <4 x i64>, ptr %x
   %b = load <4 x i64>, ptr %y
   %c = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
   store <4 x i64> %c, ptr %x
   ret void
 }