[SDAG][RISCV] Avoid expanding is-power-of-2 pattern on riscv32/64 with zbb

[dtcxzyw]

Summary

This patch adjusts the legality check for riscv to use cpop/cpopw since isOperationLegal(ISD::CTPOP, MVT::i32) returns false on rv64gc_zbb.
Clang vs gcc: https://godbolt.org/z/rc3s4hjPh

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

18223

The vector part of this isn't tested.

[dtcxzyw]

Add missing vector tests.

[dtcxzyw]

Tune instseq for ctpop_i32/64_ugt_one.

[luke]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

18223

I think if we check that the operation is legal then we don’t need to check for zbb/zvbb, since it’s only legal with those extensions

[dtcxzyw]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

18223

I am not familiar with the riscv vector crypto extension. I just do the same check as https://github.com/llvm/llvm-project/blob/3c4d2221de0e9e7b259a308aab7e72e3f2f926f1/llvm/lib/Target/RISCV/RISCVISelLowering.cpp#L764.

It is worth noting that isOperationLegal(ISD::CTPOP, MVT::i32) returns false on rv64gc_zbb. We will miss some optimizations if we only check the legality of ctpop.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

18223

@luke957 it won't be "legal" for fixed vectors since we will need to Custom lower to a RISCVISD node with VL.

[dtcxzyw]

Ping.

[craig.topper]

llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll

860

These tests already pass.

[dtcxzyw]

llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll

860

The vector part of this isn't tested.

Should I remove these useless tests? We always emit vcpop for the vector part.

[craig.topper]

llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll

860

You can keep them. Just be clear in the description that they already pass.

We do need additional tests for fixed vector without "vscale x". I bet they don't pass without this patch.

[dtcxzyw]

• Rebase
• Add fixed-length vector tests

[craig.topper]

llvm/test/CodeGen/RISCV/rv64zbb.ll

700

I meant with V and the Zvbb extensions enabled so we have vector cpop instructions.

[dtcxzyw]

• Rebase
• Add fixed-length vector tests with +v,+zvbb

[dtcxzyw]

llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll

1366

These fixed-vector tests should be moved into fixed-vectors-ctpop.ll.

[dtcxzyw]

• Rebase
• Move fixed-length vector tests into fixed-vectors-ctpop.ll

[dtcxzyw]

Ping.

[dtcxzyw]

Ping.

[craig.topper]

LGTM

llvm/include/llvm/CodeGen/TargetLowering.h

virtual bool isCheapToSpeculateCtlz(Type *Ty) const {
     return false;
   }
 
   /// Return true if ctlz instruction is fast.
   virtual bool isCtlzFast() const {
     return false;
   }
 
+  /// Return true if ctpop instruction is fast.
+  virtual bool isCtpopFast(EVT VT) const {
+    return isOperationLegal(ISD::CTPOP, VT);
+  }
+
   /// Return the maximum number of "x & (x - 1)" operations that can be done
   /// instead of deferring to a custom CTPOP.
   virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
     return 1;
   }
 
   /// Return true if instruction generated for equality comparison is folded
   /// with instruction generated for signed comparison.

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
 
   EVT CTVT = CTPOP.getValueType();
   SDValue CTOp = CTPOP.getOperand(0);
 
   // Expand a power-of-2-or-zero comparison based on ctpop:
   // (ctpop x) u< 2 -> (x & x-1) == 0
   // (ctpop x) u> 1 -> (x & x-1) != 0
   if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
-    // Keep the CTPOP if it is a legal vector op.
-    if (CTVT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
+    // Keep the CTPOP if it is a cheap vector op.
+    if (CTVT.isVector() && TLI.isCtpopFast(CTVT))
       return SDValue();
 
     unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
     if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
       return SDValue();
     if (C1 == 0 && (Cond == ISD::SETULT))
       return SDValue(); // This is handled elsewhere.
 
     unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
 
     SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
     SDValue Result = CTOp;
     for (unsigned i = 0; i < Passes; i++) {
       SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
       Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
     }
     ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
     return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
   }
 
   // Expand a power-of-2 comparison based on ctpop:
   // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
   // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
   if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
-    // Keep the CTPOP if it is legal.
-    if (TLI.isOperationLegal(ISD::CTPOP, CTVT))
+    // Keep the CTPOP if it is cheap.
+    if (TLI.isCtpopFast(CTVT))
       return SDValue();
 
     SDValue Zero = DAG.getConstant(0, dl, CTVT);
     SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
     assert(CTVT.isInteger());
     ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
     SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
     SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

llvm/lib/Target/RISCV/RISCVISelLowering.h

bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
     return NumElem >= 4;
   }
 
   bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
     return VT.isScalarInteger();
   }
   bool convertSelectOfConstantsToMath(EVT VT) const override { return true; }
 
+  bool isCtpopFast(EVT VT) const override;
+
+  unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override;
+
   bool preferZeroCompareBranch() const override { return true; }
 
   bool shouldInsertFencesForAtomic(const Instruction *I) const override {
     return isa<LoadInst>(I) || isa<StoreInst>(I);
   }
   Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
                                 AtomicOrdering Ord) const override;
   Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

RISCVTargetLowering::getTargetMMOFlags(const MemSDNode &Node) const {
   return TargetFlags;
 }
 
 bool RISCVTargetLowering::areTwoSDNodeTargetMMOFlagsMergeable(
     const MemSDNode &NodeX, const MemSDNode &NodeY) const {
   return getTargetMMOFlags(NodeX) == getTargetMMOFlags(NodeY);
 }
 
+bool RISCVTargetLowering::isCtpopFast(EVT VT) const {
+  if (VT.isScalableVector())
+    return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
+  return Subtarget.hasStdExtZbb() &&
+         (VT == MVT::i32 || VT == MVT::i64 || VT.isFixedLengthVector());
+}
+
+unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT,
+                                                 ISD::CondCode Cond) const {
+  return isCtpopFast(VT) ? 0 : 1;
+}
+
 namespace llvm::RISCVVIntrinsicsTable {
 
 #define GET_RISCVVIntrinsicsTable_IMPL
 #include "RISCVGenSearchableTables.inc"
 
 } // namespace llvm::RISCVVIntrinsicsTable

llvm/test/CodeGen/RISCV/rv32zbb.ll

 ; RV32ZBB-LABEL: ctpop_i32:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    cpop a0, a0
 ; RV32ZBB-NEXT:    ret
   %1 = call i32 @llvm.ctpop.i32(i32 %a)
   ret i32 %1
 }
 
+define i1 @ctpop_i32_ult_two(i32 signext %a) nounwind {
+; RV32I-LABEL: ctpop_i32_ult_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i32_ult_two:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ult i32 %1, 2
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_ugt_one(i32 signext %a) nounwind {
+; RV32I-LABEL: ctpop_i32_ugt_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i32_ugt_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    xori a0, a0, 1
+; RV32ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ugt i32 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_eq_one(i32 signext %a) nounwind {
+; RV32I-LABEL: ctpop_i32_eq_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i32_eq_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    seqz a0, a0
+; RV32ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp eq i32 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_ne_one(i32 signext %a) nounwind {
+; RV32I-LABEL: ctpop_i32_ne_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    snez a1, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i32_ne_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    snez a0, a0
+; RV32ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ne i32 %1, 1
+  ret i1 %2
+}
+
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+
+define <2 x i32> @ctpop_v2i32(<2 x i32> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s3, a2, 1365
+; RV32I-NEXT:    and a1, a1, s3
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lui a1, 209715
+; RV32I-NEXT:    addi s4, a1, 819
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, s4
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lui a1, 61681
+; RV32I-NEXT:    addi s5, a1, -241
+; RV32I-NEXT:    and a0, a0, s5
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli s2, a0, 24
+; RV32I-NEXT:    srli a0, s0, 1
+; RV32I-NEXT:    and a0, a0, s3
+; RV32I-NEXT:    sub s0, s0, a0
+; RV32I-NEXT:    and a0, s0, s4
+; RV32I-NEXT:    srli s0, s0, 2
+; RV32I-NEXT:    and a1, s0, s4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    and a0, a0, s5
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    mv a0, s2
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i32:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  ret <2 x i32> %1
+}
+
+define <2 x i1> @ctpop_v2i32_ult_two(<2 x i32> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i32_ult_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi a2, a1, -1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i32_ult_two:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    sltiu a1, a1, 2
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ult <2 x i32> %1, <i32 2, i32 2>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_ugt_one(<2 x i32> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i32_ugt_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    addi a2, a1, -1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    snez a1, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i32_ugt_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    xori a0, a0, 1
+; RV32ZBB-NEXT:    sltiu a1, a1, 2
+; RV32ZBB-NEXT:    xori a1, a1, 1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ugt <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_eq_one(<2 x i32> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i32_eq_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    addi a2, a1, -1
+; RV32I-NEXT:    and a2, a1, a2
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    snez a1, a1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i32_eq_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    seqz a0, a0
+; RV32ZBB-NEXT:    addi a1, a1, -1
+; RV32ZBB-NEXT:    seqz a1, a1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp eq <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_ne_one(<2 x i32> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i32_ne_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    snez a2, a2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    addi a2, a1, -1
+; RV32I-NEXT:    and a2, a1, a2
+; RV32I-NEXT:    snez a2, a2
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i32_ne_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    snez a0, a0
+; RV32ZBB-NEXT:    addi a1, a1, -1
+; RV32ZBB-NEXT:    snez a1, a1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ne <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
 declare i64 @llvm.ctpop.i64(i64)
 
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctpop_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
 ; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill

 ; RV32ZBB-NEXT:    cpop a0, a0
 ; RV32ZBB-NEXT:    add a0, a0, a1
 ; RV32ZBB-NEXT:    li a1, 0
 ; RV32ZBB-NEXT:    ret
   %1 = call i64 @llvm.ctpop.i64(i64 %a)
   ret i64 %1
 }
 
+define i1 @ctpop_i64_ugt_two(i64 %a) nounwind {
+; RV32I-LABEL: ctpop_i64_ugt_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i64_ugt_two:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ult i64 %1, 2
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_ugt_one(i64 %a) nounwind {
+; RV32I-LABEL: ctpop_i64_ugt_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i64_ugt_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    xori a0, a0, 1
+; RV32ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ugt i64 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_eq_one(i64 %a) nounwind {
+; RV32I-LABEL: ctpop_i64_eq_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    seqz a3, a0
+; RV32I-NEXT:    sub a3, a1, a3
+; RV32I-NEXT:    and a3, a1, a3
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i64_eq_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    seqz a0, a0
+; RV32ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp eq i64 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_ne_one(i64 %a) nounwind {
+; RV32I-LABEL: ctpop_i64_ne_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a0, -1
+; RV32I-NEXT:    and a2, a0, a2
+; RV32I-NEXT:    seqz a3, a0
+; RV32I-NEXT:    sub a3, a1, a3
+; RV32I-NEXT:    and a3, a1, a3
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    snez a2, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_i64_ne_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    add a0, a0, a1
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    snez a0, a0
+; RV32ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ne i64 %1, 1
+  ret i1 %2
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+
+define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -48
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a0, 4(a1)
+; RV32I-NEXT:    lw s2, 8(a1)
+; RV32I-NEXT:    lw s5, 12(a1)
+; RV32I-NEXT:    lw s6, 0(a1)
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s3, a2, 1365
+; RV32I-NEXT:    and a1, a1, s3
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lui a1, 209715
+; RV32I-NEXT:    addi s4, a1, 819
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, s4
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lui a1, 61681
+; RV32I-NEXT:    addi s7, a1, -241
+; RV32I-NEXT:    and a0, a0, s7
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli s8, a0, 24
+; RV32I-NEXT:    srli a0, s6, 1
+; RV32I-NEXT:    and a0, a0, s3
+; RV32I-NEXT:    sub a0, s6, a0
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, s4
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    and a0, a0, s7
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    add s8, a0, s8
+; RV32I-NEXT:    srli a0, s5, 1
+; RV32I-NEXT:    and a0, a0, s3
+; RV32I-NEXT:    sub a0, s5, a0
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, s4
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    and a0, a0, s7
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli s5, a0, 24
+; RV32I-NEXT:    srli a0, s2, 1
+; RV32I-NEXT:    and a0, a0, s3
+; RV32I-NEXT:    sub a0, s2, a0
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, s4
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    and a0, a0, s7
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    add a0, a0, s5
+; RV32I-NEXT:    sw zero, 12(s0)
+; RV32I-NEXT:    sw zero, 4(s0)
+; RV32I-NEXT:    sw a0, 8(s0)
+; RV32I-NEXT:    sw s8, 0(s0)
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i64:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a2, 4(a1)
+; RV32ZBB-NEXT:    lw a3, 0(a1)
+; RV32ZBB-NEXT:    lw a4, 8(a1)
+; RV32ZBB-NEXT:    lw a1, 12(a1)
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    cpop a3, a3
+; RV32ZBB-NEXT:    add a2, a3, a2
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a3, a4
+; RV32ZBB-NEXT:    add a1, a3, a1
+; RV32ZBB-NEXT:    sw zero, 12(a0)
+; RV32ZBB-NEXT:    sw zero, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 8(a0)
+; RV32ZBB-NEXT:    sw a2, 0(a0)
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  ret <2 x i64> %1
+}
+
+define <2 x i1> @ctpop_v2i64_ult_two(<2 x i64> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i64_ult_two:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a1, 0(a0)
+; RV32I-NEXT:    lw a2, 12(a0)
+; RV32I-NEXT:    lw a3, 8(a0)
+; RV32I-NEXT:    lw a0, 4(a0)
+; RV32I-NEXT:    addi a4, a1, -1
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    sub a1, a0, a1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi a1, a3, -1
+; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    seqz a3, a3
+; RV32I-NEXT:    sub a3, a2, a3
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i64_ult_two:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a1, 12(a0)
+; RV32ZBB-NEXT:    lw a2, 8(a0)
+; RV32ZBB-NEXT:    lw a3, 0(a0)
+; RV32ZBB-NEXT:    lw a0, 4(a0)
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    add a1, a2, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    cpop a2, a3
+; RV32ZBB-NEXT:    add a0, a2, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    sltiu a1, a1, 2
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ult <2 x i64> %1, <i64 2, i64 2>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_ugt_one(<2 x i64> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i64_ugt_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a1, 0(a0)
+; RV32I-NEXT:    lw a2, 12(a0)
+; RV32I-NEXT:    lw a3, 8(a0)
+; RV32I-NEXT:    lw a0, 4(a0)
+; RV32I-NEXT:    addi a4, a1, -1
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    sub a1, a0, a1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    addi a1, a3, -1
+; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    seqz a3, a3
+; RV32I-NEXT:    sub a3, a2, a3
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    snez a1, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i64_ugt_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a1, 12(a0)
+; RV32ZBB-NEXT:    lw a2, 8(a0)
+; RV32ZBB-NEXT:    lw a3, 0(a0)
+; RV32ZBB-NEXT:    lw a0, 4(a0)
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    add a1, a2, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    cpop a2, a3
+; RV32ZBB-NEXT:    add a0, a2, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    xori a0, a0, 1
+; RV32ZBB-NEXT:    sltiu a1, a1, 2
+; RV32ZBB-NEXT:    xori a1, a1, 1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ugt <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_eq_one(<2 x i64> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i64_eq_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a1, 0(a0)
+; RV32I-NEXT:    lw a2, 12(a0)
+; RV32I-NEXT:    lw a3, 8(a0)
+; RV32I-NEXT:    lw a0, 4(a0)
+; RV32I-NEXT:    addi a4, a1, -1
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    seqz a5, a1
+; RV32I-NEXT:    sub a5, a0, a5
+; RV32I-NEXT:    and a5, a0, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    seqz a4, a4
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    and a0, a0, a4
+; RV32I-NEXT:    addi a1, a3, -1
+; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    seqz a4, a3
+; RV32I-NEXT:    sub a4, a2, a4
+; RV32I-NEXT:    and a4, a2, a4
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    snez a2, a2
+; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i64_eq_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a1, 12(a0)
+; RV32ZBB-NEXT:    lw a2, 8(a0)
+; RV32ZBB-NEXT:    lw a3, 0(a0)
+; RV32ZBB-NEXT:    lw a0, 4(a0)
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    add a1, a2, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    cpop a2, a3
+; RV32ZBB-NEXT:    add a0, a2, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    seqz a0, a0
+; RV32ZBB-NEXT:    addi a1, a1, -1
+; RV32ZBB-NEXT:    seqz a1, a1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp eq <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_ne_one(<2 x i64> %a) nounwind {
+; RV32I-LABEL: ctpop_v2i64_ne_one:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a1, 0(a0)
+; RV32I-NEXT:    lw a2, 12(a0)
+; RV32I-NEXT:    lw a3, 8(a0)
+; RV32I-NEXT:    lw a0, 4(a0)
+; RV32I-NEXT:    addi a4, a1, -1
+; RV32I-NEXT:    and a4, a1, a4
+; RV32I-NEXT:    seqz a5, a1
+; RV32I-NEXT:    sub a5, a0, a5
+; RV32I-NEXT:    and a5, a0, a5
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    snez a4, a4
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, a0, a4
+; RV32I-NEXT:    addi a1, a3, -1
+; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    seqz a4, a3
+; RV32I-NEXT:    sub a4, a2, a4
+; RV32I-NEXT:    and a4, a2, a4
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    snez a1, a1
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop_v2i64_ne_one:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a1, 12(a0)
+; RV32ZBB-NEXT:    lw a2, 8(a0)
+; RV32ZBB-NEXT:    lw a3, 0(a0)
+; RV32ZBB-NEXT:    lw a0, 4(a0)
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    add a1, a2, a1
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    cpop a2, a3
+; RV32ZBB-NEXT:    add a0, a2, a0
+; RV32ZBB-NEXT:    addi a0, a0, -1
+; RV32ZBB-NEXT:    snez a0, a0
+; RV32ZBB-NEXT:    addi a1, a1, -1
+; RV32ZBB-NEXT:    snez a1, a1
+; RV32ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ne <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
 define i32 @sextb_i32(i32 %a) nounwind {
 ; RV32I-LABEL: sextb_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    srai a0, a0, 24
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: sextb_i32:
; RV32ZBB-NEXT:    ret
   %shl = shl i64 %a, 48
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
 }
 
 define i32 @min_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: min_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    blt a0, a1, .LBB10_2
+; RV32I-NEXT:    blt a0, a1, .LBB28_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:  .LBB10_2:
+; RV32I-NEXT:  .LBB28_2:
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: min_i32:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    min a0, a0, a1
 ; RV32ZBB-NEXT:    ret
   %cmp = icmp slt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 ; As we are not matching directly i64 code patterns on RV32 some i64 patterns
 ; don't have yet any matching bit manipulation instructions on RV32.
 ; This test is presented here in case future expansions of the Bitmanip
 ; extensions introduce instructions suitable for this pattern.
 
 define i64 @min_i64(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: min_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    beq a1, a3, .LBB11_2
+; CHECK-NEXT:    beq a1, a3, .LBB29_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    slt a4, a1, a3
-; CHECK-NEXT:    beqz a4, .LBB11_3
-; CHECK-NEXT:    j .LBB11_4
-; CHECK-NEXT:  .LBB11_2:
+; CHECK-NEXT:    beqz a4, .LBB29_3
+; CHECK-NEXT:    j .LBB29_4
+; CHECK-NEXT:  .LBB29_2:
 ; CHECK-NEXT:    sltu a4, a0, a2
-; CHECK-NEXT:    bnez a4, .LBB11_4
-; CHECK-NEXT:  .LBB11_3:
+; CHECK-NEXT:    bnez a4, .LBB29_4
+; CHECK-NEXT:  .LBB29_3:
 ; CHECK-NEXT:    mv a0, a2
 ; CHECK-NEXT:    mv a1, a3
-; CHECK-NEXT:  .LBB11_4:
+; CHECK-NEXT:  .LBB29_4:
 ; CHECK-NEXT:    ret
   %cmp = icmp slt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define i32 @max_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: max_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    blt a1, a0, .LBB12_2
+; RV32I-NEXT:    blt a1, a0, .LBB30_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:  .LBB12_2:
+; RV32I-NEXT:  .LBB30_2:
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: max_i32:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    max a0, a0, a1
 ; RV32ZBB-NEXT:    ret
   %cmp = icmp sgt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 ; As we are not matching directly i64 code patterns on RV32 some i64 patterns
 ; don't have yet any matching bit manipulation instructions on RV32.
 ; This test is presented here in case future expansions of the Bitmanip
 ; extensions introduce instructions suitable for this pattern.
 
 define i64 @max_i64(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: max_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    beq a1, a3, .LBB13_2
+; CHECK-NEXT:    beq a1, a3, .LBB31_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    slt a4, a3, a1
-; CHECK-NEXT:    beqz a4, .LBB13_3
-; CHECK-NEXT:    j .LBB13_4
-; CHECK-NEXT:  .LBB13_2:
+; CHECK-NEXT:    beqz a4, .LBB31_3
+; CHECK-NEXT:    j .LBB31_4
+; CHECK-NEXT:  .LBB31_2:
 ; CHECK-NEXT:    sltu a4, a2, a0
-; CHECK-NEXT:    bnez a4, .LBB13_4
-; CHECK-NEXT:  .LBB13_3:
+; CHECK-NEXT:    bnez a4, .LBB31_4
+; CHECK-NEXT:  .LBB31_3:
 ; CHECK-NEXT:    mv a0, a2
 ; CHECK-NEXT:    mv a1, a3
-; CHECK-NEXT:  .LBB13_4:
+; CHECK-NEXT:  .LBB31_4:
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define i32 @minu_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: minu_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    bltu a0, a1, .LBB14_2
+; RV32I-NEXT:    bltu a0, a1, .LBB32_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:  .LBB14_2:
+; RV32I-NEXT:  .LBB32_2:
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: minu_i32:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    minu a0, a0, a1
 ; RV32ZBB-NEXT:    ret
   %cmp = icmp ult i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 ; As we are not matching directly i64 code patterns on RV32 some i64 patterns
 ; don't have yet any matching bit manipulation instructions on RV32.
 ; This test is presented here in case future expansions of the Bitmanip
 ; extensions introduce instructions suitable for this pattern.
 
 define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: minu_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    beq a1, a3, .LBB15_2
+; CHECK-NEXT:    beq a1, a3, .LBB33_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    sltu a4, a1, a3
-; CHECK-NEXT:    beqz a4, .LBB15_3
-; CHECK-NEXT:    j .LBB15_4
-; CHECK-NEXT:  .LBB15_2:
+; CHECK-NEXT:    beqz a4, .LBB33_3
+; CHECK-NEXT:    j .LBB33_4
+; CHECK-NEXT:  .LBB33_2:
 ; CHECK-NEXT:    sltu a4, a0, a2
-; CHECK-NEXT:    bnez a4, .LBB15_4
-; CHECK-NEXT:  .LBB15_3:
+; CHECK-NEXT:    bnez a4, .LBB33_4
+; CHECK-NEXT:  .LBB33_3:
 ; CHECK-NEXT:    mv a0, a2
 ; CHECK-NEXT:    mv a1, a3
-; CHECK-NEXT:  .LBB15_4:
+; CHECK-NEXT:  .LBB33_4:
 ; CHECK-NEXT:    ret
   %cmp = icmp ult i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define i32 @maxu_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: maxu_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    bltu a1, a0, .LBB16_2
+; RV32I-NEXT:    bltu a1, a0, .LBB34_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:  .LBB16_2:
+; RV32I-NEXT:  .LBB34_2:
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: maxu_i32:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    maxu a0, a0, a1
 ; RV32ZBB-NEXT:    ret
   %cmp = icmp ugt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 ; As we are not matching directly i64 code patterns on RV32 some i64 patterns
 ; don't have yet any matching bit manipulation instructions on RV32.
 ; This test is presented here in case future expansions of the Bitmanip
 ; extensions introduce instructions suitable for this pattern.
 
 define i64 @maxu_i64(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: maxu_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    beq a1, a3, .LBB17_2
+; CHECK-NEXT:    beq a1, a3, .LBB35_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    sltu a4, a3, a1
-; CHECK-NEXT:    beqz a4, .LBB17_3
-; CHECK-NEXT:    j .LBB17_4
-; CHECK-NEXT:  .LBB17_2:
+; CHECK-NEXT:    beqz a4, .LBB35_3
+; CHECK-NEXT:    j .LBB35_4
+; CHECK-NEXT:  .LBB35_2:
 ; CHECK-NEXT:    sltu a4, a2, a0
-; CHECK-NEXT:    bnez a4, .LBB17_4
-; CHECK-NEXT:  .LBB17_3:
+; CHECK-NEXT:    bnez a4, .LBB35_4
+; CHECK-NEXT:  .LBB35_3:
 ; CHECK-NEXT:    mv a0, a2
 ; CHECK-NEXT:    mv a1, a3
-; CHECK-NEXT:  .LBB17_4:
+; CHECK-NEXT:  .LBB35_4:
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 declare i32 @llvm.abs.i32(i32, i1 immarg)
 
; RV32ZBB-NEXT:    ret
   ret i32 %abs
 }
 
 declare i64 @llvm.abs.i64(i64, i1 immarg)
 
 define i64 @abs_i64(i64 %x) {
 ; CHECK-LABEL: abs_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    bgez a1, .LBB19_2
+; CHECK-NEXT:    bgez a1, .LBB37_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    snez a2, a0
 ; CHECK-NEXT:    neg a0, a0
 ; CHECK-NEXT:    neg a1, a1
 ; CHECK-NEXT:    sub a1, a1, a2
-; CHECK-NEXT:  .LBB19_2:
+; CHECK-NEXT:  .LBB37_2:
 ; CHECK-NEXT:    ret
   %abs = tail call i64 @llvm.abs.i64(i64 %x, i1 true)
   ret i64 %abs
 }
 
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV32I-LABEL: zexth_i32:
 ; RV32I:       # %bb.0:

llvm/test/CodeGen/RISCV/rv64zbb.ll

 ; RV64ZBB-LABEL: ctpop_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    cpopw a0, a0
 ; RV64ZBB-NEXT:    ret
   %1 = call i32 @llvm.ctpop.i32(i32 %a)
   ret i32 %1
 }
 
+define i1 @ctpop_i32_ult_two(i32 signext %a) nounwind {
+; RV64I-LABEL: ctpop_i32_ult_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i32_ult_two:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ult i32 %1, 2
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_ugt_one(i32 signext %a) nounwind {
+; RV64I-LABEL: ctpop_i32_ugt_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i32_ugt_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    xori a0, a0, 1
+; RV64ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ugt i32 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_eq_one(i32 signext %a) nounwind {
+; RV64I-LABEL: ctpop_i32_eq_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a1, a0, -1
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i32_eq_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    seqz a0, a0
+; RV64ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp eq i32 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i32_ne_one(i32 signext %a) nounwind {
+; RV64I-LABEL: ctpop_i32_ne_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a1, a0, -1
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i32_ne_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    snez a0, a0
+; RV64ZBB-NEXT:    ret
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  %2 = icmp ne i32 %1, 1
+  ret i1 %2
+}
+
 define signext i32 @ctpop_i32_load(ptr %p) nounwind {
 ; RV64I-LABEL: ctpop_i32_load:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 0(a0)
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525

 ; RV64ZBB-NEXT:    lwu a0, 0(a0)
 ; RV64ZBB-NEXT:    cpopw a0, a0
 ; RV64ZBB-NEXT:    ret
   %a = load i32, ptr %p
   %1 = call i32 @llvm.ctpop.i32(i32 %a)
   ret i32 %1
 }
 
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+
+define <2 x i32> @ctpop_v2i32(<2 x i32> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 349525
+; RV64I-NEXT:    addiw s3, a2, 1365
+; RV64I-NEXT:    and a1, a1, s3
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    addiw s4, a1, 819
+; RV64I-NEXT:    and a1, a0, s4
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, s4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 61681
+; RV64I-NEXT:    addiw s5, a1, -241
+; RV64I-NEXT:    and a0, a0, s5
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw s1, a1, 257
+; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srliw s2, a0, 24
+; RV64I-NEXT:    srli a0, s0, 1
+; RV64I-NEXT:    and a0, a0, s3
+; RV64I-NEXT:    sub s0, s0, a0
+; RV64I-NEXT:    and a0, s0, s4
+; RV64I-NEXT:    srli s0, s0, 2
+; RV64I-NEXT:    and a1, s0, s4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    and a0, a0, s5
+; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srliw a1, a0, 24
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i32:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    cpopw a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  ret <2 x i32> %1
+}
+
+define <2 x i1> @ctpop_v2i32_ult_two(<2 x i32> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i32_ult_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a2, a0, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    addiw a2, a1, -1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i32_ult_two:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a1, a1
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    sltiu a1, a1, 2
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ult <2 x i32> %1, <i32 2, i32 2>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_ugt_one(<2 x i32> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i32_ugt_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addiw a2, a0, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    addiw a2, a1, -1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i32_ugt_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a1, a1
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    xori a0, a0, 1
+; RV64ZBB-NEXT:    sltiu a1, a1, 2
+; RV64ZBB-NEXT:    xori a1, a1, 1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ugt <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_eq_one(<2 x i32> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i32_eq_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a2, a1
+; RV64I-NEXT:    sext.w a3, a0
+; RV64I-NEXT:    addiw a4, a0, -1
+; RV64I-NEXT:    and a0, a0, a4
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    snez a3, a3
+; RV64I-NEXT:    and a0, a3, a0
+; RV64I-NEXT:    addiw a3, a1, -1
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    snez a2, a2
+; RV64I-NEXT:    and a1, a2, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i32_eq_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a1, a1
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    seqz a0, a0
+; RV64ZBB-NEXT:    addi a1, a1, -1
+; RV64ZBB-NEXT:    seqz a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp eq <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i32_ne_one(<2 x i32> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i32_ne_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a2, a1
+; RV64I-NEXT:    sext.w a3, a0
+; RV64I-NEXT:    addiw a4, a0, -1
+; RV64I-NEXT:    and a0, a0, a4
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    seqz a3, a3
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    addiw a3, a1, -1
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i32_ne_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpopw a1, a1
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    snez a0, a0
+; RV64ZBB-NEXT:    addi a1, a1, -1
+; RV64ZBB-NEXT:    snez a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a)
+  %2 = icmp ne <2 x i32> %1, <i32 1, i32 1>
+  ret <2 x i1> %2
+}
+
 declare i64 @llvm.ctpop.i64(i64)
 
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1

 ; RV64ZBB-LABEL: ctpop_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    cpop a0, a0
 ; RV64ZBB-NEXT:    ret
   %1 = call i64 @llvm.ctpop.i64(i64 %a)
   ret i64 %1
 }
 
+define i1 @ctpop_i64_ugt_two(i64 %a) nounwind {
+; RV64I-LABEL: ctpop_i64_ugt_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i64_ugt_two:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ult i64 %1, 2
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_ugt_one(i64 %a) nounwind {
+; RV64I-LABEL: ctpop_i64_ugt_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i64_ugt_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    xori a0, a0, 1
+; RV64ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ugt i64 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_eq_one(i64 %a) nounwind {
+; RV64I-LABEL: ctpop_i64_eq_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i64_eq_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    seqz a0, a0
+; RV64ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp eq i64 %1, 1
+  ret i1 %2
+}
+
+define i1 @ctpop_i64_ne_one(i64 %a) nounwind {
+; RV64I-LABEL: ctpop_i64_ne_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i64_ne_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    snez a0, a0
+; RV64ZBB-NEXT:    ret
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  %2 = icmp ne i64 %1, 1
+  ret i1 %2
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+
+define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 349525
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a3, a2, 32
+; RV64I-NEXT:    add s3, a2, a3
+; RV64I-NEXT:    and a1, a1, s3
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a2, a1, 32
+; RV64I-NEXT:    add s4, a1, a2
+; RV64I-NEXT:    and a1, a0, s4
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, s4
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 61681
+; RV64I-NEXT:    addiw a1, a1, -241
+; RV64I-NEXT:    slli a2, a1, 32
+; RV64I-NEXT:    add s5, a1, a2
+; RV64I-NEXT:    and a0, a0, s5
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw s1, a1, 257
+; RV64I-NEXT:    slli a1, s1, 32
+; RV64I-NEXT:    add s1, s1, a1
+; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli s2, a0, 56
+; RV64I-NEXT:    srli a0, s0, 1
+; RV64I-NEXT:    and a0, a0, s3
+; RV64I-NEXT:    sub s0, s0, a0
+; RV64I-NEXT:    and a0, s0, s4
+; RV64I-NEXT:    srli s0, s0, 2
+; RV64I-NEXT:    and a1, s0, s4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    and a0, a0, s5
+; RV64I-NEXT:    mv a1, s1
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i64:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  ret <2 x i64> %1
+}
+
+define <2 x i1> @ctpop_v2i64_ult_two(<2 x i64> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i64_ult_two:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a0, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    addi a2, a1, -1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i64_ult_two:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    sltiu a1, a1, 2
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ult <2 x i64> %1, <i64 2, i64 2>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_ugt_one(<2 x i64> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i64_ugt_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a0, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    addi a2, a1, -1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i64_ugt_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    xori a0, a0, 1
+; RV64ZBB-NEXT:    sltiu a1, a1, 2
+; RV64ZBB-NEXT:    xori a1, a1, 1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ugt <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_eq_one(<2 x i64> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i64_eq_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a0, -1
+; RV64I-NEXT:    and a2, a0, a2
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    addi a2, a1, -1
+; RV64I-NEXT:    and a2, a1, a2
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    snez a1, a1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i64_eq_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    seqz a0, a0
+; RV64ZBB-NEXT:    addi a1, a1, -1
+; RV64ZBB-NEXT:    seqz a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp eq <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
+define <2 x i1> @ctpop_v2i64_ne_one(<2 x i64> %a) nounwind {
+; RV64I-LABEL: ctpop_v2i64_ne_one:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a0, -1
+; RV64I-NEXT:    and a2, a0, a2
+; RV64I-NEXT:    snez a2, a2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    addi a2, a1, -1
+; RV64I-NEXT:    and a2, a1, a2
+; RV64I-NEXT:    snez a2, a2
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_v2i64_ne_one:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    snez a0, a0
+; RV64ZBB-NEXT:    addi a1, a1, -1
+; RV64ZBB-NEXT:    snez a1, a1
+; RV64ZBB-NEXT:    ret
+  %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  %2 = icmp ne <2 x i64> %1, <i64 1, i64 1>
+  ret <2 x i1> %2
+}
+
 define signext i32 @sextb_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: sextb_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 56
 ; RV64I-NEXT:    srai a0, a0, 56
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: sextb_i32:
; RV64ZBB-NEXT:    ret
   %shl = shl i64 %a, 48
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
 }
 
 define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: min_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB18_2
+; RV64I-NEXT:    blt a0, a1, .LBB36_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB18_2:
+; RV64I-NEXT:  .LBB36_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    min a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp slt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 define i64 @min_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: min_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB19_2
+; RV64I-NEXT:    blt a0, a1, .LBB37_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB19_2:
+; RV64I-NEXT:  .LBB37_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    min a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp slt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: max_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB20_2
+; RV64I-NEXT:    blt a1, a0, .LBB38_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB20_2:
+; RV64I-NEXT:  .LBB38_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    max a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp sgt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 define i64 @max_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: max_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB21_2
+; RV64I-NEXT:    blt a1, a0, .LBB39_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB21_2:
+; RV64I-NEXT:  .LBB39_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    max a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp sgt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: minu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB22_2
+; RV64I-NEXT:    bltu a0, a1, .LBB40_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB22_2:
+; RV64I-NEXT:  .LBB40_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    minu a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp ult i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: minu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB23_2
+; RV64I-NEXT:    bltu a0, a1, .LBB41_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB23_2:
+; RV64I-NEXT:  .LBB41_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    minu a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp ult i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b
   ret i64 %cond
 }
 
 define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: maxu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB24_2
+; RV64I-NEXT:    bltu a1, a0, .LBB42_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB24_2:
+; RV64I-NEXT:  .LBB42_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    maxu a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp ugt i32 %a, %b
   %cond = select i1 %cmp, i32 %a, i32 %b
   ret i32 %cond
 }
 
 define i64 @maxu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: maxu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB25_2
+; RV64I-NEXT:    bltu a1, a0, .LBB43_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB25_2:
+; RV64I-NEXT:  .LBB43_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    maxu a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %cmp = icmp ugt i64 %a, %b
   %cond = select i1 %cmp, i64 %a, i64 %b

llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll

 ; CHECK-ZVBB-LABEL: ctpop_nxv16i32:
 ; CHECK-ZVBB:       # %bb.0:
 ; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
 ; CHECK-ZVBB-NEXT:    ret
   %a = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> %va)
   ret <vscale x 16 x i32> %a
 }
+
+; We always emit vcpop.v for the scalable vector
+define <vscale x 16 x i1> @ctpop_nxv16i32_ult_two(<vscale x 16 x i32> %va) {
+; CHECK-LABEL: ctpop_nxv16i32_ult_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v8, v8, v16
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv16i32_ult_two:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsleu.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> %va)
+  %cmp = icmp ult <vscale x 16 x i32> %a, shufflevector (<vscale x 16 x i32> insertelement (<vscale x 16 x i32> poison, i32 2, i64 0), <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer)
+  ret <vscale x 16 x i1> %cmp
+}
+
+define <vscale x 16 x i1> @ctpop_nxv16i32_ugt_one(<vscale x 16 x i32> %va) {
+; CHECK-LABEL: ctpop_nxv16i32_ugt_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v8, v8, v16
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv16i32_ugt_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsgtu.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> %va)
+  %cmp = icmp ugt <vscale x 16 x i32> %a, shufflevector (<vscale x 16 x i32> insertelement (<vscale x 16 x i32> poison, i32 1, i64 0), <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer)
+  ret <vscale x 16 x i1> %cmp
+}
+
+define <vscale x 16 x i1> @ctpop_nxv16i32_eq_one(<vscale x 16 x i32> %va) {
+; CHECK-LABEL: ctpop_nxv16i32_eq_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v16, v8, v16
+; CHECK-NEXT:    vmseq.vi v24, v16, 0
+; CHECK-NEXT:    vmsne.vi v16, v8, 0
+; CHECK-NEXT:    vmand.mm v0, v16, v24
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv16i32_eq_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmseq.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> %va)
+  %cmp = icmp eq <vscale x 16 x i32> %a, shufflevector (<vscale x 16 x i32> insertelement (<vscale x 16 x i32> poison, i32 1, i64 0), <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer)
+  ret <vscale x 16 x i1> %cmp
+}
+
+define <vscale x 16 x i1> @ctpop_nxv16i32_ne_one(<vscale x 16 x i32> %va) {
+; CHECK-LABEL: ctpop_nxv16i32_ne_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v16, v8, v16
+; CHECK-NEXT:    vmsne.vi v24, v16, 0
+; CHECK-NEXT:    vmseq.vi v16, v8, 0
+; CHECK-NEXT:    vmor.mm v0, v16, v24
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv16i32_ne_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsne.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> %va)
+  %cmp = icmp ne <vscale x 16 x i32> %a, shufflevector (<vscale x 16 x i32> insertelement (<vscale x 16 x i32> poison, i32 1, i64 0), <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer)
+  ret <vscale x 16 x i1> %cmp
+}
+
 declare <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32>)
 
 define <vscale x 1 x i64> @ctpop_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32-LABEL: ctpop_nxv1i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vsrl.vi v9, v8, 1
 ; RV32-NEXT:    lui a0, 349525

 ; CHECK-ZVBB-LABEL: ctpop_nxv8i64:
 ; CHECK-ZVBB:       # %bb.0:
 ; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
 ; CHECK-ZVBB-NEXT:    ret
   %a = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> %va)
   ret <vscale x 8 x i64> %a
 }
+
+; We always emit vcpop.v for the scalable vector
+define <vscale x 8 x i1> @ctpop_nxv8i64_ult_two(<vscale x 8 x i64> %va) {
+; CHECK-LABEL: ctpop_nxv8i64_ult_two:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v8, v8, v16
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv8i64_ult_two:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsleu.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> %va)
+  %cmp = icmp ult <vscale x 8 x i64> %a, shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 2, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+  ret <vscale x 8 x i1> %cmp
+}
+
+define <vscale x 8 x i1> @ctpop_nxv8i64_ugt_one(<vscale x 8 x i64> %va) {
+; CHECK-LABEL: ctpop_nxv8i64_ugt_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v8, v8, v16
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv8i64_ugt_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsgtu.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> %va)
+  %cmp = icmp ugt <vscale x 8 x i64> %a, shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+  ret <vscale x 8 x i1> %cmp
+}
+
+define <vscale x 8 x i1> @ctpop_nxv8i64_eq_one(<vscale x 8 x i64> %va) {
+; CHECK-LABEL: ctpop_nxv8i64_eq_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v16, v8, v16
+; CHECK-NEXT:    vmseq.vi v24, v16, 0
+; CHECK-NEXT:    vmsne.vi v16, v8, 0
+; CHECK-NEXT:    vmand.mm v0, v16, v24
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv8i64_eq_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmseq.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> %va)
+  %cmp = icmp eq <vscale x 8 x i64> %a, shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+  ret <vscale x 8 x i1> %cmp
+}
+
+define <vscale x 8 x i1> @ctpop_nxv8i64_ne_one(<vscale x 8 x i64> %va) {
+; CHECK-LABEL: ctpop_nxv8i64_ne_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vand.vv v16, v8, v16
+; CHECK-NEXT:    vmsne.vi v24, v16, 0
+; CHECK-NEXT:    vmseq.vi v16, v8, 0
+; CHECK-NEXT:    vmor.mm v0, v16, v24
+; CHECK-NEXT:    ret
+;
+; CHECK-ZVBB-LABEL: ctpop_nxv8i64_ne_one:
+; CHECK-ZVBB:       # %bb.0:
+; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-ZVBB-NEXT:    vcpop.v v8, v8
+; CHECK-ZVBB-NEXT:    vmsne.vi v0, v8, 1
+; CHECK-ZVBB-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64> %va)
+  %cmp = icmp ne <vscale x 8 x i64> %a, shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+  ret <vscale x 8 x i1> %cmp
+}
+
 declare <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64>)