[LoongArch] Add codegen support for ISD::CTPOP, ISD::CTTZ and ISD::CTLZ

[xen0n]

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

394

switch (NumOp)?

llvm/lib/Target/LoongArch/LoongArchISelLowering.h

47

nit: Bit counting

49

Other *.w nodes are named *_W, so add an underscore for consistency?

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

690–693

no need to guard with Predicates = [IsLA32]?

[gonglingqin]

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

394

Thanks, I will change that.

llvm/lib/Target/LoongArch/LoongArchISelLowering.h

47

Thanks, I will change that.

49

Thanks, I will add an underscore.

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

690–693

It needs to guard with Predicates = [IsLA32], I will fix it, thanks.

[gonglingqin]

Address @xen0n's comments.

[xen0n]

Seems good to me.

P.S. If this lands after D131512 you have to refresh your testcases. But I'd happily rebase if that patch lands after this one too.

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

678–685

Okay so after you grouped the LA32 bit-counting ops together it may be better to move this block below, and add a // Bit counting operations to describe the two groups of operations. Adding these patterns near the multiplications just for saving an if Predicates = [IsLA64] block doesn't help with readability.

[gonglingqin]

In D131550#3712086, @xen0n wrote:

Seems good to me.

P.S. If this lands after D131512 you have to refresh your testcases. But I'd happily rebase if that patch lands after this one too.

Thanks for your reminding. I will refresh my testcases after D131512 landed.

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

678–685

Thanks, I will change that.

[gonglingqin]

Address @xen0n's comments.
Modify the location of LA64 bit-counting Ops and add comments to increase readability.

[SixWeining]

In D131550#3714696, @gonglingqin wrote:

In D131550#3712086, @xen0n wrote:

Seems good to me.

P.S. If this lands after D131512 you have to refresh your testcases. But I'd happily rebase if that patch lands after this one too.

Thanks for your reminding. I will refresh my testcases after D131512 landed.

D131512 has landed. You can rebase now.

[gonglingqin]

rebase

[SixWeining]

LGTM.

llvm/lib/Target/LoongArch/LoongArchISelLowering.h

enum NodeType : unsigned {
   ROTR_W,
 
   // FPR<->GPR transfer operations
   MOVGR2FR_W_LA64,
   MOVFR2GR_S_LA64,
 
   FTINT,
 
+  // Bit counting operations
+  CLZ_W,
+  CTZ_W,
+
   BSTRINS,
   BSTRPICK,
 
   // Byte swapping operations
   REVB_2H,
   REVB_2W,
 };
 } // end namespace LoongArchISD
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       LLVMContext &Context) const override;
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
                       SelectionDAG &DAG) const override;
   SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
+  bool isCheapToSpeculateCttz() const override;
+  bool isCheapToSpeculateCtlz() const override;
 
 private:
   /// Target-specific function used to lower LoongArch calling conventions.
   typedef bool LoongArchCCAssignFn(const DataLayout &DL, LoongArchABI::ABI ABI,
                                    unsigned ValNo, MVT ValVT,
                                    CCValAssign::LocInfo LocInfo,
                                    ISD::ArgFlagsTy ArgFlags, CCState &State,
                                    bool IsFixed, bool IsReg, Type *OrigTy);

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, GRLenVT,
                    MVT::i1, Promote);
 
   // TODO: add necessary setOperationAction calls later.
   setOperationAction(ISD::SHL_PARTS, GRLenVT, Custom);
   setOperationAction(ISD::SRA_PARTS, GRLenVT, Custom);
   setOperationAction(ISD::SRL_PARTS, GRLenVT, Custom);
   setOperationAction(ISD::FP_TO_SINT, GRLenVT, Custom);
   setOperationAction(ISD::ROTL, GRLenVT, Expand);
+  setOperationAction(ISD::CTPOP, GRLenVT, Expand);
 
   setOperationAction({ISD::GlobalAddress, ISD::ConstantPool}, GRLenVT, Custom);
 
   setOperationAction(ISD::DYNAMIC_STACKALLOC, GRLenVT, Expand);
   setOperationAction(ISD::VASTART, MVT::Other, Custom);
   setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
 
   if (Subtarget.is64Bit()) {
     setOperationAction(ISD::SHL, MVT::i32, Custom);
     setOperationAction(ISD::SRA, MVT::i32, Custom);
     setOperationAction(ISD::SRL, MVT::i32, Custom);
     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
     setOperationAction(ISD::ROTR, MVT::i32, Custom);
     setOperationAction(ISD::ROTL, MVT::i32, Custom);
+    setOperationAction(ISD::CTTZ, MVT::i32, Custom);
+    setOperationAction(ISD::CTLZ, MVT::i32, Custom);
     if (Subtarget.hasBasicF() && !Subtarget.hasBasicD())
       setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
   }
 
   // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
   // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
   // and i32 could still be byte-swapped relatively cheaply.
   setOperationAction(ISD::BSWAP, MVT::i16, Custom);
static LoongArchISD::NodeType getLoongArchWOpcode(unsigned Opcode) {
   case ISD::SRA:
     return LoongArchISD::SRA_W;
   case ISD::SRL:
     return LoongArchISD::SRL_W;
   case ISD::ROTR:
     return LoongArchISD::ROTR_W;
   case ISD::ROTL:
     return LoongArchISD::ROTL_W;
+  case ISD::CTTZ:
+    return LoongArchISD::CTZ_W;
+  case ISD::CTLZ:
+    return LoongArchISD::CLZ_W;
   }
 }
 
 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
 // node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
 // otherwise be promoted to i64, making it difficult to select the
 // SLL_W/.../*W later one because the fact the operation was originally of
 // type i8/i16/i32 is lost.
-static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
+static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
   SDLoc DL(N);
   LoongArchISD::NodeType WOpcode = getLoongArchWOpcode(N->getOpcode());
-  SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
+  SDValue NewOp0, NewRes;
+
+  switch (NumOp) {
+  default:
+    llvm_unreachable("Unexpected NumOp");
+  case 1: {
+    NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
+    NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0);
+    break;
+  }
+  case 2: {
+    NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
     SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
-  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
-  // ReplaceNodeResults requires we maintain the same type for the return value.
+    NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
+    break;
+  }
+    // TODO:Handle more NumOp.
+  }
+
+  // ReplaceNodeResults requires we maintain the same type for the return
+  // value.
   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
 }
 
 void LoongArchTargetLowering::ReplaceNodeResults(
     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
   SDLoc DL(N);
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Don't know how to legalize this operation");
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:
   case ISD::ROTR:
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
     if (N->getOperand(1).getOpcode() != ISD::Constant) {
-      Results.push_back(customLegalizeToWOp(N, DAG));
+      Results.push_back(customLegalizeToWOp(N, DAG, 2));
       break;
     }
     break;
   case ISD::ROTL:
     ConstantSDNode *CN;
     if ((CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))) {
-      Results.push_back(customLegalizeToWOp(N, DAG));
+      Results.push_back(customLegalizeToWOp(N, DAG, 2));
       break;
     }
     break;
   case ISD::FP_TO_SINT: {
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
     SDValue Src = N->getOperand(0);
     EVT VT = EVT::getFloatingPointVT(N->getValueSizeInBits(0));
case 32:
       // Only LA64 will get to here due to the size mismatch between VT and
       // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
       Tmp = DAG.getNode(LoongArchISD::REVB_2W, DL, GRLenVT, NewSrc);
       break;
     }
     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp));
     break;
   }
+  case ISD::CTLZ:
+  case ISD::CTTZ: {
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           "Unexpected custom legalisation");
+    Results.push_back(customLegalizeToWOp(N, DAG, 1));
+    break;
+  }
   }
 }
 
 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const LoongArchSubtarget &Subtarget) {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
case LoongArchISD::node:                                                     \
     NODE_NAME_CASE(BSTRPICK)
     NODE_NAME_CASE(MOVGR2FR_W_LA64)
     NODE_NAME_CASE(MOVFR2GR_S_LA64)
     NODE_NAME_CASE(FTINT)
     NODE_NAME_CASE(REVB_2H)
     NODE_NAME_CASE(REVB_2W)
     NODE_NAME_CASE(ROTR_W)
     NODE_NAME_CASE(ROTL_W)
+    NODE_NAME_CASE(CLZ_W)
+    NODE_NAME_CASE(CTZ_W)
   }
 #undef NODE_NAME_CASE
   return nullptr;
 }
 
 //===----------------------------------------------------------------------===//
 //                     Calling Convention Implementation
 //===----------------------------------------------------------------------===//
bool LoongArchTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   assert((VT == MVT::f32 || VT == MVT::f64) && "Unexpected VT");
 
   if (VT == MVT::f32 && !Subtarget.hasBasicF())
     return false;
   if (VT == MVT::f64 && !Subtarget.hasBasicD())
     return false;
   return (Imm.isZero() || Imm.isExactlyValue(+1.0));
 }
+
+bool LoongArchTargetLowering::isCheapToSpeculateCttz() const { return true; }
+
+bool LoongArchTargetLowering::isCheapToSpeculateCtlz() const { return true; }

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

 def loongarch_rotr_w : SDNode<"LoongArchISD::ROTR_W", SDT_LoongArchIntBinOpW>;
 def loongarch_rotl_w : SDNode<"LoongArchISD::ROTL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_bstrins
     : SDNode<"LoongArchISD::BSTRINS", SDT_LoongArchBStrIns>;
 def loongarch_bstrpick
     : SDNode<"LoongArchISD::BSTRPICK", SDT_LoongArchBStrPick>;
 def loongarch_revb_2h : SDNode<"LoongArchISD::REVB_2H", SDTUnaryOp>;
 def loongarch_revb_2w : SDNode<"LoongArchISD::REVB_2W", SDTUnaryOp>;
+def loongarch_clzw : SDNode<"LoongArchISD::CLZ_W", SDTIntBitCountUnaryOp>;
+def loongarch_ctzw : SDNode<"LoongArchISD::CTZ_W", SDTIntBitCountUnaryOp>;
 
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
 //===----------------------------------------------------------------------===//
 
 class ImmAsmOperand<string prefix, int width, string suffix>
     : AsmOperandClass {
   let Name = prefix # "Imm" # width # suffix;

 //===----------------------------------------------------------------------===//
 
 /// Generic pattern classes
 
 class PatGprGpr<SDPatternOperator OpNode, LAInst Inst>
     : Pat<(OpNode GPR:$rj, GPR:$rk), (Inst GPR:$rj, GPR:$rk)>;
 class PatGprGpr_32<SDPatternOperator OpNode, LAInst Inst>
     : Pat<(sext_inreg (OpNode GPR:$rj, GPR:$rk), i32), (Inst GPR:$rj, GPR:$rk)>;
+class PatGpr<SDPatternOperator OpNode, LAInst Inst>
+    : Pat<(OpNode GPR:$rj), (Inst GPR:$rj)>;
 
 class PatGprImm<SDPatternOperator OpNode, LAInst Inst, Operand ImmOpnd>
     : Pat<(OpNode GPR:$rj, ImmOpnd:$imm),
           (Inst GPR:$rj, ImmOpnd:$imm)>;
 class PatGprImm_32<SDPatternOperator OpNode, LAInst Inst, Operand ImmOpnd>
     : Pat<(sext_inreg (OpNode GPR:$rj, ImmOpnd:$imm), i32),
           (Inst GPR:$rj, ImmOpnd:$imm)>;
 

 // multiplication.
 def : Pat<(i64 (mul (sext_inreg GPR:$rj, i32), (sext_inreg GPR:$rk, i32))),
           (MULW_D_W GPR:$rj, GPR:$rk)>;
 // Select MULW_D_WU for calculating the full 64 bits product of i32xi32
 // unsigned multiplication.
 def : Pat<(i64 (mul (loongarch_bstrpick GPR:$rj, (i64 31), (i64 0)),
                     (loongarch_bstrpick GPR:$rk, (i64 31), (i64 0)))),
           (MULW_D_WU GPR:$rj, GPR:$rk)>;
 } // Predicates = [IsLA64]
 
 def : PatGprGpr<and, AND>;
 def : PatGprImm<and, ANDI, uimm12>;
 def : PatGprGpr<or, OR>;
 def : PatGprImm<or, ORI, uimm12>;
 def : PatGprGpr<xor, XOR>;
 def : PatGprImm<xor, XORI, uimm12>;
 def : Pat<(not GPR:$rj), (NOR GPR:$rj, R0)>;
 def : Pat<(not (or GPR:$rj, GPR:$rk)), (NOR GPR:$rj, GPR:$rk)>;
 def : Pat<(or GPR:$rj, (not GPR:$rk)), (ORN GPR:$rj, GPR:$rk)>;
 def : Pat<(and GPR:$rj, (not GPR:$rk)), (ANDN GPR:$rj, GPR:$rk)>;
 
+/// Bit counting operations
+
+let Predicates = [IsLA64] in {
+def : PatGpr<ctlz, CLZ_D>;
+def : PatGpr<cttz, CTZ_D>;
+def : Pat<(ctlz (not GPR:$rj)), (CLO_D GPR:$rj)>;
+def : Pat<(cttz (not GPR:$rj)), (CTO_D GPR:$rj)>;
+def : PatGpr<loongarch_clzw, CLZ_W>;
+def : PatGpr<loongarch_ctzw, CTZ_W>;
+def : Pat<(loongarch_clzw (not GPR:$rj)), (CLO_W GPR:$rj)>;
+def : Pat<(loongarch_ctzw (not GPR:$rj)), (CTO_W GPR:$rj)>;
+} // Predicates = [IsLA64]
+
+let Predicates = [IsLA32] in {
+def : PatGpr<ctlz, CLZ_W>;
+def : PatGpr<cttz, CTZ_W>;
+def : Pat<(ctlz (not GPR:$rj)), (CLO_W GPR:$rj)>;
+def : Pat<(cttz (not GPR:$rj)), (CTO_W GPR:$rj)>;
+} // Predicates = [IsLA32]
+
 /// FrameIndex calculations
 let Predicates = [IsLA32] in {
 def : Pat<(AddLike (i32 BaseAddr:$rj), simm12:$imm12),
           (ADDI_W (i32 BaseAddr:$rj), simm12:$imm12)>;
 } // Predicates = [IsLA32]
 let Predicates = [IsLA64] in {
 def : Pat<(AddLike (i64 BaseAddr:$rj), simm12:$imm12),
           (ADDI_D (i64 BaseAddr:$rj), simm12:$imm12)>;

llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+
+declare i8 @llvm.ctlz.i8(i8, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i8 @llvm.ctpop.i8(i8)
+declare i16 @llvm.ctpop.i16(i16)
+declare i32 @llvm.ctpop.i32(i32)
+declare i64 @llvm.ctpop.i64(i64)
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
+define i8 @test_ctlz_i8(i8 %a) nounwind {
+; LA32-LABEL: test_ctlz_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a0, $a0, 255
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, -24
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctlz_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a0, $a0, 255
+; LA64-NEXT:    clz.d $a0, $a0
+; LA64-NEXT:    addi.d $a0, $a0, -56
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i8 @llvm.ctlz.i8(i8 %a, i1 false)
+  ret i8 %tmp
+}
+
+define i16 @test_ctlz_i16(i16 %a) nounwind {
+; LA32-LABEL: test_ctlz_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, -16
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctlz_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
+; LA64-NEXT:    clz.d $a0, $a0
+; LA64-NEXT:    addi.d $a0, $a0, -48
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i16 @llvm.ctlz.i16(i16 %a, i1 false)
+  ret i16 %tmp
+}
+
+define i32 @test_ctlz_i32(i32 %a) nounwind {
+; LA32-LABEL: test_ctlz_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctlz_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    clz.w $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+  ret i32 %tmp
+}
+
+define i64 @test_ctlz_i64(i64 %a) nounwind {
+; LA32-LABEL: test_ctlz_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    sltu $a2, $zero, $a1
+; LA32-NEXT:    clz.w $a1, $a1
+; LA32-NEXT:    maskeqz $a1, $a1, $a2
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, 32
+; LA32-NEXT:    masknez $a0, $a0, $a2
+; LA32-NEXT:    or $a0, $a1, $a0
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctlz_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    clz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+  ret i64 %tmp
+}
+
+define i8 @test_not_ctlz_i8(i8 %a) nounwind {
+; LA32-LABEL: test_not_ctlz_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ori $a1, $zero, 255
+; LA32-NEXT:    andn $a0, $a1, $a0
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, -24
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_ctlz_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ori $a1, $zero, 255
+; LA64-NEXT:    andn $a0, $a1, $a0
+; LA64-NEXT:    clz.d $a0, $a0
+; LA64-NEXT:    addi.d $a0, $a0, -56
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i8 %a, -1
+  %tmp = call i8 @llvm.ctlz.i8(i8 %neg, i1 false)
+  ret i8 %tmp
+}
+
+define i16 @test_not_ctlz_i16(i16 %a) nounwind {
+; LA32-LABEL: test_not_ctlz_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    nor $a0, $a0, $zero
+; LA32-NEXT:    bstrpick.w $a0, $a0, 15, 0
+; LA32-NEXT:    clz.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, -16
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_ctlz_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    nor $a0, $a0, $zero
+; LA64-NEXT:    bstrpick.d $a0, $a0, 15, 0
+; LA64-NEXT:    clz.d $a0, $a0
+; LA64-NEXT:    addi.d $a0, $a0, -48
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i16 %a, -1
+  %tmp = call i16 @llvm.ctlz.i16(i16 %neg, i1 false)
+  ret i16 %tmp
+}
+
+define i32 @test_not_ctlz_i32(i32 %a) nounwind {
+; LA32-LABEL: test_not_ctlz_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    clo.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_ctlz_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    clo.w $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i32 %a, -1
+  %tmp = call i32 @llvm.ctlz.i32(i32 %neg, i1 false)
+  ret i32 %tmp
+}
+
+define i64 @test_not_ctlz_i64(i64 %a) nounwind {
+; LA32-LABEL: test_not_ctlz_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    nor $a2, $a1, $zero
+; LA32-NEXT:    sltu $a2, $zero, $a2
+; LA32-NEXT:    clo.w $a0, $a0
+; LA32-NEXT:    addi.w $a0, $a0, 32
+; LA32-NEXT:    masknez $a0, $a0, $a2
+; LA32-NEXT:    clo.w $a1, $a1
+; LA32-NEXT:    maskeqz $a1, $a1, $a2
+; LA32-NEXT:    or $a0, $a1, $a0
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_ctlz_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    clo.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i64 %a, -1
+  %tmp = call i64 @llvm.ctlz.i64(i64 %neg, i1 false)
+  ret i64 %tmp
+}
+
+define i8 @test_ctpop_i8(i8 %a) nounwind {
+; LA32-LABEL: test_ctpop_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    srli.w $a1, $a0, 1
+; LA32-NEXT:    andi $a1, $a1, 85
+; LA32-NEXT:    sub.w $a0, $a0, $a1
+; LA32-NEXT:    andi $a1, $a0, 51
+; LA32-NEXT:    srli.w $a0, $a0, 2
+; LA32-NEXT:    andi $a0, $a0, 51
+; LA32-NEXT:    add.w $a0, $a1, $a0
+; LA32-NEXT:    srli.w $a1, $a0, 4
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    andi $a0, $a0, 15
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctpop_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    srli.d $a1, $a0, 1
+; LA64-NEXT:    andi $a1, $a1, 85
+; LA64-NEXT:    sub.d $a0, $a0, $a1
+; LA64-NEXT:    andi $a1, $a0, 51
+; LA64-NEXT:    srli.d $a0, $a0, 2
+; LA64-NEXT:    andi $a0, $a0, 51
+; LA64-NEXT:    add.d $a0, $a1, $a0
+; LA64-NEXT:    srli.d $a1, $a0, 4
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    andi $a0, $a0, 15
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %1 = call i8 @llvm.ctpop.i8(i8 %a)
+  ret i8 %1
+}
+
+define i16 @test_ctpop_i16(i16 %a) nounwind {
+; LA32-LABEL: test_ctpop_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 5
+; LA32-NEXT:    ori $a1, $a1, 1365
+; LA32-NEXT:    srli.w $a2, $a0, 1
+; LA32-NEXT:    and $a1, $a2, $a1
+; LA32-NEXT:    sub.w $a0, $a0, $a1
+; LA32-NEXT:    lu12i.w $a1, 3
+; LA32-NEXT:    ori $a1, $a1, 819
+; LA32-NEXT:    and $a2, $a0, $a1
+; LA32-NEXT:    srli.w $a0, $a0, 2
+; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    add.w $a0, $a2, $a0
+; LA32-NEXT:    srli.w $a1, $a0, 4
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    bstrpick.w $a1, $a0, 11, 8
+; LA32-NEXT:    andi $a0, $a0, 15
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctpop_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 5
+; LA64-NEXT:    ori $a1, $a1, 1365
+; LA64-NEXT:    srli.d $a2, $a0, 1
+; LA64-NEXT:    and $a1, $a2, $a1
+; LA64-NEXT:    sub.d $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 3
+; LA64-NEXT:    ori $a1, $a1, 819
+; LA64-NEXT:    and $a2, $a0, $a1
+; LA64-NEXT:    srli.d $a0, $a0, 2
+; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    add.d $a0, $a2, $a0
+; LA64-NEXT:    srli.d $a1, $a0, 4
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a1, $a0, 11, 8
+; LA64-NEXT:    andi $a0, $a0, 15
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %1 = call i16 @llvm.ctpop.i16(i16 %a)
+  ret i16 %1
+}
+
+define i32 @test_ctpop_i32(i32 %a) nounwind {
+; LA32-LABEL: test_ctpop_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 349525
+; LA32-NEXT:    ori $a1, $a1, 1365
+; LA32-NEXT:    srli.w $a2, $a0, 1
+; LA32-NEXT:    and $a1, $a2, $a1
+; LA32-NEXT:    sub.w $a0, $a0, $a1
+; LA32-NEXT:    lu12i.w $a1, 209715
+; LA32-NEXT:    ori $a1, $a1, 819
+; LA32-NEXT:    and $a2, $a0, $a1
+; LA32-NEXT:    srli.w $a0, $a0, 2
+; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    add.w $a0, $a2, $a0
+; LA32-NEXT:    srli.w $a1, $a0, 4
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    lu12i.w $a1, 61680
+; LA32-NEXT:    ori $a1, $a1, 3855
+; LA32-NEXT:    and $a0, $a0, $a1
+; LA32-NEXT:    lu12i.w $a1, 4112
+; LA32-NEXT:    ori $a1, $a1, 257
+; LA32-NEXT:    mul.w $a0, $a0, $a1
+; LA32-NEXT:    srli.w $a0, $a0, 24
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctpop_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 349525
+; LA64-NEXT:    ori $a1, $a1, 1365
+; LA64-NEXT:    srli.d $a2, $a0, 1
+; LA64-NEXT:    and $a1, $a2, $a1
+; LA64-NEXT:    sub.d $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 209715
+; LA64-NEXT:    ori $a1, $a1, 819
+; LA64-NEXT:    and $a2, $a0, $a1
+; LA64-NEXT:    srli.d $a0, $a0, 2
+; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    add.d $a0, $a2, $a0
+; LA64-NEXT:    srli.d $a1, $a0, 4
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 61680
+; LA64-NEXT:    ori $a1, $a1, 3855
+; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 4112
+; LA64-NEXT:    ori $a1, $a1, 257
+; LA64-NEXT:    mul.d $a0, $a0, $a1
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 24
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %1 = call i32 @llvm.ctpop.i32(i32 %a)
+  ret i32 %1
+}
+
+define i64 @test_ctpop_i64(i64 %a) nounwind {
+; LA32-LABEL: test_ctpop_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 349525
+; LA32-NEXT:    ori $a2, $a2, 1365
+; LA32-NEXT:    srli.w $a3, $a0, 1
+; LA32-NEXT:    and $a3, $a3, $a2
+; LA32-NEXT:    sub.w $a0, $a0, $a3
+; LA32-NEXT:    lu12i.w $a3, 209715
+; LA32-NEXT:    ori $a3, $a3, 819
+; LA32-NEXT:    and $a4, $a0, $a3
+; LA32-NEXT:    srli.w $a0, $a0, 2
+; LA32-NEXT:    and $a0, $a0, $a3
+; LA32-NEXT:    add.w $a0, $a4, $a0
+; LA32-NEXT:    srli.w $a4, $a1, 1
+; LA32-NEXT:    and $a2, $a4, $a2
+; LA32-NEXT:    sub.w $a1, $a1, $a2
+; LA32-NEXT:    srli.w $a2, $a0, 4
+; LA32-NEXT:    add.w $a0, $a0, $a2
+; LA32-NEXT:    and $a2, $a1, $a3
+; LA32-NEXT:    srli.w $a1, $a1, 2
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    add.w $a1, $a2, $a1
+; LA32-NEXT:    srli.w $a2, $a1, 4
+; LA32-NEXT:    add.w $a1, $a1, $a2
+; LA32-NEXT:    lu12i.w $a2, 61680
+; LA32-NEXT:    ori $a2, $a2, 3855
+; LA32-NEXT:    and $a1, $a1, $a2
+; LA32-NEXT:    and $a0, $a0, $a2
+; LA32-NEXT:    lu12i.w $a2, 4112
+; LA32-NEXT:    ori $a2, $a2, 257
+; LA32-NEXT:    mul.w $a0, $a0, $a2
+; LA32-NEXT:    mul.w $a1, $a1, $a2
+; LA32-NEXT:    srli.w $a1, $a1, 24
+; LA32-NEXT:    srli.w $a0, $a0, 24
+; LA32-NEXT:    add.w $a0, $a0, $a1
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_ctpop_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 349525
+; LA64-NEXT:    ori $a1, $a1, 1365
+; LA64-NEXT:    lu32i.d $a1, 349525
+; LA64-NEXT:    lu52i.d $a1, $a1, 1365
+; LA64-NEXT:    srli.d $a2, $a0, 1
+; LA64-NEXT:    and $a1, $a2, $a1
+; LA64-NEXT:    sub.d $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 209715
+; LA64-NEXT:    ori $a1, $a1, 819
+; LA64-NEXT:    lu32i.d $a1, 209715
+; LA64-NEXT:    lu52i.d $a1, $a1, 819
+; LA64-NEXT:    and $a2, $a0, $a1
+; LA64-NEXT:    srli.d $a0, $a0, 2
+; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    add.d $a0, $a2, $a0
+; LA64-NEXT:    srli.d $a1, $a0, 4
+; LA64-NEXT:    add.d $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 61680
+; LA64-NEXT:    ori $a1, $a1, 3855
+; LA64-NEXT:    lu32i.d $a1, -61681
+; LA64-NEXT:    lu52i.d $a1, $a1, 240
+; LA64-NEXT:    and $a0, $a0, $a1
+; LA64-NEXT:    lu12i.w $a1, 4112
+; LA64-NEXT:    ori $a1, $a1, 257
+; LA64-NEXT:    lu32i.d $a1, 65793
+; LA64-NEXT:    lu52i.d $a1, $a1, 16
+; LA64-NEXT:    mul.d $a0, $a0, $a1
+; LA64-NEXT:    srli.d $a0, $a0, 56
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %1 = call i64 @llvm.ctpop.i64(i64 %a)
+  ret i64 %1
+}
+
+define i8 @test_cttz_i8(i8 %a) nounwind {
+; LA32-LABEL: test_cttz_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ori $a0, $a0, 256
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_cttz_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ori $a0, $a0, 256
+; LA64-NEXT:    ctz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+  ret i8 %tmp
+}
+
+define i16 @test_cttz_i16(i16 %a) nounwind {
+; LA32-LABEL: test_cttz_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_cttz_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    ctz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+  ret i16 %tmp
+}
+
+define i32 @test_cttz_i32(i32 %a) nounwind {
+; LA32-LABEL: test_cttz_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_cttz_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ctz.w $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+  ret i32 %tmp
+}
+
+define i64 @test_cttz_i64(i64 %a) nounwind {
+; LA32-LABEL: test_cttz_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    sltu $a2, $zero, $a0
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    maskeqz $a0, $a0, $a2
+; LA32-NEXT:    ctz.w $a1, $a1
+; LA32-NEXT:    addi.w $a1, $a1, 32
+; LA32-NEXT:    masknez $a1, $a1, $a2
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_cttz_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ctz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %tmp = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+  ret i64 %tmp
+}
+
+define i8 @test_not_cttz_i8(i8 %a) nounwind {
+; LA32-LABEL: test_not_cttz_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ori $a1, $zero, 256
+; LA32-NEXT:    orn $a0, $a1, $a0
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_cttz_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ori $a1, $zero, 256
+; LA64-NEXT:    orn $a0, $a1, $a0
+; LA64-NEXT:    ctz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i8 %a, -1
+  %tmp = call i8 @llvm.cttz.i8(i8 %neg, i1 false)
+  ret i8 %tmp
+}
+
+define i16 @test_not_cttz_i16(i16 %a) nounwind {
+; LA32-LABEL: test_not_cttz_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 16
+; LA32-NEXT:    orn $a0, $a1, $a0
+; LA32-NEXT:    ctz.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_cttz_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 16
+; LA64-NEXT:    orn $a0, $a1, $a0
+; LA64-NEXT:    ctz.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i16 %a, -1
+  %tmp = call i16 @llvm.cttz.i16(i16 %neg, i1 false)
+  ret i16 %tmp
+}
+
+define i32 @test_not_cttz_i32(i32 %a) nounwind {
+; LA32-LABEL: test_not_cttz_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    cto.w $a0, $a0
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_cttz_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    cto.w $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i32 %a, -1
+  %tmp = call i32 @llvm.cttz.i32(i32 %neg, i1 false)
+  ret i32 %tmp
+}
+
+define i64 @test_not_cttz_i64(i64 %a) nounwind {
+; LA32-LABEL: test_not_cttz_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    nor $a2, $a0, $zero
+; LA32-NEXT:    sltu $a2, $zero, $a2
+; LA32-NEXT:    cto.w $a1, $a1
+; LA32-NEXT:    addi.w $a1, $a1, 32
+; LA32-NEXT:    masknez $a1, $a1, $a2
+; LA32-NEXT:    cto.w $a0, $a0
+; LA32-NEXT:    maskeqz $a0, $a0, $a2
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    jirl $zero, $ra, 0
+;
+; LA64-LABEL: test_not_cttz_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    cto.d $a0, $a0
+; LA64-NEXT:    jirl $zero, $ra, 0
+  %neg = xor i64 %a, -1
+  %tmp = call i64 @llvm.cttz.i64(i64 %neg, i1 false)
+  ret i64 %tmp
+}