Index: include/llvm/Support/TargetOpcodes.def =================================================================== --- include/llvm/Support/TargetOpcodes.def +++ include/llvm/Support/TargetOpcodes.def @@ -464,6 +464,21 @@ /// Generic shufflevector. HANDLE_TARGET_OPCODE(G_SHUFFLE_VECTOR) +/// Generic count trailing zeroes. +HANDLE_TARGET_OPCODE(G_CTTZ) + +/// Same as above, undefined for zero inputs. +HANDLE_TARGET_OPCODE(G_CTTZ_ZERO_UNDEF) + +/// Generic count leading zeroes. +HANDLE_TARGET_OPCODE(G_CTLZ) + +/// Same as above, undefined for zero inputs. +HANDLE_TARGET_OPCODE(G_CTLZ_ZERO_UNDEF) + +/// Generic count bits. +HANDLE_TARGET_OPCODE(G_CTPOP) + /// Generic byte swap. HANDLE_TARGET_OPCODE(G_BSWAP) Index: include/llvm/Target/GenericOpcodes.td =================================================================== --- include/llvm/Target/GenericOpcodes.td +++ include/llvm/Target/GenericOpcodes.td @@ -120,6 +120,36 @@ let mayStore = 1; } +def G_CTLZ : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + +def G_CTLZ_ZERO_UNDEF : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + +def G_CTTZ : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + +def G_CTTZ_ZERO_UNDEF : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + +def G_CTPOP : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + def G_BSWAP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src); Index: lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- lib/CodeGen/GlobalISel/IRTranslator.cpp +++ lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -892,6 +892,26 @@ PtrTy.getSizeInBits() / 8, 8)); return true; } + case Intrinsic::cttz: + case Intrinsic::ctlz: { + ConstantInt *Cst = cast(CI.getArgOperand(1)); + bool isTrailing = ID == Intrinsic::cttz; + unsigned Opcode = isTrailing + ? Cst->isZero() ? TargetOpcode::G_CTTZ + : TargetOpcode::G_CTTZ_ZERO_UNDEF + : Cst->isZero() ? TargetOpcode::G_CTLZ + : TargetOpcode::G_CTLZ_ZERO_UNDEF; + MIRBuilder.buildInstr(Opcode) + .addDef(getOrCreateVReg(CI)) + .addUse(getOrCreateVReg(*CI.getArgOperand(0))); + return true; + } + case Intrinsic::ctpop: { + MIRBuilder.buildInstr(TargetOpcode::G_CTPOP) + .addDef(getOrCreateVReg(CI)) + .addUse(getOrCreateVReg(*CI.getArgOperand(0))); + return true; + } } return false; } Index: test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll =================================================================== --- test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1408,6 +1408,36 @@ ret float %res } +declare i32 @llvm.ctlz.i32(i32, i1) +define i32 @test_ctlz_intrinsic_zero_not_undef(i32 %a) { +; CHECK-LABEL: name: test_ctlz_intrinsic_zero_not_undef +; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0 +; CHECK: [[RES:%[0-9]+]]:_(s32) = G_CTLZ [[A]] +; CHECK: $w0 = COPY [[RES]] + %res = call i32 @llvm.ctlz.i32(i32 %a, i1 0) + ret i32 %res +} + +declare i32 @llvm.cttz.i32(i32, i1) +define i32 @test_cttz_intrinsic_zero_undef(i32 %a) { +; CHECK-LABEL: name: test_cttz_intrinsic_zero_undef +; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0 +; CHECK: [[RES:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[A]] +; CHECK: $w0 = COPY [[RES]] + %res = call i32 @llvm.cttz.i32(i32 %a, i1 1) + ret i32 %res +} + +declare i32 @llvm.ctpop.i32(i32) +define i32 @test_ctpop_intrinsic(i32 %a) { +; CHECK-LABEL: name: test_ctpop +; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0 +; CHECK: [[RES:%[0-9]+]]:_(s32) = G_CTPOP [[A]] +; CHECK: $w0 = COPY [[RES]] + %res = call i32 @llvm.ctpop.i32(i32 %a) + ret i32 %res +} + declare void @llvm.lifetime.start.p0i8(i64, i8*) declare void @llvm.lifetime.end.p0i8(i64, i8*) define void @test_lifetime_intrin() { Index: test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir =================================================================== --- test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -282,6 +282,21 @@ # DEBUG-NEXT: G_SHUFFLE_VECTOR (opcode {{[0-9]+}}): 3 type indices # DEBUG: .. type index coverage check SKIPPED: no rules defined # +# DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 1 type index +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# +# DEBUG-NEXT: G_CTTZ_ZERO_UNDEF (opcode {{[0-9]+}}): 1 type index +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# +# DEBUG-NEXT: G_CTLZ (opcode {{[0-9]+}}): 1 type index +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# +# DEBUG-NEXT: G_CTLZ_ZERO_UNDEF (opcode {{[0-9]+}}): 1 type index +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# +# DEBUG-NEXT: G_CTPOP (opcode {{[0-9]+}}): 1 type index +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# # DEBUG-NEXT: G_BSWAP (opcode {{[0-9]+}}): 1 type index # DEBUG: .. the first uncovered type index: 1, OK Index: test/TableGen/trydecode-emission.td =================================================================== --- test/TableGen/trydecode-emission.td +++ test/TableGen/trydecode-emission.td @@ -34,10 +34,10 @@ } // CHECK: /* 0 */ MCD::OPC_ExtractField, 4, 4, // Inst{7-4} ... -// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 14, 0, // Skip to: 21 -// CHECK-NEXT: /* 7 */ MCD::OPC_CheckField, 2, 2, 0, 5, 0, // Skip to: 18 -// CHECK-NEXT: /* 13 */ MCD::OPC_TryDecode, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 18 -// CHECK-NEXT: /* 18 */ MCD::OPC_Decode, {{[0-9]+}}, 1, // Opcode: InstA -// CHECK-NEXT: /* 21 */ MCD::OPC_Fail, +// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23 +// CHECK-NEXT: /* 7 */ MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19 +// CHECK-NEXT: /* 13 */ MCD::OPC_TryDecode, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstB, skip to: 19 +// CHECK-NEXT: /* 19 */ MCD::OPC_Decode, {{[0-9]+}}, 1, 1, // Opcode: InstA +// CHECK-NEXT: /* 23 */ MCD::OPC_Fail, // CHECK: if (DecodeInstB(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; } Index: test/TableGen/trydecode-emission2.td =================================================================== --- test/TableGen/trydecode-emission2.td +++ test/TableGen/trydecode-emission2.td @@ -31,14 +31,14 @@ } // CHECK: /* 0 */ MCD::OPC_ExtractField, 2, 1, // Inst{2} ... -// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 29, 0, // Skip to: 36 +// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38 // CHECK-NEXT: /* 7 */ MCD::OPC_ExtractField, 5, 3, // Inst{7-5} ... -// CHECK-NEXT: /* 10 */ MCD::OPC_FilterValue, 0, 22, 0, // Skip to: 36 -// CHECK-NEXT: /* 14 */ MCD::OPC_CheckField, 0, 2, 3, 5, 0, // Skip to: 25 -// CHECK-NEXT: /* 20 */ MCD::OPC_TryDecode, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 25 -// CHECK-NEXT: /* 25 */ MCD::OPC_CheckField, 3, 2, 0, 5, 0, // Skip to: 36 -// CHECK-NEXT: /* 31 */ MCD::OPC_TryDecode, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 36 -// CHECK-NEXT: /* 36 */ MCD::OPC_Fail, +// CHECK-NEXT: /* 10 */ MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38 +// CHECK-NEXT: /* 14 */ MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26 +// CHECK-NEXT: /* 20 */ MCD::OPC_TryDecode, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstB, skip to: 26 +// CHECK-NEXT: /* 26 */ MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38 +// CHECK-NEXT: /* 32 */ MCD::OPC_TryDecode, {{[0-9]+}}, 1, 1, 0, 0, // Opcode: InstA, skip to: 38 +// CHECK-NEXT: /* 38 */ MCD::OPC_Fail, // CHECK: if (DecodeInstB(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; } // CHECK: if (DecodeInstA(MI, insn, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; } Index: test/TableGen/trydecode-emission3.td =================================================================== --- test/TableGen/trydecode-emission3.td +++ test/TableGen/trydecode-emission3.td @@ -35,10 +35,10 @@ } // CHECK: /* 0 */ MCD::OPC_ExtractField, 4, 4, // Inst{7-4} ... -// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 14, 0, // Skip to: 21 -// CHECK-NEXT: /* 7 */ MCD::OPC_CheckField, 2, 2, 0, 5, 0, // Skip to: 18 -// CHECK-NEXT: /* 13 */ MCD::OPC_TryDecode, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 18 -// CHECK-NEXT: /* 18 */ MCD::OPC_Decode, {{[0-9]+}}, 1, // Opcode: InstA -// CHECK-NEXT: /* 21 */ MCD::OPC_Fail, +// CHECK-NEXT: /* 3 */ MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23 +// CHECK-NEXT: /* 7 */ MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19 +// CHECK-NEXT: /* 13 */ MCD::OPC_TryDecode, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstB, skip to: 19 +// CHECK-NEXT: /* 19 */ MCD::OPC_Decode, {{[0-9]+}}, 1, 1, // Opcode: InstA +// CHECK-NEXT: /* 23 */ MCD::OPC_Fail, // CHECK: if (DecodeInstBOp(MI, tmp, Address, Decoder) == MCDisassembler::Fail) { DecodeComplete = false; return MCDisassembler::Fail; }