Index: llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3978,23 +3978,24 @@ if (TypeIdx != 1) return UnableToLegalize; - LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(SrcReg); unsigned NarrowSize = NarrowTy.getSizeInBits(); if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { MachineIRBuilder &B = MIRBuilder; - auto UnmergeSrc = B.buildUnmerge(NarrowTy, MI.getOperand(1)); + auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi) auto C_0 = B.buildConstant(NarrowTy, 0); auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), UnmergeSrc.getReg(1), C_0); - auto LoCTLZ = B.buildCTLZ(NarrowTy, UnmergeSrc.getReg(0)); + auto LoCTLZ = B.buildCTLZ(DstTy, UnmergeSrc.getReg(0)); auto C_NarrowSize = B.buildConstant(NarrowTy, NarrowSize); - auto HiIsZeroCTLZ = B.buildAdd(NarrowTy, LoCTLZ, C_NarrowSize); - auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(NarrowTy, UnmergeSrc.getReg(1)); - auto LoOut = B.buildSelect(NarrowTy, HiIsZero, HiIsZeroCTLZ, HiCTLZ); - - B.buildMerge(MI.getOperand(0), {LoOut.getReg(0), C_0.getReg(0)}); + auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize); + auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)); + B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ); MI.eraseFromParent(); return Legalized; @@ -4009,24 +4010,24 @@ if (TypeIdx != 1) return UnableToLegalize; - LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(SrcReg); unsigned NarrowSize = NarrowTy.getSizeInBits(); if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { MachineIRBuilder &B = MIRBuilder; - auto UnmergeSrc = B.buildUnmerge(NarrowTy, MI.getOperand(1)); + auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo) auto C_0 = B.buildConstant(NarrowTy, 0); auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), UnmergeSrc.getReg(0), C_0); - auto HiCTTZ = B.buildCTTZ(NarrowTy, UnmergeSrc.getReg(1)); - auto C_NarrowSize = B.buildConstant(NarrowTy, NarrowSize); - auto LoIsZeroCTTZ = B.buildAdd(NarrowTy, HiCTTZ, C_NarrowSize); - auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(NarrowTy, UnmergeSrc.getReg(0)); - auto LoOut = B.buildSelect(NarrowTy, LoIsZero, LoIsZeroCTTZ, LoCTTZ); - - B.buildMerge(MI.getOperand(0), {LoOut.getReg(0), C_0.getReg(0)}); - + auto HiCTTZ = B.buildCTTZ(DstTy, UnmergeSrc.getReg(1)); + auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); + auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize); + auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)); + B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ); MI.eraseFromParent(); return Legalized; } Index: llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp =================================================================== --- llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -90,6 +90,76 @@ EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } +// CTLZ scalar narrowing +TEST_F(GISelMITest, NarrowScalarCTLZ) { + setUp(); + if (!TM) + return; + + // Declare your legalization info + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_CTLZ).legalFor({{s32, s64}}); + }); + // Build Instr + auto CTLZ = + B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(32)}, {Copies[0]}); + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*CTLZ, 1, LLT::scalar(32))); + + auto CheckStr = R"( + CHECK: [[UNMERGE_LO:%[0-9]+]]:_(s32), [[UNMERGE_HI:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %0:_(s64) + CHECK: [[ZERO:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UNMERGE_HI]]:_(s32), [[ZERO]]:_ + CHECK: [[CTLZ_LO:%[0-9]+]]:_(s32) = G_CTLZ [[UNMERGE_LO]]:_(s32) + CHECK: [[THIRTYTWO:%[0-9]+]]:_(s32) = G_CONSTANT i32 32 + CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTLZ_LO]]:_, [[THIRTYTWO]]:_ + CHECK: [[CTLZ_HI:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[UNMERGE_HI]]:_(s32) + CHECK: %{{[0-9]+}}:_(s32) = G_SELECT [[CMP]]:_(s1), [[ADD]]:_, [[CTLZ_HI]]:_ + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// CTTZ scalar narrowing +TEST_F(GISelMITest, NarrowScalarCTTZ) { + setUp(); + if (!TM) + return; + + // Declare your legalization info + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_CTTZ).legalFor({{s32, s64}}); + }); + // Build Instr + auto CTTZ = + B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(32)}, {Copies[0]}); + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*CTTZ, 1, LLT::scalar(32))); + + auto CheckStr = R"( + CHECK: [[UNMERGE_LO:%[0-9]+]]:_(s32), [[UNMERGE_HI:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %0:_(s64) + CHECK: [[ZERO:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UNMERGE_LO]]:_(s32), [[ZERO]]:_ + CHECK: [[CTTZ_HI:%[0-9]+]]:_(s32) = G_CTTZ [[UNMERGE_HI]]:_(s32) + CHECK: [[THIRTYTWO:%[0-9]+]]:_(s32) = G_CONSTANT i32 32 + CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTTZ_HI]]:_, [[THIRTYTWO]]:_ + CHECK: [[CTTZ_LO:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[UNMERGE_LO]]:_(s32) + CHECK: %{{[0-9]+}}:_(s32) = G_SELECT [[CMP]]:_(s1), [[ADD]]:_, [[CTTZ_LO]]:_ + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + // CTTZ expansion in terms of CTPOP TEST_F(GISelMITest, LowerBitCountingCTTZ2) { setUp();