diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -825,6 +825,45 @@ return Legalized; } + case TargetOpcode::G_FREEZE: { + const Register DstReg = MI.getOperand(0).getReg(); + const Register SrcReg = MI.getOperand(1).getReg(); + const LLT DstTy = MRI.getType(DstReg); + const LLT SrcTy = MRI.getType(SrcReg); + + // FIXME: support narrowing to smaller vector element type + if (DstTy.isVector() && NarrowTy.isVector()) { + LLVM_DEBUG( + dbgs() + << "Cannot narrow G_FREEZE to smaller vector element type yet."); + return UnableToLegalize; + } + + LLT LeftoverTy; + SmallVector SrcRegs, LeftoverRegs, DstRegs, LeftoverDstRegs; + if (!extractParts(SrcReg, SrcTy, NarrowTy, LeftoverTy, SrcRegs, + LeftoverRegs)) + return UnableToLegalize; + + for (Register PartReg : SrcRegs) { + const Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildInstr(TargetOpcode::G_FREEZE, {PartDstReg}, {PartReg}); + DstRegs.emplace_back(PartDstReg); + } + + for (Register LeftoverReg : LeftoverRegs) { + const Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy); + MIRBuilder.buildInstr(TargetOpcode::G_FREEZE, {PartDstReg}, + {LeftoverReg}); + LeftoverDstRegs.emplace_back(PartDstReg); + } + + insertParts(DstReg, DstTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs); + + MI.eraseFromParent(); + return Legalized; + } + case TargetOpcode::G_ADD: { // FIXME: add support for when SizeOp0 isn't an exact multiple of // NarrowSize. @@ -1728,6 +1767,13 @@ Observer.changedInstr(MI); return Legalized; } + case TargetOpcode::G_FREEZE: + Observer.changingInstr(MI); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + widenScalarDst(MI, WideTy); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_ADD: case TargetOpcode::G_AND: case TargetOpcode::G_MUL: @@ -3293,6 +3339,7 @@ case G_FMAXIMUM: case G_FSHL: case G_FSHR: + case G_FREEZE: return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy); case G_SHL: case G_LSHR: @@ -3606,6 +3653,7 @@ Observer.changedInstr(MI); return Legalized; case TargetOpcode::G_INSERT: + case TargetOpcode::G_FREEZE: if (TypeIdx != 0) return UnableToLegalize; Observer.changingInstr(MI); diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -2804,4 +2804,204 @@ EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } +// Test widening of G_FREEZE +TEST_F(AArch64GISelMITest, WidenFreeze) { + setUp(); + if (!TM) + return; + + DefineLegalizerInfo(A, {}); + + // Make sure that G_FREEZE is widened with anyext + LLT S64{LLT::scalar(64)}; + LLT S128{LLT::scalar(128)}; + LLT V2S32{LLT::vector(2, 32)}; + LLT V2S64{LLT::vector(2, 64)}; + + auto Vector = B.buildBitcast(V2S32, Copies[0]); + + auto FreezeScalar = B.buildInstr(TargetOpcode::G_FREEZE, {S64}, {Copies[0]}); + auto FreezeVector = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector}); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.widenScalar(*FreezeScalar, 0, S128)); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.widenScalar(*FreezeVector, 0, V2S64)); + + const auto *CheckStr = R"( + CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY + CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]] + + CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[COPY]] + CHECK: [[FREEZE:%[0-9]+]]:_(s128) = G_FREEZE [[ANYEXT]] + CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[FREEZE]] + + CHECK: [[ANYEXT1:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[BITCAST]] + CHECK: [[FREEZE1:%[0-9]+]]:_(<2 x s64>) = G_FREEZE [[ANYEXT1]] + CHECK: [[TRUNC1:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[FREEZE1]] + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// Test narrowing of G_FREEZE +TEST_F(AArch64GISelMITest, NarrowFreeze) { + setUp(); + if (!TM) + return; + + DefineLegalizerInfo(A, {}); + + // Make sure that G_FREEZE is narrowed using unmerge/extract + LLT S16{LLT::scalar(16)}; + LLT S32{LLT::scalar(32)}; + LLT S33{LLT::scalar(33)}; + LLT S64{LLT::scalar(64)}; + LLT V2S16{LLT::vector(2, 16)}; + LLT V2S32{LLT::vector(2, 32)}; + + auto Trunc = B.buildTrunc(S33, {Copies[0]}); + auto Vector = B.buildBitcast(V2S32, Copies[0]); + + auto FreezeScalar = B.buildInstr(TargetOpcode::G_FREEZE, {S64}, {Copies[0]}); + auto FreezeOdd = B.buildInstr(TargetOpcode::G_FREEZE, {S33}, {Trunc}); + auto FreezeVector = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector}); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*FreezeScalar, 0, S32)); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*FreezeOdd, 0, S32)); + EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize, + Helper.narrowScalar(*FreezeVector, 0, V2S16)); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*FreezeVector, 0, S16)); + + const auto *CheckStr = R"( + CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY + CHECK: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]] + CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]] + + CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]] + CHECK: [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[UV]] + CHECK: [[FREEZE1:%[0-9]+]]:_(s32) = G_FREEZE [[UV1]] + CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FREEZE]]:_(s32), [[FREEZE1]] + + CHECK: [[EXTR:%[0-9]+]]:_(s32) = G_EXTRACT [[TRUNC]]:_(s33), 0 + CHECK: [[EXTR1:%[0-9]+]]:_(s1) = G_EXTRACT [[TRUNC]]:_(s33), 32 + CHECK: [[FREEZE2:%[0-9]+]]:_(s32) = G_FREEZE [[EXTR]] + CHECK: [[FREEZE3:%[0-9]+]]:_(s1) = G_FREEZE [[EXTR1]] + CHECK: [[UNDEF:%[0-9]+]]:_(s33) = G_IMPLICIT_DEF + CHECK: [[INSERT:%[0-9]+]]:_(s33) = G_INSERT [[UNDEF]]:_, [[FREEZE2]]:_(s32), 0 + CHECK: [[INSERT1:%[0-9]+]]:_(s33) = G_INSERT [[INSERT]]:_, [[FREEZE3]]:_(s1), 32 + + CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[BITCAST]] + CHECK: [[FREEZE4:%[0-9]+]]:_(s16) = G_FREEZE [[UV]] + CHECK: [[FREEZE5:%[0-9]+]]:_(s16) = G_FREEZE [[UV1]] + CHECK: [[FREEZE6:%[0-9]+]]:_(s16) = G_FREEZE [[UV2]] + CHECK: [[FREEZE7:%[0-9]+]]:_(s16) = G_FREEZE [[UV3]] + CHECK: [[BV:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FREEZE4]]:_(s16), [[FREEZE5]]:_(s16), [[FREEZE6]]:_(s16), [[FREEZE7]] + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// Test fewer elements of G_FREEZE +TEST_F(AArch64GISelMITest, FewerElementsFreeze) { + setUp(); + if (!TM) + return; + + DefineLegalizerInfo(A, {}); + + // Make sure that G_FREEZE is narrowed using unmerge/extract + LLT S32{LLT::scalar(32)}; + LLT V2S16{LLT::vector(2, 16)}; + LLT V2S32{LLT::vector(2, 32)}; + LLT V4S16{LLT::vector(4, 16)}; + + auto Vector1 = B.buildBitcast(V2S32, Copies[0]); + auto Vector2 = B.buildBitcast(V4S16, Copies[0]); + + auto FreezeVector1 = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector1}); + auto FreezeVector2 = B.buildInstr(TargetOpcode::G_FREEZE, {V4S16}, {Vector2}); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.fewerElementsVector(*FreezeVector1, 0, S32)); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.fewerElementsVector(*FreezeVector2, 0, V2S16)); + + const auto *CheckStr = R"( + CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY + CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]] + CHECK: [[BITCAST1:%[0-9]+]]:_(<4 x s16>) = G_BITCAST [[COPY]] + + CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]] + CHECK: [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[UV]] + CHECK: [[FREEZE1:%[0-9]+]]:_(s32) = G_FREEZE [[UV1]] + CHECK: [[MV:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FREEZE]]:_(s32), [[FREEZE1]] + + CHECK: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BITCAST1]] + CHECK: [[FREEZE2:%[0-9]+]]:_(<2 x s16>) = G_FREEZE [[UV]] + CHECK: [[FREEZE3:%[0-9]+]]:_(<2 x s16>) = G_FREEZE [[UV1]] + CHECK: [[MV:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FREEZE2]]:_(<2 x s16>), [[FREEZE3]] + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// Test more elements of G_FREEZE +TEST_F(AArch64GISelMITest, MoreElementsFreeze) { + setUp(); + if (!TM) + return; + + DefineLegalizerInfo(A, {}); + + // Make sure that G_FREEZE is narrowed using unmerge/extract + LLT V2S32{LLT::vector(2, 32)}; + LLT V4S32{LLT::vector(4, 32)}; + + auto Vector1 = B.buildBitcast(V2S32, Copies[0]); + auto FreezeVector1 = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector1}); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + + // Perform Legalization + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.moreElementsVector(*FreezeVector1, 0, V4S32)); + + const auto *CheckStr = R"( + CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY + CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]] + + CHECK: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF + CHECK: [[CV:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[BITCAST]]:_(<2 x s32>), [[UNDEF]] + CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[CV]] + CHECK: [[EXTR:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[FREEZE]]:_(<4 x s32>), 0 + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + } // namespace