diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -170,8 +170,10 @@ widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); LegalizeResult widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); - LegalizeResult - widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult widenScalarAddoSubo(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy); + LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy); /// Helper function to split a wide generic register into bitwise blocks with /// the given Type (which implies the number of blocks needed). The generic diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1757,6 +1757,34 @@ return Legalized; } +LegalizerHelper::LegalizeResult +LegalizerHelper::widenScalarAddoSubo(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy) { + if (TypeIdx == 1) + return UnableToLegalize; // TODO + unsigned Op = MI.getOpcode(); + unsigned Opcode = Op == TargetOpcode::G_UADDO || Op == TargetOpcode::G_SADDO + ? TargetOpcode::G_ADD + : TargetOpcode::G_SUB; + unsigned ExtOpcode = + Op == TargetOpcode::G_UADDO || Op == TargetOpcode::G_USUBO + ? TargetOpcode::G_ZEXT + : TargetOpcode::G_SEXT; + auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)}); + auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)}); + // Do the arithmetic in the larger type. + auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}); + LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); + auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp); + auto ExtOp = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {TruncOp}); + // There is no overflow if the ExtOp is the same as NewOp. + MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp); + // Now trunc the NewOp to the original result. + MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); + MI.eraseFromParent(); + return Legalized; +} + LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { @@ -1815,48 +1843,10 @@ case TargetOpcode::G_UNMERGE_VALUES: return widenScalarUnmergeValues(MI, TypeIdx, WideTy); case TargetOpcode::G_SADDO: - case TargetOpcode::G_SSUBO: { - if (TypeIdx == 1) - return UnableToLegalize; // TODO - auto LHSExt = MIRBuilder.buildSExt(WideTy, MI.getOperand(2)); - auto RHSExt = MIRBuilder.buildSExt(WideTy, MI.getOperand(3)); - unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SADDO - ? TargetOpcode::G_ADD - : TargetOpcode::G_SUB; - auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}); - LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); - auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp); - auto ExtOp = MIRBuilder.buildSExt(WideTy, TruncOp); - // There is no overflow if the re-extended result is the same as NewOp. - MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp); - // Now trunc the NewOp to the original result. - MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); - MI.eraseFromParent(); - return Legalized; - } + case TargetOpcode::G_SSUBO: case TargetOpcode::G_UADDO: - case TargetOpcode::G_USUBO: { - if (TypeIdx == 1) - return UnableToLegalize; // TODO - auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2)); - auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3)); - unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO - ? TargetOpcode::G_ADD - : TargetOpcode::G_SUB; - // Do the arithmetic in the larger type. - auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); - LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); - APInt Mask = - APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits()); - auto AndOp = MIRBuilder.buildAnd( - WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask)); - // There is no overflow if the AndOp is the same as NewOp. - MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp); - // Now trunc the NewOp to the original result. - MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); - MI.eraseFromParent(); - return Legalized; - } + case TargetOpcode::G_USUBO: + return widenScalarAddoSubo(MI, TypeIdx, WideTy); case TargetOpcode::G_SADDSAT: case TargetOpcode::G_SSUBSAT: case TargetOpcode::G_SSHLSAT: diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-add.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-add.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-add.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-add.mir @@ -87,7 +87,8 @@ ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]] ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]] ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32) ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[ICMP]](s32) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sub.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sub.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sub.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sub.mir @@ -107,7 +107,8 @@ ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]] ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]] ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32) ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[ICMP]](s32) diff --git a/llvm/test/CodeGen/AArch64/legalize-uaddo.mir b/llvm/test/CodeGen/AArch64/legalize-uaddo.mir --- a/llvm/test/CodeGen/AArch64/legalize-uaddo.mir +++ b/llvm/test/CodeGen/AArch64/legalize-uaddo.mir @@ -18,8 +18,11 @@ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]] ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2 ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[C1]] - ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]] + ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[ADD]](s32) + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]] ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[ADD]](s32), [[AND1]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s16) = COPY [[TRUNC1]](s16) ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ICMP]](s32) ; CHECK: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]] diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir @@ -37,10 +37,11 @@ ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]] ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]] - ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) - ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] + ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]] ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1) ; CHECK: $vgpr0 = COPY [[AND3]](s32) ; CHECK: $vgpr1 = COPY [[ZEXT]](s32) @@ -71,10 +72,11 @@ ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]] ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]] - ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) - ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]] + ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD]](s32) + ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]] ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1) ; CHECK: $vgpr0 = COPY [[AND3]](s32) ; CHECK: $vgpr1 = COPY [[ZEXT]](s32) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir @@ -37,10 +37,11 @@ ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]] ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C]] - ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]] ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) - ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]] + ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) + ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]] ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1) ; CHECK: $vgpr0 = COPY [[AND3]](s32) ; CHECK: $vgpr1 = COPY [[ZEXT]](s32) @@ -71,10 +72,11 @@ ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32) ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]] ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]] - ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C]] - ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]] ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) - ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]] + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]] + ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) + ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]] ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1) ; CHECK: $vgpr0 = COPY [[AND3]](s32) ; CHECK: $vgpr1 = COPY [[ZEXT]](s32) diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -589,9 +589,9 @@ CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]] CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]] CHECK: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[LHS]]:_, [[RHS]]:_ - CHECK: [[CST:%[0-9]+]]:_(s16) = G_CONSTANT i16 255 - CHECK: [[AND:%[0-9]+]]:_(s16) = G_AND [[ADD]]:_, [[CST]]:_ - CHECK: G_ICMP intpred(ne), [[ADD]]:_(s16), [[AND]]:_ + CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[ADD]] + CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]] + CHECK: G_ICMP intpred(ne), [[ADD]]:_(s16), [[ZEXT]]:_ CHECK: G_TRUNC [[ADD]] )"; @@ -628,9 +628,87 @@ CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]] CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]] CHECK: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[LHS]]:_, [[RHS]]:_ - CHECK: [[CST:%[0-9]+]]:_(s16) = G_CONSTANT i16 255 - CHECK: [[AND:%[0-9]+]]:_(s16) = G_AND [[SUB]]:_, [[CST]]:_ - CHECK: G_ICMP intpred(ne), [[SUB]]:_(s16), [[AND]]:_ + CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SUB]] + CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]] + CHECK: G_ICMP intpred(ne), [[SUB]]:_(s16), [[ZEXT]]:_ + CHECK: G_TRUNC [[SUB]] + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// SADDO widening. +TEST_F(AArch64GISelMITest, WidenSADDO) { + setUp(); + if (!TM) + return; + + // Declare your legalization info + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_ADD).legalFor({{s16, s16}}); + }); + // Build + // Trunc it to s8. + LLT s8{LLT::scalar(8)}; + LLT s16{LLT::scalar(16)}; + auto MIBTrunc = B.buildTrunc(s8, Copies[0]); + unsigned CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1)); + auto MIBSAddO = + B.buildInstr(TargetOpcode::G_SADDO, {s8, CarryReg}, {MIBTrunc, MIBTrunc}); + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_TRUE(Helper.widenScalar(*MIBSAddO, 0, s16) == + LegalizerHelper::LegalizeResult::Legalized); + + auto CheckStr = R"( + CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC + CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]] + CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]] + CHECK: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[LHS]]:_, [[RHS]]:_ + CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[ADD]] + CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]] + CHECK: G_ICMP intpred(ne), [[ADD]]:_(s16), [[SEXT]]:_ + CHECK: G_TRUNC [[ADD]] + )"; + + // Check + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +// SSUBO widening. +TEST_F(AArch64GISelMITest, WidenSSUBO) { + setUp(); + if (!TM) + return; + + // Declare your legalization info + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_SUB).legalFor({{s16, s16}}); + }); + // Build + // Trunc it to s8. + LLT s8{LLT::scalar(8)}; + LLT s16{LLT::scalar(16)}; + auto MIBTrunc = B.buildTrunc(s8, Copies[0]); + unsigned CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1)); + auto MIBSSUBO = + B.buildInstr(TargetOpcode::G_SSUBO, {s8, CarryReg}, {MIBTrunc, MIBTrunc}); + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_TRUE(Helper.widenScalar(*MIBSSUBO, 0, s16) == + LegalizerHelper::LegalizeResult::Legalized); + + auto CheckStr = R"( + CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC + CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]] + CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]] + CHECK: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[LHS]]:_, [[RHS]]:_ + CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SUB]] + CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]] + CHECK: G_ICMP intpred(ne), [[SUB]]:_(s16), [[SEXT]]:_ CHECK: G_TRUNC [[SUB]] )";