diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -861,6 +861,8 @@ return reduceOperationWidth(MI, TypeIdx, NarrowTy); case TargetOpcode::G_ADD: case TargetOpcode::G_SUB: + case TargetOpcode::G_SADDO: + case TargetOpcode::G_SSUBO: case TargetOpcode::G_UADDO: case TargetOpcode::G_USUBO: return narrowScalarAddSub(MI, TypeIdx, NarrowTy); @@ -4466,17 +4468,26 @@ // Expand in terms of carry-setting/consuming G_E instructions. int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); - unsigned OpO, OpE; - switch (MI.getOpcode()) { + unsigned Opcode = MI.getOpcode(); + unsigned OpO, OpE, OpF; + switch (Opcode) { + case TargetOpcode::G_SADDO: case TargetOpcode::G_UADDO: case TargetOpcode::G_ADD: OpO = TargetOpcode::G_UADDO; OpE = TargetOpcode::G_UADDE; + OpF = TargetOpcode::G_UADDE; + if (Opcode == TargetOpcode::G_SADDO) + OpF = TargetOpcode::G_SADDE; break; + case TargetOpcode::G_SSUBO: case TargetOpcode::G_USUBO: case TargetOpcode::G_SUB: OpO = TargetOpcode::G_USUBO; OpE = TargetOpcode::G_USUBE; + OpF = TargetOpcode::G_USUBE; + if (Opcode == TargetOpcode::G_SSUBO) + OpF = TargetOpcode::G_SSUBE; break; default: llvm_unreachable("Unexpected add/sub opcode!"); @@ -4502,10 +4513,13 @@ if (i == NumParts - 1 && CarryDst) CarryOut = CarryDst; - if (i == 0) + if (i == 0) { MIRBuilder.buildInstr(OpO, {DstReg, CarryOut}, {Src1Regs[i], Src2Regs[i]}); - else { + } else if (i == NumParts - 1) { + MIRBuilder.buildInstr(OpF, {DstReg, CarryOut}, + {Src1Regs[i], Src2Regs[i], CarryIn}); + } else { MIRBuilder.buildInstr(OpE, {DstReg, CarryOut}, {Src1Regs[i], Src2Regs[i], CarryIn}); } diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -954,6 +954,80 @@ EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } +TEST_F(AArch64GISelMITest, NarrowSADDO) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder({G_UADDO, G_UADDE, G_SADDE}) + .legalFor({{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto SADDO = B.buildSAddo(S96, S1, Op0, Op1); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*SADDO, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[SADDO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDO [[OP0_0]]:_, [[OP1_0]]:_ + CHECK: [[SADDO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[SADDO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[SADDO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SADDO0]]:_(s32), [[SADDO1]]:_(s32), [[SADDO2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +TEST_F(AArch64GISelMITest, NarrowSSUBO) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder({G_USUBO, G_USUBE, G_SSUBE}) + .legalFor({{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto SSUBO = B.buildSSubo(S96, S1, Op0, Op1); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*SSUBO, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[SSUBO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBO [[OP0_0]]:_, [[OP1_0]]:_ + CHECK: [[SSUBO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[SSUBO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SSUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[SSUBO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SSUBO0]]:_(s32), [[SSUBO1]]:_(s32), [[SSUBO2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + TEST_F(AArch64GISelMITest, FewerElementsAnd) { setUp(); if (!TM)