diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -863,8 +863,12 @@ case TargetOpcode::G_SUB: case TargetOpcode::G_SADDO: case TargetOpcode::G_SSUBO: + case TargetOpcode::G_SADDE: + case TargetOpcode::G_SSUBE: case TargetOpcode::G_UADDO: case TargetOpcode::G_USUBO: + case TargetOpcode::G_UADDE: + case TargetOpcode::G_USUBE: return narrowScalarAddSub(MI, TypeIdx, NarrowTy); case TargetOpcode::G_MUL: case TargetOpcode::G_UMULH: @@ -4472,21 +4476,25 @@ unsigned OpO, OpE, OpF; switch (Opcode) { case TargetOpcode::G_SADDO: + case TargetOpcode::G_SADDE: case TargetOpcode::G_UADDO: + case TargetOpcode::G_UADDE: case TargetOpcode::G_ADD: OpO = TargetOpcode::G_UADDO; OpE = TargetOpcode::G_UADDE; OpF = TargetOpcode::G_UADDE; - if (Opcode == TargetOpcode::G_SADDO) + if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE) OpF = TargetOpcode::G_SADDE; break; case TargetOpcode::G_SSUBO: + case TargetOpcode::G_SSUBE: case TargetOpcode::G_USUBO: + case TargetOpcode::G_USUBE: case TargetOpcode::G_SUB: OpO = TargetOpcode::G_USUBO; OpE = TargetOpcode::G_USUBE; OpF = TargetOpcode::G_USUBE; - if (Opcode == TargetOpcode::G_SSUBO) + if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE) OpF = TargetOpcode::G_SSUBE; break; default: @@ -4500,12 +4508,14 @@ Register CarryDst; if (NumDefs == 2) CarryDst = MI.getOperand(1).getReg(); + Register CarryIn; + if (MI.getNumOperands() == NumDefs + 3) + CarryIn = MI.getOperand(NumDefs + 2).getReg(); SmallVector Src1Regs, Src2Regs, DstRegs; extractParts(Src1, NarrowTy, NumParts, Src1Regs); extractParts(Src2, NarrowTy, NumParts, Src2Regs); - Register CarryIn; for (int i = 0; i < NumParts; ++i) { Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); @@ -4513,7 +4523,7 @@ if (i == NumParts - 1 && CarryDst) CarryOut = CarryDst; - if (i == 0) { + if (!CarryIn) { MIRBuilder.buildInstr(OpO, {DstReg, CarryOut}, {Src1Regs[i], Src2Regs[i]}); } else if (i == NumParts - 1) { diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -1028,6 +1028,162 @@ EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } +TEST_F(AArch64GISelMITest, NarrowUADDE) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_UADDE).legalFor( + {{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto Op2 = B.buildUndef(S1); + auto UADDE = B.buildUAdde(S96, S1, Op0, Op1, Op2); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*UADDE, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[UADDE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_ + CHECK: [[UADDE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[UADDE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_UADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[UADDE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[UADDE0]]:_(s32), [[UADDE1]]:_(s32), [[UADDE2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +TEST_F(AArch64GISelMITest, NarrowUSUBE) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder(G_USUBE).legalFor( + {{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto Op2 = B.buildUndef(S1); + auto USUBE = B.buildUSube(S96, S1, Op0, Op1, Op2); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*USUBE, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[USUBE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_ + CHECK: [[USUBE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[USUBE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_USUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[USUBE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[USUBE0]]:_(s32), [[USUBE1]]:_(s32), [[USUBE2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +TEST_F(AArch64GISelMITest, NarrowSADDE) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder({G_SADDE, G_UADDE}) + .legalFor({{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto Op2 = B.buildUndef(S1); + auto SADDE = B.buildSAdde(S96, S1, Op0, Op1, Op2); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*SADDE, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[SADDE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_ + CHECK: [[SADDE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[SADDE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[SADDE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SADDE0]]:_(s32), [[SADDE1]]:_(s32), [[SADDE2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + +TEST_F(AArch64GISelMITest, NarrowSSUBE) { + setUp(); + if (!TM) + return; + + LLT S1 = LLT::scalar(1); + LLT S32 = LLT::scalar(32); + LLT S96 = LLT::scalar(96); + DefineLegalizerInfo(A, { + getActionDefinitionsBuilder({G_SSUBE, G_USUBE}) + .legalFor({{LLT::scalar(32), LLT::scalar(1)}}); + }); + + auto Op0 = B.buildUndef(S96); + auto Op1 = B.buildUndef(S96); + auto Op2 = B.buildUndef(S1); + auto SSUBE = B.buildSSube(S96, S1, Op0, Op1, Op2); + + AInfo Info(MF->getSubtarget()); + DummyGISelObserver Observer; + LegalizerHelper Helper(*MF, Info, Observer, B); + EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized, + Helper.narrowScalar(*SSUBE, 0, S32)); + + const char *CheckStr = R"( + CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF + CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF + CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]] + CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]] + CHECK: [[SSUBE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_ + CHECK: [[SSUBE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_ + CHECK: [[SSUBE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SSUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_ + CHECK: [[SSUBE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SSUBE0]]:_(s32), [[SSUBE1]]:_(s32), [[SSUBE2]]:_(s32) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} + TEST_F(AArch64GISelMITest, FewerElementsAnd) { setUp(); if (!TM)