[AArch64] Fix over-eager fusing of NEON SIMD MUL/ADD

[sanwou01]

Summary

The ISel pattern for SIMD MLA is a bit too eager: it replaces the ADD with an
MLA even when the MUL cannot be eliminated, e.g. when it has another use. An
MLA is usually has a higher latency than an ADD (and there are fewer pipes
available that can execute it), so trading an MLA for an ADD is not great.

ISel is not taking the number of uses of the MUL result into account, nor any
other factors such as the length of the critical path or other resource pressure.

The MachineCombiner is able to make these judgments so this patch ports the ISel
pattern for MUL/ADD fusing to the MachineCombiner.

Similarly for MUL/SUB -> MLS, as well as the indexed variants.

The change has no impact on SPEC CPU© intrate nor fprate.

[sanwou01]

Added performance evaluation results

[dmgreen]

This is because a mla is usually more expensive than a add? Seems sensible to me.

I don't know this code very well though.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

3796–3797

You might want to keep these on separate lines, with the comment making them quite long.

[sanwou01]

Clarified commit message; fixed long lines.

[dmgreen]

Is the plan to add the indexed variants too? To treat them in the same way.

llvm/include/llvm/CodeGen/MachineCombinerPattern.h

42

Do these need V's on the front? The type is already in the name.

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

4141

Is ReplacedAddend ever not nullptr?

[sanwou01]

Addressed @dmgreen's comments.

[sanwou01]

Is the plan to add the indexed variants too? To treat them in the same way.

Good shout. I've given the indexed variants the same treatment.

[dmgreen]

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

4070

Can you explain what this means? Are these not already SIMD MULADDs?

[sanwou01]

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

4070

Ah, sorry, that TODO was there to remind me to add them :-)

[sanwou01]

Remove redundant TODO

[dmgreen]

Thanks! LGTM

[mstorsjo]

This broke vectorized code in one case. To repro, fetch https://martin.st/temp/g723.c and compile with clang -target aarch64-w64-mingw32 -c -O3 g723.c. The diff in generated code is available at https://martin.st/temp/g723.s.diff.

The key change is this:

-       mul     v4.2s, v3.2s, v4.2s
-       mls     v4.2s, v5.2s, v2.2s
+       mul     v5.2s, v5.2s, v2.2s
+       mls     v5.2s, v3.2s, v4.2s

Thus it seems the sign is flipped; it used to calculate v3*v4-v5*v2, now it calculates v5*v2-v3*v4.

[sanwou01]

In D70673#1770293, @mstorsjo wrote:

This broke vectorized code in one case. To repro, fetch https://martin.st/temp/g723.c and compile with clang -target aarch64-w64-mingw32 -c -O3 g723.c. The diff in generated code is available at https://martin.st/temp/g723.s.diff.

The key change is this:

-       mul     v4.2s, v3.2s, v4.2s
-       mls     v4.2s, v5.2s, v2.2s
+       mul     v5.2s, v5.2s, v2.2s
+       mls     v5.2s, v3.2s, v4.2s

Thus it seems the sign is flipped; it used to calculate v3*v4-v5*v2, now it calculates v5*v2-v3*v4.

Ah, I messed up MUL + SUB fusing when the MUL is the first operand of the SUB. Looks like those cases weren't being tested either, ah well! A patch is on the way.

llvm/include/llvm/CodeGen/MachineCombinerPattern.h

enum class MachineCombinerPattern {
   MULADDWI_OP1,
   MULSUBWI_OP1,
   MULADDX_OP1,
   MULADDX_OP2,
   MULSUBX_OP1,
   MULSUBX_OP2,
   MULADDXI_OP1,
   MULSUBXI_OP1,
+  // NEON integers vectors
+  VMULADDv8i8_OP1,
+  VMULADDv8i8_OP2,
+  VMULADDv16i8_OP1,
+  VMULADDv16i8_OP2,
+  VMULADDv4i16_OP1,
+  VMULADDv4i16_OP2,
+  VMULADDv8i16_OP1,
+  VMULADDv8i16_OP2,
+  VMULADDv2i32_OP1,
+  VMULADDv2i32_OP2,
+  VMULADDv4i32_OP1,
+  VMULADDv4i32_OP2,
+
+  VMULSUBv8i8_OP1,
+  VMULSUBv8i8_OP2,
+  VMULSUBv16i8_OP1,
+  VMULSUBv16i8_OP2,
+  VMULSUBv4i16_OP1,
+  VMULSUBv4i16_OP2,
+  VMULSUBv8i16_OP1,
+  VMULSUBv8i16_OP2,
+  VMULSUBv2i32_OP1,
+  VMULSUBv2i32_OP2,
+  VMULSUBv4i32_OP1,
+  VMULSUBv4i32_OP2,
+
   // Floating Point
   FMULADDH_OP1,
   FMULADDH_OP2,
   FMULSUBH_OP1,
   FMULSUBH_OP2,
   FMULADDS_OP1,
   FMULADDS_OP2,
   FMULSUBS_OP1,

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

static bool isCombineInstrCandidate64(unsigned Opc) {
   case AArch64::ADDXri:
   case AArch64::SUBXrr:
   case AArch64::ADDSXrr:
   case AArch64::ADDSXri:
   case AArch64::SUBSXrr:
   // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
   case AArch64::SUBXri:
   case AArch64::SUBSXri:
+  case AArch64::ADDv8i8:
+  case AArch64::ADDv16i8:
+  case AArch64::ADDv4i16:
+  case AArch64::ADDv8i16:
+  case AArch64::ADDv2i32:
+  case AArch64::ADDv4i32:
+  case AArch64::SUBv8i8:
+  case AArch64::SUBv16i8:
+  case AArch64::SUBv4i16:
+  case AArch64::SUBv8i16:
+  case AArch64::SUBv2i32:
+  case AArch64::SUBv4i32:
     return true;
   default:
     break;
   }
   return false;
 }
 
 // FP Opcodes that can be combined with a FMUL
static bool getMaddPatterns(MachineInstr &Root,
   auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
                       MachineCombinerPattern Pattern) {
     if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
       Patterns.push_back(Pattern);
       Found = true;
     }
   };
 
+  auto setVFound = [&](int Opcode, int Operand, MachineCombinerPattern Pattern) {
+    if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
+      Patterns.push_back(Pattern);
+      Found = true;
+    }
+  };
+
   typedef MachineCombinerPattern MCP;
 
   switch (Opc) {
   default:
     break;
   case AArch64::ADDWrr:
     assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
            "ADDWrr does not have register operands");
case AArch64::ADDXri:
     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
     break;
   case AArch64::SUBWri:
     setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
     break;
   case AArch64::SUBXri:
     setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
     break;
+  case AArch64::ADDv8i8:
+    setVFound(AArch64::MULv8i8, 1, MCP::VMULADDv8i8_OP1);
+    setVFound(AArch64::MULv8i8, 2, MCP::VMULADDv8i8_OP2);
+    break;
+  case AArch64::ADDv16i8:
+    setVFound(AArch64::MULv16i8, 1, MCP::VMULADDv16i8_OP1);
+    setVFound(AArch64::MULv16i8, 2, MCP::VMULADDv16i8_OP2);
+    break;
+  case AArch64::ADDv4i16:
+    setVFound(AArch64::MULv4i16, 1, MCP::VMULADDv4i16_OP1);
+    setVFound(AArch64::MULv4i16, 2, MCP::VMULADDv4i16_OP2);
+    break;
+  case AArch64::ADDv8i16:
+    setVFound(AArch64::MULv8i16, 1, MCP::VMULADDv8i16_OP1);
+    setVFound(AArch64::MULv8i16, 2, MCP::VMULADDv8i16_OP2);
+    break;
+  case AArch64::ADDv2i32:
+    setVFound(AArch64::MULv2i32, 1, MCP::VMULADDv2i32_OP1);
+    setVFound(AArch64::MULv2i32, 2, MCP::VMULADDv2i32_OP2);
+    break;
+  case AArch64::ADDv4i32:
+    setVFound(AArch64::MULv4i32, 1, MCP::VMULADDv4i32_OP1);
+    setVFound(AArch64::MULv4i32, 2, MCP::VMULADDv4i32_OP2);
+    break;
+  case AArch64::SUBv8i8:
+    setVFound(AArch64::MULv8i8, 1, MCP::VMULSUBv8i8_OP1);
+    setVFound(AArch64::MULv8i8, 2, MCP::VMULSUBv8i8_OP2);
+    break;
+  case AArch64::SUBv16i8:
+    setVFound(AArch64::MULv16i8, 1, MCP::VMULSUBv16i8_OP1);
+    setVFound(AArch64::MULv16i8, 2, MCP::VMULSUBv16i8_OP2);
+    break;
+  case AArch64::SUBv4i16:
+    setVFound(AArch64::MULv4i16, 1, MCP::VMULSUBv4i16_OP1);
+    setVFound(AArch64::MULv4i16, 2, MCP::VMULSUBv4i16_OP2);
+    break;
+  case AArch64::SUBv8i16:
+    setVFound(AArch64::MULv8i16, 1, MCP::VMULSUBv8i16_OP1);
+    setVFound(AArch64::MULv8i16, 2, MCP::VMULSUBv8i16_OP2);
+    break;
+  case AArch64::SUBv2i32:
+    setVFound(AArch64::MULv2i32, 1, MCP::VMULSUBv2i32_OP1);
+    setVFound(AArch64::MULv2i32, 2, MCP::VMULSUBv2i32_OP2);
+    break;
+  case AArch64::SUBv4i32:
+    setVFound(AArch64::MULv4i32, 1, MCP::VMULSUBv4i32_OP1);
+    setVFound(AArch64::MULv4i32, 2, MCP::VMULSUBv4i32_OP2);
+    break;
   }
   return Found;
 }
 /// Floating-Point Support
 
 /// Find instructions that can be turned into madd.
 static bool getFMAPatterns(MachineInstr &Root,
                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {

 /// \param Root is the F|ADD instruction
 /// \param [out] InsInstrs is a vector of machine instructions and will
 /// contain the generated madd instruction
 /// \param IdxMulOpd is index of operand in Root that is the result of
 /// the F|MUL. In the example above IdxMulOpd is 1.
 /// \param MaddOpc the opcode fo the f|madd instruction
 /// \param RC Register class of operands
 /// \param kind of fma instruction (addressing mode) to be generated
 /// \param ReplacedAddend is the result register from the instruction
 /// replacing the non-combined operand, if any.
 static MachineInstr *
 genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
                  const TargetInstrInfo *TII, MachineInstr &Root,
                  SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
                  unsigned MaddOpc, const TargetRegisterClass *RC,
                  FMAInstKind kind = FMAInstKind::Default,
                  const Register *ReplacedAddend = nullptr) {
MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc), ResultReg)
               .addReg(SrcReg1, getKillRegState(Src1IsKill));
   else
     assert(false && "Invalid FMA instruction kind \n");
   // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
   InsInstrs.push_back(MIB);
   return MUL;
 }
 
+/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
+/// instructions.
+///
+/// \see genFusedMultiply
+static MachineInstr *genFusedMultiplyAcc(
+    MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
+    MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
+    unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC,
+    const Register *ReplacedAddend = nullptr) {
+  return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
+                          FMAInstKind::Accumulator, ReplacedAddend);
+}
+
 /// genMaddR - Generate madd instruction and combine mul and add using
 /// an extra virtual register
 /// Example - an ADD intermediate needs to be stored in a register:
 ///   MUL I=A,B,0
 ///   ADD R,I,Imm
 ///   ==> ORR  V, ZR, Imm
 ///   ==> MADD R,A,B,V
 /// \param MF Containing MachineFunction
if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
               .addReg(ZeroReg)
               .addImm(Encoding);
       InsInstrs.push_back(MIB1);
       InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
       MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
     }
     break;
   }
+
+  case MachineCombinerPattern::VMULADDv8i8_OP1:
+    Opc = AArch64::MLAv8i8;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv8i8_OP2:
+    Opc = AArch64::MLAv8i8;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv16i8_OP1:
+    Opc = AArch64::MLAv16i8;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv16i8_OP2:
+    Opc = AArch64::MLAv16i8;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv4i16_OP1:
+    Opc = AArch64::MLAv4i16;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv4i16_OP2:
+    Opc = AArch64::MLAv4i16;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv8i16_OP1:
+    Opc = AArch64::MLAv8i16;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv8i16_OP2:
+    Opc = AArch64::MLAv8i16;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv2i32_OP1:
+    Opc = AArch64::MLAv2i32;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv2i32_OP2:
+    Opc = AArch64::MLAv2i32;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv4i32_OP1:
+    Opc = AArch64::MLAv4i32;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULADDv4i32_OP2:
+    Opc = AArch64::MLAv4i32;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+
+  case MachineCombinerPattern::VMULSUBv8i8_OP1:
+    Opc = AArch64::MLSv8i8;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv8i8_OP2:
+    Opc = AArch64::MLSv8i8;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv16i8_OP1:
+    Opc = AArch64::MLSv16i8;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv16i8_OP2:
+    Opc = AArch64::MLSv16i8;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv4i16_OP1:
+    Opc = AArch64::MLSv4i16;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv4i16_OP2:
+    Opc = AArch64::MLSv4i16;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv8i16_OP1:
+    Opc = AArch64::MLSv8i16;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv8i16_OP2:
+    Opc = AArch64::MLSv8i16;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv2i32_OP1:
+    Opc = AArch64::MLSv2i32;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv2i32_OP2:
+    Opc = AArch64::MLSv2i32;
+    RC = &AArch64::FPR64RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv4i32_OP1:
+    Opc = AArch64::MLSv4i32;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
+    break;
+  case MachineCombinerPattern::VMULSUBv4i32_OP2:
+    Opc = AArch64::MLSv4i32;
+    RC = &AArch64::FPR128RegClass;
+    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
+    break;
+
   // Floating Point Support
   case MachineCombinerPattern::FMULADDH_OP1:
     Opc = AArch64::FMADDHrrr;
     RC = &AArch64::FPR16RegClass;
     MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
     break;
   case MachineCombinerPattern::FMULADDS_OP1:
     Opc = AArch64::FMADDSrrr;

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 def : Pat<(v2f64 (fma (fneg V128:$Rn), V128:$Rm, V128:$Rd)),
           (FMLSv2f64 V128:$Rd, V128:$Rn, V128:$Rm)>;
 
 defm FMULX    : SIMDThreeSameVectorFP<0,0,0b011,"fmulx", int_aarch64_neon_fmulx>;
 defm FMUL     : SIMDThreeSameVectorFP<1,0,0b011,"fmul", fmul>;
 defm FRECPS   : SIMDThreeSameVectorFP<0,0,0b111,"frecps", int_aarch64_neon_frecps>;
 defm FRSQRTS  : SIMDThreeSameVectorFP<0,1,0b111,"frsqrts", int_aarch64_neon_frsqrts>;
 defm FSUB     : SIMDThreeSameVectorFP<0,1,0b010,"fsub", fsub>;
-defm MLA      : SIMDThreeSameVectorBHSTied<0, 0b10010, "mla",
-                      TriOpFrag<(add node:$LHS, (mul node:$MHS, node:$RHS))> >;
-defm MLS      : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls",
-                      TriOpFrag<(sub node:$LHS, (mul node:$MHS, node:$RHS))> >;
+defm MLA      : SIMDThreeSameVectorBHSTied<0, 0b10010, "mla", null_frag>; // Generated in MachineCombine
+defm MLS      : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls", null_frag>; // Generated in MachineCombine
 defm MUL      : SIMDThreeSameVectorBHS<0, 0b10011, "mul", mul>;
 defm PMUL     : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
 defm SABA     : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
       TriOpFrag<(add node:$LHS, (int_aarch64_neon_sabd node:$MHS, node:$RHS))> >;
 defm SABD     : SIMDThreeSameVectorBHS<0,0b01110,"sabd", int_aarch64_neon_sabd>;
 defm SHADD    : SIMDThreeSameVectorBHS<0,0b00000,"shadd", int_aarch64_neon_shadd>;
 defm SHSUB    : SIMDThreeSameVectorBHS<0,0b00100,"shsub", int_aarch64_neon_shsub>;
 defm SMAXP    : SIMDThreeSameVectorBHS<0,0b10100,"smaxp", int_aarch64_neon_smaxp>;

llvm/test/CodeGen/AArch64/GlobalISel/select-with-no-legality-check.mir

body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule944_id3803_at_idx61417
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLAv8i8_:%[0-9]+]]:fpr64 = MLAv8i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv8i8_]]
+    ; CHECK: [[MULv8i8_:%[0-9]+]]:fpr64 = MULv8i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv8i8_:%[0-9]+]]:fpr64 = ADDv8i8 [[MULv8i8_]], [[COPY2]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv8i8_]]
     %4:fpr(<8 x s8>) = COPY $d2
     %3:fpr(<8 x s8>) = COPY $d1
     %2:fpr(<8 x s8>) = COPY $d0
     %0:fpr(<8 x s8>) = G_MUL %3, %4
     %1:fpr(<8 x s8>) = G_ADD %0, %2
     $noreg = PATCHABLE_RET %1(<8 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule945_id3804_at_idx61505
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLAv16i8_:%[0-9]+]]:fpr128 = MLAv16i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv16i8_]]
+    ; CHECK: [[MULv16i8_:%[0-9]+]]:fpr128 = MULv16i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv16i8_:%[0-9]+]]:fpr128 = ADDv16i8 [[MULv16i8_]], [[COPY2]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv16i8_]]
     %4:fpr(<16 x s8>) = COPY $q2
     %3:fpr(<16 x s8>) = COPY $q1
     %2:fpr(<16 x s8>) = COPY $q0
     %0:fpr(<16 x s8>) = G_MUL %3, %4
     %1:fpr(<16 x s8>) = G_ADD %0, %2
     $noreg = PATCHABLE_RET %1(<16 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule946_id3805_at_idx61593
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLAv4i16_:%[0-9]+]]:fpr64 = MLAv4i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv4i16_]]
+    ; CHECK: [[MULv4i16_:%[0-9]+]]:fpr64 = MULv4i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv4i16_:%[0-9]+]]:fpr64 = ADDv4i16 [[MULv4i16_]], [[COPY2]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv4i16_]]
     %4:fpr(<4 x s16>) = COPY $d2
     %3:fpr(<4 x s16>) = COPY $d1
     %2:fpr(<4 x s16>) = COPY $d0
     %0:fpr(<4 x s16>) = G_MUL %3, %4
     %1:fpr(<4 x s16>) = G_ADD %0, %2
     $noreg = PATCHABLE_RET %1(<4 x s16>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule947_id3806_at_idx61681
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLAv8i16_:%[0-9]+]]:fpr128 = MLAv8i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv8i16_]]
+    ; CHECK: [[MULv8i16_:%[0-9]+]]:fpr128 = MULv8i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv8i16_:%[0-9]+]]:fpr128 = ADDv8i16 [[MULv8i16_]], [[COPY2]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv8i16_]]
     %4:fpr(<8 x s16>) = COPY $q2
     %3:fpr(<8 x s16>) = COPY $q1
     %2:fpr(<8 x s16>) = COPY $q0
     %0:fpr(<8 x s16>) = G_MUL %3, %4
     %1:fpr(<8 x s16>) = G_ADD %0, %2
     $noreg = PATCHABLE_RET %1(<8 x s16>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule956_id927_at_idx62401
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLAv8i8_:%[0-9]+]]:fpr64 = MLAv8i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv8i8_]]
+    ; CHECK: [[MULv8i8_:%[0-9]+]]:fpr64 = MULv8i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv8i8_:%[0-9]+]]:fpr64 = ADDv8i8 [[COPY2]], [[MULv8i8_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv8i8_]]
     %4:fpr(<8 x s8>) = COPY $d2
     %3:fpr(<8 x s8>) = COPY $d1
     %2:fpr(<8 x s8>) = COPY $d0
     %0:fpr(<8 x s8>) = G_MUL %3, %4
     %1:fpr(<8 x s8>) = G_ADD %2, %0
     $noreg = PATCHABLE_RET %1(<8 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule957_id928_at_idx62489
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLAv16i8_:%[0-9]+]]:fpr128 = MLAv16i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv16i8_]]
+    ; CHECK: [[MULv16i8_:%[0-9]+]]:fpr128 = MULv16i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv16i8_:%[0-9]+]]:fpr128 = ADDv16i8 [[COPY2]], [[MULv16i8_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv16i8_]]
     %4:fpr(<16 x s8>) = COPY $q2
     %3:fpr(<16 x s8>) = COPY $q1
     %2:fpr(<16 x s8>) = COPY $q0
     %0:fpr(<16 x s8>) = G_MUL %3, %4
     %1:fpr(<16 x s8>) = G_ADD %2, %0
     $noreg = PATCHABLE_RET %1(<16 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule958_id929_at_idx62577
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLAv4i16_:%[0-9]+]]:fpr64 = MLAv4i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv4i16_]]
+    ; CHECK: [[MULv4i16_:%[0-9]+]]:fpr64 = MULv4i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv4i16_:%[0-9]+]]:fpr64 = ADDv4i16 [[COPY2]], [[MULv4i16_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv4i16_]]
     %4:fpr(<4 x s16>) = COPY $d2
     %3:fpr(<4 x s16>) = COPY $d1
     %2:fpr(<4 x s16>) = COPY $d0
     %0:fpr(<4 x s16>) = G_MUL %3, %4
     %1:fpr(<4 x s16>) = G_ADD %2, %0
     $noreg = PATCHABLE_RET %1(<4 x s16>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule959_id930_at_idx62665
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLAv8i16_:%[0-9]+]]:fpr128 = MLAv8i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLAv8i16_]]
+    ; CHECK: [[MULv8i16_:%[0-9]+]]:fpr128 = MULv8i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[ADDv8i16_:%[0-9]+]]:fpr128 = ADDv8i16 [[COPY2]], [[MULv8i16_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[ADDv8i16_]]
     %4:fpr(<8 x s16>) = COPY $q2
     %3:fpr(<8 x s16>) = COPY $q1
     %2:fpr(<8 x s16>) = COPY $q0
     %0:fpr(<8 x s16>) = G_MUL %3, %4
     %1:fpr(<8 x s16>) = G_ADD %2, %0
     $noreg = PATCHABLE_RET %1(<8 x s16>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule977_id933_at_idx64051
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLSv8i8_:%[0-9]+]]:fpr64 = MLSv8i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLSv8i8_]]
+    ; CHECK: [[MULv8i8_:%[0-9]+]]:fpr64 = MULv8i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[SUBv8i8_:%[0-9]+]]:fpr64 = SUBv8i8 [[COPY2]], [[MULv8i8_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[SUBv8i8_]]
     %4:fpr(<8 x s8>) = COPY $d2
     %3:fpr(<8 x s8>) = COPY $d1
     %2:fpr(<8 x s8>) = COPY $d0
     %0:fpr(<8 x s8>) = G_MUL %3, %4
     %1:fpr(<8 x s8>) = G_SUB %2, %0
     $noreg = PATCHABLE_RET %1(<8 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule978_id934_at_idx64139
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLSv16i8_:%[0-9]+]]:fpr128 = MLSv16i8 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLSv16i8_]]
+    ; CHECK: [[MULv16i8_:%[0-9]+]]:fpr128 = MULv16i8 [[COPY1]], [[COPY]]
+    ; CHECK: [[SUBv16i8_:%[0-9]+]]:fpr128 = SUBv16i8 [[COPY2]], [[MULv16i8_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[SUBv16i8_]]
     %4:fpr(<16 x s8>) = COPY $q2
     %3:fpr(<16 x s8>) = COPY $q1
     %2:fpr(<16 x s8>) = COPY $q0
     %0:fpr(<16 x s8>) = G_MUL %3, %4
     %1:fpr(<16 x s8>) = G_SUB %2, %0
     $noreg = PATCHABLE_RET %1(<16 x s8>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $d0, $d1, $d2
 
     ; CHECK-LABEL: name: test_rule979_id935_at_idx64227
     ; CHECK: liveins: $d0, $d1, $d2
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK: [[MLSv4i16_:%[0-9]+]]:fpr64 = MLSv4i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLSv4i16_]]
+    ; CHECK: [[MULv4i16_:%[0-9]+]]:fpr64 = MULv4i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[SUBv4i16_:%[0-9]+]]:fpr64 = SUBv4i16 [[COPY2]], [[MULv4i16_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[SUBv4i16_]]
     %4:fpr(<4 x s16>) = COPY $d2
     %3:fpr(<4 x s16>) = COPY $d1
     %2:fpr(<4 x s16>) = COPY $d0
     %0:fpr(<4 x s16>) = G_MUL %3, %4
     %1:fpr(<4 x s16>) = G_SUB %2, %0
     $noreg = PATCHABLE_RET %1(<4 x s16>)
 
 ...
body:             |
   bb.0.entry:
     liveins: $q0, $q1, $q2
 
     ; CHECK-LABEL: name: test_rule980_id936_at_idx64315
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q2
     ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
     ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK: [[MLSv8i16_:%[0-9]+]]:fpr128 = MLSv8i16 [[COPY2]], [[COPY1]], [[COPY]]
-    ; CHECK: $noreg = PATCHABLE_RET [[MLSv8i16_]]
+    ; CHECK: [[MULv8i16_:%[0-9]+]]:fpr128 = MULv8i16 [[COPY1]], [[COPY]]
+    ; CHECK: [[SUBv8i16_:%[0-9]+]]:fpr128 = SUBv8i16 [[COPY2]], [[MULv8i16_]]
+    ; CHECK: $noreg = PATCHABLE_RET [[SUBv8i16_]]
     %4:fpr(<8 x s16>) = COPY $q2
     %3:fpr(<8 x s16>) = COPY $q1
     %2:fpr(<8 x s16>) = COPY $q0
     %0:fpr(<8 x s16>) = G_MUL %3, %4
     %1:fpr(<8 x s16>) = G_SUB %2, %0
     $noreg = PATCHABLE_RET %1(<8 x s16>)
 
 ...

llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc %s --mtriple aarch64 -verify-machineinstrs -o - | FileCheck %s
 
 define dso_local void @jsimd_idct_ifast_neon_intrinsic(i8* nocapture readonly %dct_table, i16* nocapture readonly %coef_block, i8** nocapture readonly %output_buf, i32 %output_col) local_unnamed_addr #0 {
 ; CHECK-LABEL: jsimd_idct_ifast_neon_intrinsic:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr q0, [x1, #32]
-; CHECK-NEXT:    ldr q1, [x0, #32]
-; CHECK-NEXT:    ldr q2, [x1, #96]
+; CHECK-NEXT:    ldr q1, [x1, #96]
+; CHECK-NEXT:    ldr q2, [x0, #32]
 ; CHECK-NEXT:    ldr q3, [x0, #96]
 ; CHECK-NEXT:    ldr x8, [x2, #48]
-; CHECK-NEXT:    mul v0.8h, v1.8h, v0.8h
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    mla v1.8h, v3.8h, v2.8h
 ; CHECK-NEXT:    mov w9, w3
-; CHECK-NEXT:    str q1, [x8, x9]
+; CHECK-NEXT:    mul v0.8h, v2.8h, v0.8h
+; CHECK-NEXT:    mul v1.8h, v3.8h, v1.8h
+; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    str q2, [x8, x9]
 ; CHECK-NEXT:    ldr x8, [x2, #56]
-; CHECK-NEXT:    mls v0.8h, v3.8h, v2.8h
+; CHECK-NEXT:    sub v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    str q0, [x8, x9]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr5 = getelementptr inbounds i16, i16* %coef_block, i64 16
   %0 = bitcast i16* %add.ptr5 to <8 x i16>*
   %1 = load <8 x i16>, <8 x i16>* %0, align 16
 
   %add.ptr17 = getelementptr inbounds i16, i16* %coef_block, i64 48