[AArch64] Adjust the cost model for Exynos M1 and M2

[evandro]

Summary

Add new predicate to more accurately model the cost of arithmetic and logical operations shifted left.

[javed.absar]

llvm/lib/Target/AArch64/AArch64InstrInfo.h

372 ↗

(On Diff #112704)

Is it for shift left only, and if so should it be then called out explicitly?

[MatzeB]

• I'd leave out the if (Subtarget.getProcFamily() != AArch64Subtarget::ExynosM1) return false; check in isExynosShiftExtFast() if someone chose the Exynos scheduling he probably wants that behavior even if getProcFamily() doesn't match. (You can pull the check out to isAsCheapAsAMove()).
• This must have happened in an earlier commit already, but in any case I'd really discourage the use of Subtarget.getProcFamily(), better create a subtarget feature to control the behavior.

[evandro]

In D37151#852690, @MatzeB wrote:

• I'd leave out the if (Subtarget.getProcFamily() != AArch64Subtarget::ExynosM1) return false; check in isExynosShiftExtFast() if someone chose the Exynos scheduling he probably wants that behavior even if getProcFamily() doesn't match. (You can pull the check out to isAsCheapAsAMove()).

OK

• This must have happened in an earlier commit already, but in any case I'd really discourage the use of Subtarget.getProcFamily(), better create a subtarget feature to control the behavior.

It's legacy code before the refactoring of such calls that I didn't feel this to be the time to address.

[MatzeB]

LGTM

llvm/trunk/lib/Target/AArch64/AArch64InstrInfo.h

public:
   void insertOutlinerEpilogue(MachineBasicBlock &MBB, MachineFunction &MF,
                               unsigned FrameClass) const override;
   void insertOutlinerPrologue(MachineBasicBlock &MBB, MachineFunction &MF,
                               unsigned FrameClass) const override;
   MachineBasicBlock::iterator
   insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
                      MachineBasicBlock::iterator &It, MachineFunction &MF,
                      unsigned CallClass) const override;
+  /// Returns true if the instruction has a shift left that can be executed
+  /// more efficiently.
+  bool isExynosShiftLeftFast(const MachineInstr &MI) const;
   /// Returns true if the instruction has a shift by immediate that can be
   /// executed in one cycle less.
   bool isFalkorShiftExtFast(const MachineInstr &MI) const;
 
 private:
   /// \brief Sets the offsets on outlined instructions in \p MBB which use SP
   /// so that they will be valid post-outlining.
   ///

llvm/trunk/lib/Target/AArch64/AArch64InstrInfo.cpp

static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
   return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
 }
 
 // FIXME: this implementation should be micro-architecture dependent, so a
 // micro-architecture target hook should be introduced here in future.
 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
   if (!Subtarget.hasCustomCheapAsMoveHandling())
     return MI.isAsCheapAsAMove();
-
-  unsigned Imm;
+  if (Subtarget.getProcFamily() == AArch64Subtarget::ExynosM1 &&
+      isExynosShiftLeftFast(MI))
+    return true;
 
   switch (MI.getOpcode()) {
   default:
     return false;
 
   // add/sub on register without shift
   case AArch64::ADDWri:
   case AArch64::ADDXri:
   case AArch64::SUBWri:
   case AArch64::SUBXri:
-    return (Subtarget.getProcFamily() == AArch64Subtarget::ExynosM1 ||
-            MI.getOperand(3).getImm() == 0);
-
-  // add/sub on register with shift
-  case AArch64::ADDWrs:
-  case AArch64::ADDXrs:
-  case AArch64::SUBWrs:
-  case AArch64::SUBXrs:
-    Imm = MI.getOperand(3).getImm();
-    return (Subtarget.getProcFamily() == AArch64Subtarget::ExynosM1 &&
-            AArch64_AM::getArithShiftValue(Imm) < 4);
+    return (MI.getOperand(3).getImm() == 0);
 
   // logical ops on immediate
   case AArch64::ANDWri:
   case AArch64::ANDXri:
   case AArch64::EORWri:
   case AArch64::EORXri:
   case AArch64::ORRWri:
   case AArch64::ORRXri:
bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
   case AArch64::EORWrr:
   case AArch64::EORXrr:
   case AArch64::ORNWrr:
   case AArch64::ORNXrr:
   case AArch64::ORRWrr:
   case AArch64::ORRXrr:
     return true;
 
-  // logical ops on register with shift
-  case AArch64::ANDWrs:
-  case AArch64::ANDXrs:
-  case AArch64::BICWrs:
-  case AArch64::BICXrs:
-  case AArch64::EONWrs:
-  case AArch64::EONXrs:
-  case AArch64::EORWrs:
-  case AArch64::EORXrs:
-  case AArch64::ORNWrs:
-  case AArch64::ORNXrs:
-  case AArch64::ORRWrs:
-  case AArch64::ORRXrs:
-    Imm = MI.getOperand(3).getImm();
-    return (Subtarget.getProcFamily() == AArch64Subtarget::ExynosM1 &&
-            AArch64_AM::getShiftValue(Imm) < 4 &&
-            AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL);
-
   // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
   // ORRXri, it is as cheap as MOV
   case AArch64::MOVi32imm:
     return canBeExpandedToORR(MI, 32);
   case AArch64::MOVi64imm:
     return canBeExpandedToORR(MI, 64);
 
   // It is cheap to zero out registers if the subtarget has ZeroCycleZeroing
   // feature.
   case AArch64::FMOVH0:
   case AArch64::FMOVS0:
   case AArch64::FMOVD0:
     return Subtarget.hasZeroCycleZeroing();
   case TargetOpcode::COPY:
     return (Subtarget.hasZeroCycleZeroing() &&
             (MI.getOperand(1).getReg() == AArch64::WZR ||
              MI.getOperand(1).getReg() == AArch64::XZR));
   }
 
   llvm_unreachable("Unknown opcode to check as cheap as a move!");
 }
 
+bool AArch64InstrInfo::isExynosShiftLeftFast(const MachineInstr &MI) const {
+  unsigned Imm, Shift;
+
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+
+  // WriteI
+  case AArch64::ADDSWri:
+  case AArch64::ADDSXri:
+  case AArch64::ADDWri:
+  case AArch64::ADDXri:
+  case AArch64::SUBSWri:
+  case AArch64::SUBSXri:
+  case AArch64::SUBWri:
+  case AArch64::SUBXri:
+    return true;
+
+  // WriteISReg
+  case AArch64::ADDSWrs:
+  case AArch64::ADDSXrs:
+  case AArch64::ADDWrs:
+  case AArch64::ADDXrs:
+  case AArch64::ANDSWrs:
+  case AArch64::ANDSXrs:
+  case AArch64::ANDWrs:
+  case AArch64::ANDXrs:
+  case AArch64::BICSWrs:
+  case AArch64::BICSXrs:
+  case AArch64::BICWrs:
+  case AArch64::BICXrs:
+  case AArch64::EONWrs:
+  case AArch64::EONXrs:
+  case AArch64::EORWrs:
+  case AArch64::EORXrs:
+  case AArch64::ORNWrs:
+  case AArch64::ORNXrs:
+  case AArch64::ORRWrs:
+  case AArch64::ORRXrs:
+  case AArch64::SUBSWrs:
+  case AArch64::SUBSXrs:
+  case AArch64::SUBWrs:
+  case AArch64::SUBXrs:
+    Imm = MI.getOperand(3).getImm();
+    Shift = AArch64_AM::getShiftValue(Imm);
+    return (Shift == 0 ||
+            (Shift <= 3 && AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL));
+
+  // WriteIEReg
+  case AArch64::ADDSWrx:
+  case AArch64::ADDSXrx:
+  case AArch64::ADDSXrx64:
+  case AArch64::ADDWrx:
+  case AArch64::ADDXrx:
+  case AArch64::ADDXrx64:
+  case AArch64::SUBSWrx:
+  case AArch64::SUBSXrx:
+  case AArch64::SUBSXrx64:
+  case AArch64::SUBWrx:
+  case AArch64::SUBXrx:
+  case AArch64::SUBXrx64:
+    Imm = MI.getOperand(3).getImm();
+    Shift = AArch64_AM::getArithShiftValue(Imm);
+    return (Shift == 0 ||
+            (Shift <= 3 && AArch64_AM::getExtendType(Imm) == AArch64_AM::UXTX));
+  }
+}
+
 bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   default:
     return false;
 
   case AArch64::ADDWrs:
   case AArch64::ADDXrs:
   case AArch64::ADDSWrs:

llvm/trunk/lib/Target/AArch64/AArch64SchedM1.td

def M1UnitALU  : ProcResGroup<[M1UnitA,
                                  M1UnitC]>;    // All integer
   def M1UnitNALU : ProcResGroup<[M1UnitNAL0,
                                  M1UnitNAL1]>; // All simple vector
 }
 
 let SchedModel = ExynosM1Model in {
 
 //===----------------------------------------------------------------------===//
-// Coarse scheduling model for the Exynos-M1.
+// Predicates.
+
+def M1ShiftLeftFastPred : SchedPredicate<[{TII->isExynosShiftLeftFast(*MI)}]>;
+
+//===----------------------------------------------------------------------===//
+// Coarse scheduling model.
 
 def M1WriteA1 : SchedWriteRes<[M1UnitALU]> { let Latency = 1; }
 def M1WriteA2 : SchedWriteRes<[M1UnitALU]> { let Latency = 2; }
+def M1WriteAA : SchedWriteRes<[M1UnitALU]> { let Latency = 2;
+                                             let ResourceCycles = [2]; }
+def M1WriteAX : SchedWriteVariant<[SchedVar<M1ShiftLeftFastPred, [M1WriteA1]>,
+                                   SchedVar<NoSchedPred,         [M1WriteAA]>]>;
 def M1WriteC1 : SchedWriteRes<[M1UnitC]>   { let Latency = 1; }
 def M1WriteC2 : SchedWriteRes<[M1UnitC]>   { let Latency = 2; }
 
-def M1WriteB1 : SchedWriteRes<[M1UnitB]>      { let Latency = 1; }
+def M1WriteB1 : SchedWriteRes<[M1UnitB]> { let Latency = 1; }
 
-def M1WriteL5 : SchedWriteRes<[M1UnitL]>   { let Latency = 5; }
+def M1WriteL5 : SchedWriteRes<[M1UnitL]> { let Latency = 5; }
 def M1WriteLX : SchedWriteVariant<[SchedVar<ScaledIdxPred, [M1WriteL5,
                                                             M1WriteA1]>,
                                    SchedVar<NoSchedPred,   [M1WriteL5]>]>;
 
 def M1WriteS1 : SchedWriteRes<[M1UnitS]> { let Latency = 1; }
 def M1WriteS2 : SchedWriteRes<[M1UnitS]> { let Latency = 2; }
 def M1WriteS4 : SchedWriteRes<[M1UnitS]> { let Latency = 4; }
 def M1WriteSX : SchedWriteVariant<[SchedVar<ScaledIdxPred, [M1WriteS2,
                                                             M1WriteA1]>,
                                    SchedVar<NoSchedPred,   [M1WriteS1]>]>;
 
 def M1ReadAdrBase : SchedReadVariant<[SchedVar<ScaledIdxPred, [ReadDefault]>,
                                       SchedVar<NoSchedPred,   [ReadDefault]>]>;
-def : SchedAlias<ReadAdrBase, M1ReadAdrBase>;
 
 // Branch instructions.
 // NOTE: Unconditional direct branches actually take neither cycles nor units.
 def : WriteRes<WriteBr,    [M1UnitB]> { let Latency = 1; }
 def : WriteRes<WriteBrReg, [M1UnitC]> { let Latency = 1; }
 
 // Arithmetic and logical integer instructions.
 def : WriteRes<WriteI,     [M1UnitALU]> { let Latency = 1; }
-// TODO: Shift over 3 and some extensions take 2 cycles.
 def : WriteRes<WriteISReg, [M1UnitALU]> { let Latency = 1; }
 def : WriteRes<WriteIEReg, [M1UnitALU]> { let Latency = 1; }
 def : WriteRes<WriteIS,    [M1UnitALU]> { let Latency = 1; }
 
 // Move instructions.
 def : WriteRes<WriteImm, [M1UnitALU]> { let Latency = 1; }
 
 // Divide and multiply instructions.
 def : WriteRes<WriteID32, [M1UnitC,
                            M1UnitD]> { let Latency = 13;
                                        let ResourceCycles = [1, 13]; }
 def : WriteRes<WriteID64, [M1UnitC,
                            M1UnitD]> { let Latency = 21;
                                        let ResourceCycles = [1, 21]; }
 // TODO: Long multiplication take 5 cycles and also the ALU.
-// TODO: Multiplication with accumulation can be advanced.
 def : WriteRes<WriteIM32, [M1UnitC]> { let Latency = 3; }
 // TODO: 64-bit multiplication has a throughput of 1/2.
 def : WriteRes<WriteIM64, [M1UnitC]> { let Latency = 4; }
 
 // Miscellaneous instructions.
 def : WriteRes<WriteExtr, [M1UnitALU,
                            M1UnitALU]> { let Latency = 2; }
 
+// Addressing modes.
 // TODO: The latency for the post or pre register is 1 cycle.
 def : WriteRes<WriteAdr, []> { let Latency = 0; }
+def : SchedAlias<ReadAdrBase, M1ReadAdrBase>;
 
 // Load instructions.
 def : WriteRes<WriteLD,    [M1UnitL]>   { let Latency = 4; }
 def : WriteRes<WriteLDHi,  [M1UnitALU]> { let Latency = 4; }
 def : SchedAlias<WriteLDIdx, M1WriteLX>;
 
 // Store instructions.
 def : WriteRes<WriteST,    [M1UnitS]> { let Latency = 1; }

 def : WriteRes<WriteBarrier, []> { let Latency = 1; }
 def : WriteRes<WriteHint,    []> { let Latency = 1; }
 def : WriteRes<WriteSys,     []> { let Latency = 1; }
 
 //===----------------------------------------------------------------------===//
 // Generic fast forwarding.
 
 // TODO: Add FP register forwarding rules.
-
 def : ReadAdvance<ReadI,       0>;
 def : ReadAdvance<ReadISReg,   0>;
 def : ReadAdvance<ReadIEReg,   0>;
 def : ReadAdvance<ReadIM,      0>;
-// Integer multiply-accumulate.
 // TODO: The forwarding for WriteIM64 saves actually 3 cycles.
 def : ReadAdvance<ReadIMA,     2, [WriteIM32, WriteIM64]>;
 def : ReadAdvance<ReadID,      0>;
 def : ReadAdvance<ReadExtrHi,  0>;
 def : ReadAdvance<ReadAdrBase, 0>;
 def : ReadAdvance<ReadVLD,     0>;
 
 //===----------------------------------------------------------------------===//
-// Finer scheduling model for the Exynos-M1.
+// Finer scheduling model.
 
 def M1WriteNEONA   : SchedWriteRes<[M1UnitNALU,
                                     M1UnitNALU,
                                     M1UnitFADD]>   { let Latency = 9; }
 def M1WriteNEONB   : SchedWriteRes<[M1UnitNALU,
                                     M1UnitFST]>    { let Latency = 5; }
 def M1WriteNEONC   : SchedWriteRes<[M1UnitNALU,
                                     M1UnitFST]>    { let Latency = 6; }
def M1WriteVLDM    : SchedWriteRes<[M1UnitL,
                                     M1UnitNALU,
                                     M1UnitNALU]>   { let Latency = 7;
                                                      let ResourceCycles = [2]; }
 def M1WriteVLDN    : SchedWriteRes<[M1UnitL,
                                     M1UnitL,
                                     M1UnitL,
                                     M1UnitL]>      { let Latency = 14;
                                                      let ResourceCycles = [7]; }
-
 def M1WriteVSTA    : WriteSequence<[WriteVST], 2>;
 def M1WriteVSTB    : WriteSequence<[WriteVST], 3>;
 def M1WriteVSTC    : WriteSequence<[WriteVST], 4>;
 def M1WriteVSTD    : SchedWriteRes<[M1UnitS,
                                     M1UnitFST,
                                     M1UnitFST]>    { let Latency = 7;
                                                      let ResourceCycles = [7]; }
 def M1WriteVSTE    : SchedWriteRes<[M1UnitS,
def M1WriteVSTI    : SchedWriteRes<[M1UnitNALU,
                                     M1UnitS,
                                     M1UnitFST,
                                     M1UnitFST,
                                     M1UnitFST]>      { let Latency = 17;
                                                        let ResourceCycles = [17]; }
 
 // Branch instructions
 def : InstRW<[M1WriteB1], (instrs Bcc)>;
-// NOTE: Conditional branch and link adds a B uop.
 def : InstRW<[M1WriteA1], (instrs BL)>;
 // NOTE: Indirect branch and link with LR adds an ALU uop.
 def : InstRW<[M1WriteA1,
               M1WriteC1], (instrs BLR)>;
 def : InstRW<[M1WriteC1], (instregex "^CBN?Z[WX]")>;
 def : InstRW<[M1WriteC1,
               M1WriteA2], (instregex "^TBN?Z[WX]")>;
 
 // Arithmetic and logical integer instructions.
 def : InstRW<[M1WriteA1], (instrs COPY)>;
+def : InstRW<[M1WriteAX], (instregex ".+r[sx](64)?$")>;
 
 // Divide and multiply instructions.
 
 // Miscellaneous instructions.
 
 // Load instructions.
 
 // Store instructions.

 def : InstRW<[M1WriteNMISC1], (instregex "^[SU](MIN|MAX)v")>;
 def : InstRW<[M1WriteNMISC2], (instregex "^[SU](MIN|MAX)Pv")>;
 def : InstRW<[M1WriteNMISC3], (instregex "^[SU](MIN|MAX)Vv")>;
 def : InstRW<[M1WriteNMISC4], (instregex "^(MUL|SQR?DMULH)v")>;
 def : InstRW<[M1WriteNMISC4], (instregex "^ML[AS]v")>;
 def : InstRW<[M1WriteNMISC4], (instregex "^(S|U|SQD|SQRD)ML[AS][HL]v")>;
 def : InstRW<[M1WriteNMISC4], (instregex "^(S|U|SQD)MULLv")>;
 def : InstRW<[M1WriteNAL13],  (instregex "^(S|SR|U|UR)SRAv")>;
-def : InstRW<[M1WriteNALU1],  (instregex "^[SU]?SH(L|LL|R)2?v")>;
-def : InstRW<[M1WriteNALU1],  (instregex "^S[LR]Iv")>;
-def : InstRW<[M1WriteNAL13],  (instregex "^[SU]?(Q|QR|R)?SHR(N|U|UN)?2?v")>;
-def : InstRW<[M1WriteNAL13],  (instregex "^[SU](Q|QR|R)SHLU?v")>;
+def : InstRW<[M1WriteNALU1],  (instregex "^SHL[dv]")>;
+def : InstRW<[M1WriteNALU1],  (instregex "^[SU]SH[LR][dv]")>;
+def : InstRW<[M1WriteNALU1],  (instregex "^S[RS]I[dv]")>;
+def : InstRW<[M1WriteNAL13],  (instregex "^(([SU]Q)?R)?SHRU?N[bhsv]")>;
+def : InstRW<[M1WriteNAL13],  (instregex "^[SU]RSH[LR][dv]")>;
+def : InstRW<[M1WriteNAL13],  (instregex "^[SU]QR?SHLU?[bdhsv]")>;
 
 // ASIMD FP instructions.
 def : InstRW<[M1WriteNALU1],  (instregex "^F(ABS|NEG)v")>;
 def : InstRW<[M1WriteNMISC3], (instregex "^F(ABD|ADD|SUB)v")>;
 def : InstRW<[M1WriteNEONA],  (instregex "^FADDP")>;
 def : InstRW<[M1WriteNMISC1], (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)v[^1]")>;
 def : InstRW<[M1WriteFCVT3],  (instregex "^[FVSU]CVTX?[AFLMNPZ][SU]?(_Int)?v")>;
 def : InstRW<[M1WriteFVAR15], (instregex "FDIVv.f32")>;