ARM Cortex A57 scheduler fix (missed 16-bit, v8.1/v8.2/v8.3, thumb and pseudo instructions)

[andrew.zhogin]

Summary

Added missed scheduling info for ARM Cortex A57 (AArch32) to have CompleteModel with this checkCompleteness fix: https://reviews.llvm.org/D43235.

[javed.absar]

LGTM.

By the way, did you do any bench-marking to see if there were any impact of these changes, although from the changes it looks to me there wont be anything significant, and besides I understand this is simply for CompleteModel.

[RKSimon]

Confirmed - with this fix I can remove the ARMScheduleA57.td change on D43235

[andrew.zhogin]

In D43808#1020389, @javed.absar wrote:

LGTM.

By the way, did you do any bench-marking to see if there were any impact of these changes, although from the changes it looks to me there wont be anything significant, and besides I understand this is simply for CompleteModel.

No, I didn't.
Most of those added instructions from v8.1/v8.2/v8.3. I think only 16-bit VMLA (VMLAH/VMLSH/VNMLAH/VNMLSH) may have some practical meaning.

llvm/trunk/lib/Target/ARM/ARMScheduleA57.td

 include "ARMScheduleA57WriteRes.td"
 
 // To have "CompleteModel = 1", support of pseudos and special instructions
 def : InstRW<[WriteNoop], (instregex "(t)?BKPT$", "(t2)?CDP(2)?$",
   "(t2)?CLREX$", "CONSTPOOL_ENTRY$", "COPY_STRUCT_BYVAL_I32$",
   "(t2)?CPS[123]p$", "(t2)?DBG$", "(t2)?DMB$", "(t2)?DSB$", "ERET$",
   "(t2|t)?HINT$", "(t)?HLT$", "(t2)?HVC$", "(t2)?ISB$", "ITasm$",
   "(t2)?RFE(DA|DB|IA|IB)", "(t)?SETEND", "(t2)?SETPAN", "(t2)?SMC", "SPACE",
-  "(t2)?SRS(DA|DB|IA|IB)", "SWP(B)?", "t?TRAP", "UDF$", "t2DCPS", "t2SG",
-  "t2TT", "tCPS", "CMP_SWAP", "t?SVC", "t2IT", "CompilerBarrier")>;
+  "(t2)?SRS(DA|DB|IA|IB)", "SWP(B)?", "t?TRAP", "(t2|t)?UDF$", "t2DCPS", "t2SG",
+  "t2TT", "tCPS", "CMP_SWAP", "t?SVC", "t2IT", "CompilerBarrier",
+  "t__brkdiv0")>;
 
 def : InstRW<[WriteNoop], (instregex "VMRS", "VMSR", "FMSTAT")>;
 
 // Specific memory instrs
 def : InstRW<[WriteNoop, WriteNoop], (instregex "(t2)?LDA", "(t2)?LDC", "(t2)?STC",
   "(t2)?STL", "(t2)?LDREX", "(t2)?STREX", "MEMCPY")>;
 
 // coprocessor moves
 def : InstRW<[WriteNoop, WriteNoop], (instregex
   "(t2)?MCR(2|R|R2)?$", "(t2)?MRC(2)?$",
   "(t2)?MRRC(2)?$", "(t2)?MRS(banked|sys|_AR|_M|sys_AR)?$",
   "(t2)?MSR(banked|i|_AR|_M)?$")>;
 
 // Deprecated instructions
 def : InstRW<[WriteNoop], (instregex "FLDM", "FSTM")>;
 
 // Pseudos
 def : InstRW<[WriteNoop], (instregex "(t2)?ABS$",
   "(t)?ADJCALLSTACKDOWN$", "(t)?ADJCALLSTACKUP$", "(t2|t)?Int_eh_sjlj",
-  "tLDRpci_pic", "t2SUBS_PC_LR",
+  "tLDRpci_pic", "(t2)?SUBS_PC_LR",
   "JUMPTABLE", "tInt_WIN_eh_sjlj_longjmp",
   "VLD(1|2)LN(d|q)(WB_fixed_|WB_register_)?Asm",
   "VLD(3|4)(DUP|LN)?(d|q)(WB_fixed_|WB_register_)?Asm",
   "VST(1|2)LN(d|q)(WB_fixed_|WB_register_)?Asm",
   "VST(3|4)(DUP|LN)?(d|q)(WB_fixed_|WB_register_)?Asm",
   "WIN__CHKSTK", "WIN__DBZCHK")>;
 
 // Miscellaneous

 // SMLAWT, SMLAD{X}, SMLSD{X}, SMMLA{R}, SMMLS{R}
 // Multiply-accumulate pipelines support late-forwarding of accumulate operands
 // from similar μops, allowing a typical sequence of multiply-accumulate μops
 // to issue one every 1 cycle (sched advance = 2).
 def A57WriteMLA : SchedWriteRes<[A57UnitM]> { let Latency = 3; }
 def A57WriteMLAL : SchedWriteRes<[A57UnitM]> { let Latency = 4; }
 def A57ReadMLA  : SchedReadAdvance<2, [A57WriteMLA, A57WriteMLAL]>;
 
+def : InstRW<[A57WriteMLA],
+  (instregex "t2SMLAD", "t2SMLADX", "t2SMLSD", "t2SMLSDX")>;
+
 def : SchedAlias<WriteMAC16, A57WriteMLA>;
 def : SchedAlias<WriteMAC32, A57WriteMLA>;
 def : SchedAlias<ReadMAC,    A57ReadMLA>;
 
 def : SchedAlias<WriteMAC64Lo, A57WriteMLAL>;
 def : SchedAlias<WriteMAC64Hi, A57WriteMLAL>;
 
 // Multiply long: SMULL, UMULL

 ]> { let Variadic=1; }
 
 def : InstRW<[A57WriteLDM], (instregex "(t|t2|sys)?LDM(IA|DA|DB|IB)$")>;
 
 // TODO: no writeback latency defined in documentation (implemented as 1 cyc)
 def : InstRW<[A57WriteLDM_Upd],
   (instregex "(t|t2|sys)?LDM(IA_UPD|DA_UPD|DB_UPD|IB_UPD|IA_RET)", "tPOP")>;
 
+def : InstRW<[A57Write_5cyc_1L], (instregex "VLLDM")>;
+
 // --- 3.9 Store Instructions ---
 
 // Store, immed offset
 def : InstRW<[A57Write_1cyc_1S], (instregex "STRi12", "STRBi12", "PICSTR",
   "t2STR(B?)(T|i12|i8|s)", "t2STRDi8", "t2STRH(i12|i8|s)", "tSTR")>;
 
 // Store, register offset
 // For minus or for not plus lsl2 scaled we need 3cyc "I0/I1, S",
def A57WriteSTM_Upd : SchedWriteVariant<[
     SchedVar<A57LMAddrPred8, [A57Write_8cyc_1S_1I]>,
     SchedVar<NoSchedPred,    [A57Write_2cyc_1S_1I]>
 ]>;
 
 def : InstRW<[A57WriteSTM], (instregex "(t2|sys|t)?STM(IA|DA|DB|IB)$")>;
 def : InstRW<[A57WrBackOne, A57WriteSTM_Upd],
   (instregex "(t2|sys|t)?STM(IA_UPD|DA_UPD|DB_UPD|IB_UPD)", "tPUSH")>;
 
+def : InstRW<[A57Write_5cyc_1S], (instregex "VLSTM")>;
+
 // --- 3.10 FP Data Processing Instructions ---
 def : SchedAlias<WriteFPALU32, A57Write_5cyc_1V>;
 def : SchedAlias<WriteFPALU64, A57Write_5cyc_1V>;
 
 def : InstRW<[A57Write_3cyc_1V], (instregex "VABS(S|D|H)")>;
 
 // fp compare - 3cyc F1 for unconditional, 6cyc "F0/F1, F1" for conditional
 def A57WriteVcmp : SchedWriteVariant<[
   SchedVar<IsPredicatedPred, [A57Write_6cyc_1V_1X]>,
   SchedVar<NoSchedPred,      [A57Write_3cyc_1X]>
 ]>;
 def : InstRW<[A57WriteVcmp],
   (instregex "VCMP(D|S|H|ZD|ZS|ZH)$", "VCMPE(D|S|H|ZD|ZS|ZH)")>;
 
 // fp convert
 def : InstRW<[A57Write_5cyc_1V], (instregex
   "VCVT(A|N|P|M)(SH|UH|SS|US|SD|UD)", "VCVT(BDH|THD|TDH)")>;
-
+def : InstRW<[A57Write_5cyc_1V], (instregex "VTOSLS", "VTOUHS", "VTOULS")>;
 def : SchedAlias<WriteFPCVT, A57Write_5cyc_1V>;
 
+def : InstRW<[A57Write_5cyc_1V], (instregex "VJCVT")>;
+
 // FP round to integral
 def : InstRW<[A57Write_5cyc_1V], (instregex "VRINT(A|N|P|M|Z|R|X)(H|S|D)$")>;
 
 // FP divide, FP square root
 def : SchedAlias<WriteFPDIV32, A57Write_17cyc_1W>;
 def : SchedAlias<WriteFPDIV64, A57Write_32cyc_1W>;
 def : SchedAlias<WriteFPSQRT32, A57Write_17cyc_1W>;
 def : SchedAlias<WriteFPSQRT64, A57Write_32cyc_1W>;
 
+def : InstRW<[A57Write_17cyc_1W], (instregex "VSQRTH")>;
+
 // FP max/min
 def : InstRW<[A57Write_5cyc_1V], (instregex "VMAX", "VMIN")>;
 
 // FP multiply-accumulate pipelines support late forwarding of the result
 // from FP multiply μops to the accumulate operands of an
 // FP multiply-accumulate μop. The latter can potentially be issued 1 cycle
 // after the FP multiply μop has been issued
 // FP multiply, FZ

 // def : ReadAdvance<A57ReadVFMA, 5, [A57WriteVFMA]>;
 // def : ReadAdvance<A57ReadVFMA, 4, [A57WriteVMUL]>;
 def A57ReadVFMA5 : SchedReadAdvance<5, [A57WriteVFMA, A57WriteVMUL]>;
 
 def : SchedAlias<WriteFPMAC32, A57WriteVFMA>;
 def : SchedAlias<WriteFPMAC64, A57WriteVFMA>;
 def : SchedAlias<ReadFPMAC, A57ReadVFMA5>;
 
+// VMLAH/VMLSH are not binded to scheduling classes by default, so here custom:
+def : InstRW<[A57WriteVFMA, A57ReadVFMA5, ReadFPMUL, ReadFPMUL],
+  (instregex "VMLAH", "VMLSH", "VNMLAH", "VNMLSH")>;
+
+def : InstRW<[A57WriteVMUL],
+  (instregex "VUDOTD", "VSDOTD", "VUDOTQ", "VSDOTQ")>;
+
 def : InstRW<[A57Write_3cyc_1V], (instregex "VNEG")>;
 def : InstRW<[A57Write_3cyc_1V], (instregex "VSEL")>;
 
 // --- 3.11 FP Miscellaneous Instructions ---
 // VMOV: 3cyc "F0/F1" for imm/reg
 def : InstRW<[A57Write_3cyc_1V], (instregex "FCONST(D|S|H)")>;
 def : InstRW<[A57Write_3cyc_1V], (instregex "VMOV(D|S|H)(cc)?$")>;
 
+def : InstRW<[A57Write_3cyc_1V], (instregex "VINSH")>;
+
 // 5cyc L for FP transfer, vfp to core reg,
 // 5cyc L for FP transfer, core reg to vfp
 def : SchedAlias<WriteFPMOV, A57Write_5cyc_1L>;
 // VMOVRRS/VMOVRRD in common code declared with one WriteFPMOV (instead of 2).
 def : InstRW<[A57Write_5cyc_1L, A57Write_5cyc_1L], (instregex "VMOV(RRS|RRD)")>;
 
 // 8cyc "L,F0/F1" for FP transfer, core reg to upper or lower half of vfp D-reg
 def : InstRW<[A57Write_8cyc_1L_1I], (instregex "VMOVDRR")>;
def A57WriteVQDMLAL_VecInt : SchedWriteVariant<[
   SchedVar<NoSchedPred,        [A57Write_5cyc_1W]>]>;
 def A57ReadVQDMLAL_VecInt : SchedReadVariant<[
   SchedVar<IsR1P0AndLaterPred, [SchedReadAdvance<2, [A57WriteVQDMLAL_VecInt]>]>,
   SchedVar<NoSchedPred,        [SchedReadAdvance<3, [A57WriteVQDMLAL_VecInt]>]>
 ]>;
 def : InstRW<[A57WriteVQDMLAL_VecInt, A57ReadVQDMLAL_VecInt],
   (instregex "VQDMLAL", "VQDMLSL")>;
 
+// Vector Saturating Rounding Doubling Multiply Accumulate/Subtract Long
+// Scheduling info from VQDMLAL/VQDMLSL
+def : InstRW<[A57WriteVQDMLAL_VecInt, A57ReadVQDMLAL_VecInt],
+  (instregex "VQRDMLAH", "VQRDMLSH")>;
+
 // ASIMD multiply long
 // 5cyc F0 for r0px, 4cyc F0 for r1p0 and later
 def A57WriteVMULL_VecInt : SchedWriteVariant<[
   SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>,
   SchedVar<NoSchedPred,        [A57Write_5cyc_1W]>]>;
 def : InstRW<[A57WriteVMULL_VecInt],
   (instregex "VMULL(s|u|p8|sls|slu)", "VQDMULL")>;
 

 // --- 3.15 ASIMD Floating-Point Instructions ---
 // ASIMD FP absolute value
 def : InstRW<[A57Write_3cyc_1V], (instregex "VABS(fd|fq|hd|hq)")>;
 
 // ASIMD FP arith
 def : InstRW<[A57Write_5cyc_1V], (instregex "VABD(fd|fq|hd|hq)",
   "VADD(fd|fq|hd|hq)", "VPADD(f|h)", "VSUB(fd|fq|hd|hq)")>;
 
+def : InstRW<[A57Write_5cyc_1V], (instregex "VCADD", "VCMLA")>;
+
 // ASIMD FP compare
 def : InstRW<[A57Write_5cyc_1V], (instregex "VAC(GE|GT|LE|LT)",
   "VC(EQ|GE|GT|LE)(fd|fq|hd|hq)")>;
 
 // ASIMD FP convert, integer
 def : InstRW<[A57Write_5cyc_1V], (instregex
   "VCVT(f2sd|f2ud|s2fd|u2fd|f2sq|f2uq|s2fq|u2fq|f2xsd|f2xud|xs2fd|xu2fd)",
   "VCVT(f2xsq|f2xuq|xs2fq|xu2fq)",

 def : InstRW<[A57Write_3cyc_1V], (instregex "VDUPLN(8|16|32)(d|q)")>;
 
 // ASIMD extract
 def : InstRW<[A57Write_3cyc_1V], (instregex "VEXT(d|q)(8|16|32|64)")>;
 
 // ASIMD move, immed
 def : InstRW<[A57Write_3cyc_1V], (instregex
   "VMOV(v8i8|v16i8|v4i16|v8i16|v2i32|v4i32|v1i64|v2i64|v2f32|v4f32)",
-  "VMOVQ0")>;
+  "VMOVD0", "VMOVQ0")>;
 
 // ASIMD move, narrowing
 def : InstRW<[A57Write_3cyc_1V], (instregex "VMOVN")>;
 
 // ASIMD move, saturating
 def : InstRW<[A57Write_4cyc_1X], (instregex "VQMOVN")>;
 
 // ASIMD reciprocal estimate