[AArch64][MachineScheduler] Fix operand scheduling for pre/post-increment loads

[evgeny777]

Summary

WriteAdr describes scheduling for first operand of load:

$x0, $x1, $x2 = LDPXpost $x0, 2          # x0 is first

[dmgreen]

STRWpost should be $x0 = STRWpost $xzr, $x0. The concept of scheduling a store can be a little strange, but the address update should come as the first operand.

I agree loads look backwards.

[evgeny777]

@dmgreen Thx, I've updated the diff.

[dmgreen]

Thanks. LGTM

llvm/lib/Target/AArch64/AArch64InstrFormats.td

 let hasSideEffects = 0 in {
 let mayStore = 0, mayLoad = 1 in
 class LoadPreIdx<bits<2> sz, bit V, bits<2> opc, RegisterOperand regtype,
              string asm>
     : BaseLoadStorePreIdx<sz, V, opc,
                      (outs GPR64sp:$wback, regtype:$Rt),
                      (ins GPR64sp:$Rn, simm9:$offset), asm,
                      "$Rn = $wback,@earlyclobber $wback", []>,
-      Sched<[WriteLD, WriteAdr]>;
+      Sched<[WriteAdr, WriteLD]>;
 
 let mayStore = 1, mayLoad = 0 in
 class StorePreIdx<bits<2> sz, bit V, bits<2> opc, RegisterOperand regtype,
                   string asm, SDPatternOperator storeop, ValueType Ty>
     : BaseLoadStorePreIdx<sz, V, opc,
                       (outs GPR64sp:$wback),
                       (ins regtype:$Rt, GPR64sp:$Rn, simm9:$offset),
                       asm, "$Rn = $wback,@earlyclobber $wback",
       [(set GPR64sp:$wback,
             (storeop (Ty regtype:$Rt), GPR64sp:$Rn, simm9:$offset))]>,
-      Sched<[WriteAdr, WriteST]>;
+      Sched<[WriteST, WriteAdr]>;
 } // hasSideEffects = 0
 
 //---
 // Load/store post-indexed
 //---
 
 class BaseLoadStorePostIdx<bits<2> sz, bit V, bits<2> opc, dag oops, dag iops,
                           string asm, string cstr, list<dag> pat>

 let hasSideEffects = 0 in {
 let mayStore = 0, mayLoad = 1 in
 class LoadPostIdx<bits<2> sz, bit V, bits<2> opc, RegisterOperand regtype,
              string asm>
     : BaseLoadStorePostIdx<sz, V, opc,
                       (outs GPR64sp:$wback, regtype:$Rt),
                       (ins GPR64sp:$Rn, simm9:$offset),
                       asm, "$Rn = $wback,@earlyclobber $wback", []>,
-      Sched<[WriteLD, WriteAdr]>;
+      Sched<[WriteAdr, WriteLD]>;
 
 let mayStore = 1, mayLoad = 0 in
 class StorePostIdx<bits<2> sz, bit V, bits<2> opc, RegisterOperand regtype,
                    string asm, SDPatternOperator storeop, ValueType Ty>
     : BaseLoadStorePostIdx<sz, V, opc,
                       (outs GPR64sp:$wback),
                       (ins regtype:$Rt, GPR64sp:$Rn, simm9:$offset),
                        asm, "$Rn = $wback,@earlyclobber $wback",
       [(set GPR64sp:$wback,
             (storeop (Ty regtype:$Rt), GPR64sp:$Rn, simm9:$offset))]>,
-    Sched<[WriteAdr, WriteST]>;
+    Sched<[WriteST, WriteAdr]>;
 } // hasSideEffects = 0
 
 
 //---
 // Load/store pair
 //---
 
 // (indexed, offset)

 
 let hasSideEffects = 0 in {
 let mayStore = 0, mayLoad = 1 in
 class LoadPairPreIdx<bits<2> opc, bit V, RegisterOperand regtype,
                      Operand indextype, string asm>
     : BaseLoadStorePairPreIdx<opc, V, 1,
                               (outs GPR64sp:$wback, regtype:$Rt, regtype:$Rt2),
                               (ins GPR64sp:$Rn, indextype:$offset), asm>,
-      Sched<[WriteLD, WriteLDHi, WriteAdr]>;
+      Sched<[WriteAdr, WriteLD, WriteLDHi]>;
 
 let mayStore = 1, mayLoad = 0 in
 class StorePairPreIdx<bits<2> opc, bit V, RegisterOperand regtype,
                       Operand indextype, string asm>
     : BaseLoadStorePairPreIdx<opc, V, 0, (outs GPR64sp:$wback),
                              (ins regtype:$Rt, regtype:$Rt2,
                                   GPR64sp:$Rn, indextype:$offset),
                              asm>,
-      Sched<[WriteAdr, WriteSTP]>;
+      Sched<[WriteSTP, WriteAdr]>;
 } // hasSideEffects = 0
 
 // (post-indexed)
 
 class BaseLoadStorePairPostIdx<bits<2> opc, bit V, bit L, dag oops, dag iops,
                               string asm>
     : I<oops, iops, asm, "\t$Rt, $Rt2, [$Rn], $offset", "$Rn = $wback,@earlyclobber $wback", []> {
   bits<5> Rt;

 
 let hasSideEffects = 0 in {
 let mayStore = 0, mayLoad = 1 in
 class LoadPairPostIdx<bits<2> opc, bit V, RegisterOperand regtype,
                       Operand idxtype, string asm>
     : BaseLoadStorePairPostIdx<opc, V, 1,
                               (outs GPR64sp:$wback, regtype:$Rt, regtype:$Rt2),
                               (ins GPR64sp:$Rn, idxtype:$offset), asm>,
-      Sched<[WriteLD, WriteLDHi, WriteAdr]>;
+      Sched<[WriteAdr, WriteLD, WriteLDHi]>;
 
 let mayStore = 1, mayLoad = 0 in
 class StorePairPostIdx<bits<2> opc, bit V, RegisterOperand regtype,
                        Operand idxtype, string asm>
     : BaseLoadStorePairPostIdx<opc, V, 0, (outs GPR64sp:$wback),
                              (ins regtype:$Rt, regtype:$Rt2,
                                   GPR64sp:$Rn, idxtype:$offset),
                              asm>,
-      Sched<[WriteAdr, WriteSTP]>;
+      Sched<[WriteSTP, WriteAdr]>;
 } // hasSideEffects = 0
 
 //  (no-allocate)
 
 class BaseLoadStorePairNoAlloc<bits<2> opc, bit V, bit L, dag oops, dag iops,
                               string asm>
     : I<oops, iops, asm, "\t$Rt, $Rt2, [$Rn, $offset]", "", []> {
   bits<5> Rt;

llvm/test/CodeGen/AArch64/merge-store-dependency.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mcpu cortex-a53 -mtriple=aarch64-eabi | FileCheck %s --check-prefix=A53
 
 ; PR26827 - Merge stores causes wrong dependency.
 %struct1 = type { %struct1*, %struct1*, i32, i32, i16, i16, void (i32, i32, i8*)*, i8* }
 @gv0 = internal unnamed_addr global i32 0, align 4
 @gv1 = internal unnamed_addr global %struct1** null, align 8
 
 define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg)  {
 ;CHECK-LABEL: test
 ; A53-LABEL: test:
 ; A53:       // %bb.0: // %entry
 ; A53-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; A53-NEXT:    .cfi_def_cfa_offset 16
 ; A53-NEXT:    .cfi_offset w19, -8
 ; A53-NEXT:    .cfi_offset w30, -16
-; A53-NEXT:    movi v0.2d, #0000000000000000
-; A53-NEXT:    mov x8, x0
-; A53-NEXT:    mov x19, x8
-; A53-NEXT:    mov w0, w1
-; A53-NEXT:    mov w9, #256
-; A53-NEXT:    str q0, [x19, #16]!
-; A53-NEXT:    str w1, [x19]
-; A53-NEXT:    mov w1, #4
-; A53-NEXT:    stp x2, x3, [x8, #32]
-; A53-NEXT:    mov x2, x8
-; A53-NEXT:    str q0, [x8]
-; A53-NEXT:    strh w9, [x8, #24]
-; A53-NEXT:    str wzr, [x8, #20]
+
+; A53-NEXT:    movi v0.2d, #0000000000000000
+; A53-NEXT:    mov  x8, x0
+; A53-NEXT:    mov  x19, x8
+; A53-NEXT:    mov  w0, w1
+; A53-NEXT:    mov  w9, #256
+; A53-NEXT:    str  q0, [x19, #16]!
+; A53-NEXT:    str  q0, [x8]
+; A53-NEXT:    strh w9, [x8, #24]
+; A53-NEXT:    str  wzr, [x8, #20]
+; A53-NEXT:    str  w1, [x19]
+; A53-NEXT:    mov  w1, #4
+; A53-NEXT:    stp  x2, x3, [x8, #32]
+; A53-NEXT:    mov  x2, x8
 ; A53-NEXT:    bl fcntl
 ; A53-NEXT:    adrp x9, gv0
 ; A53-NEXT:    add x9, x9, :lo12:gv0
 ; A53-NEXT:    cmp x19, x9
 ; A53-NEXT:    b.eq .LBB0_4
 ; A53-NEXT:  // %bb.1:
 ; A53-NEXT:    ldr w8, [x19]
 ; A53-NEXT:    ldr w9, [x9]

llvm/test/tools/llvm-mca/AArch64/Exynos/load.s

 ldnp  w0, w1, [sp, #8]
 ldp x0, x1, [sp], #16
 ldpsw  x0, x1, [sp, #8]!
 
 1:
 
 # ALL:      Iterations:        100
 # ALL-NEXT: Instructions:      1200
-# ALL-NEXT: Total Cycles:      1904
+# ALL-NEXT: Total Cycles:      1304
 
 # M3-NEXT:  Total uOps:        1600
 # M4-NEXT:  Total uOps:        1400
 # M5-NEXT:  Total uOps:        1400
 
 # ALL:      Dispatch Width:    6
 
-# M3-NEXT:  uOps Per Cycle:    0.84
-# M4-NEXT:  uOps Per Cycle:    0.74
-# M5-NEXT:  uOps Per Cycle:    0.74
+# M3-NEXT:  uOps Per Cycle:    1.23
+# M4-NEXT:  uOps Per Cycle:    1.07
+# M5-NEXT:  uOps Per Cycle:    1.07
 
-# ALL-NEXT: IPC:               0.63
+# ALL-NEXT: IPC:               0.92
 # ALL-NEXT: Block RThroughput: 6.0
 
 # ALL:      Instruction Info:
 # ALL-NEXT: [1]: #uOps
 # ALL-NEXT: [2]: Latency
 # ALL-NEXT: [3]: RThroughput
 # ALL-NEXT: [4]: MayLoad
 # ALL-NEXT: [5]: MayStore