[AArch64] Add MMOs to callee-save load/store instructions.

[gberry]

Summary

Without MMOs, the callee-save load/store instructions were treated as
volatile by the MI post-RA scheduler and AArch64LoadStoreOptimizer.

[rengolin]

No tests?

[gberry]

The only test I could think of other than checking the post-RA scheduling order (which seems too fragile), would be to check the MI dump after PrologEpilogInserter for MMOs. I can add such a test if you think it is useful.

[gberry]

Add potential test case.

[mcrosier]

LGTM. The test seems reasonable and I can't come up with a better solution.

[gberry]

Updated test case and make dependant on D18093

llvm/trunk/lib/Target/AArch64/AArch64FrameLowering.cpp

for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
     if (RPI.isPaired()) {
       MBB.addLiveIn(Reg1);
       MBB.addLiveIn(Reg2);
       MIB.addReg(Reg2, getPrologueDeath(MF, Reg2))
         .addReg(Reg1, getPrologueDeath(MF, Reg1))
         .addReg(AArch64::SP)
         .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit
         .setMIFlag(MachineInstr::FrameSetup);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
+          MachineMemOperand::MOStore, 8, 8));
     } else {
       MBB.addLiveIn(Reg1);
       MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
         .addReg(AArch64::SP)
         .addImm(BumpSP ? Offset * 8 : Offset) // pre-inc version is unscaled
         .setMIFlag(MachineInstr::FrameSetup);
     }
+    MIB.addMemOperand(MF.getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
+        MachineMemOperand::MOStore, 8, 8));
   }
   return true;
 }
 
 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
     const std::vector<CalleeSavedInfo> &CSI,
     const TargetRegisterInfo *TRI) const {
DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1);
             dbgs() << ", " << RPI.FrameIdx+1;
           dbgs() << ")\n");
 
     const int Offset = RPI.Offset;
     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
     if (BumpSP)
       MIB.addReg(AArch64::SP, RegState::Define);
 
-    if (RPI.isPaired())
+    if (RPI.isPaired()) {
       MIB.addReg(Reg2, getDefRegState(true))
         .addReg(Reg1, getDefRegState(true))
         .addReg(AArch64::SP)
         .addImm(Offset) // [sp], #offset * 8  or [sp, #offset * 8]
                         // where the factor * 8 is implicit
         .setMIFlag(MachineInstr::FrameDestroy);
-    else
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
+          MachineMemOperand::MOLoad, 8, 8));
+    } else {
       MIB.addReg(Reg1, getDefRegState(true))
         .addReg(AArch64::SP)
         .addImm(BumpSP ? Offset * 8 : Offset) // post-dec version is unscaled
         .setMIFlag(MachineInstr::FrameDestroy);
     }
+    MIB.addMemOperand(MF.getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
+        MachineMemOperand::MOLoad, 8, 8));
+  }
   return true;
 }
 
 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
                                                 BitVector &SavedRegs,
                                                 RegScavenger *RS) const {
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.

llvm/trunk/test/CodeGen/AArch64/arm64-csldst-mmo.ll

+; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=cortex-a57 -enable-misched=0 -debug-only=misched -o - 2>&1 > /dev/null | FileCheck %s
+
+@G = external global [0 x i32], align 4
+
+; Check that MMOs are added to epilogue calle-save restore loads so
+; that the store to G is not considered dependant on the callee-save
+; loads.
+;
+; CHECK: Before post-MI-sched:
+; CHECK-LABEL: # Machine code for function test1:
+; CHECK: SU(2):   STRWui %WZR
+; CHECK: SU(3):   %X21<def>, %X20<def> = LDPXi %SP
+; CHECK:  Predecessors:
+; CHECK-NEXT:   out SU(0)
+; CHECK-NEXT:   out SU(0)
+; CHECK-NEXT:   ch  SU(0)
+; CHECK-NEXT:  Successors:
+define void @test1() {
+entry:
+  tail call void asm sideeffect "nop", "~{x20},~{x21},~{x22},~{x23}"() nounwind
+  store i32 0, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @G, i64 0, i64 0), align 4
+  ret void
+}