[AArch64] Enable clustering memory accesses to fixed stack objects

[foad]

Summary

r347747 added support for clustering mem ops with FI base operands
including support for fixed stack objects in shouldClusterFI, but
apparently this was never tested.

This patch fixes shouldClusterFI to work with scaled as well as
unscaled load/store instructions, and fixes the ordering of memory ops
in MemOpInfo::operator< to ensure that memory addresses always
increase, regardless of which direction the stack grows.

[merge_guards_bot]

Build result: pass - 60705 tests passed, 0 failed and 726 were skipped.

Log files: console-log.txt, CMakeCache.txt

[foad]

Tighten up an assertion.

[merge_guards_bot]

Build result: pass - 60705 tests passed, 0 failed and 726 were skipped.

Log files: console-log.txt, CMakeCache.txt

[thegameg]

LGTM. Thanks for fixing this.

The usage of TII->getMemScale is a bit weird since it makes it look like it's a non-static member function. No need to change it unless you think it would help.

[foad]

The usage of TII->getMemScale is a bit weird since it makes it look like it's a non-static member function. No need to change it unless you think it would help.

It's a bit weird, but much shorter than AArch64InstrInfo::getMemScale! I don't have strong opinions about which is better so I'll leave it as-is.

llvm/lib/CodeGen/MachineScheduler.cpp

bool operator<(const MemOpInfo &RHS) const {
                               TargetFrameLowering::StackGrowsDown;
         // Can't use tuple comparison here since we might need to use a
         // different order when the stack grows down.
         if (BaseOp->getIndex() != RHS.BaseOp->getIndex())
           return StackGrowsDown ? BaseOp->getIndex() > RHS.BaseOp->getIndex()
                                 : BaseOp->getIndex() < RHS.BaseOp->getIndex();
 
         if (Offset != RHS.Offset)
-          return StackGrowsDown ? Offset > RHS.Offset : Offset < RHS.Offset;
+          return Offset < RHS.Offset;
 
         return SU->NodeNum < RHS.SU->NodeNum;
       }
 
       llvm_unreachable("MemOpClusterMutation only supports register or frame "
                        "index bases.");
     }
   };

llvm/lib/Target/AArch64/AArch64InstrInfo.h

public:
   static bool isUnscaledLdSt(MachineInstr &MI) {
     return isUnscaledLdSt(MI.getOpcode());
   }
 
   /// Returns the unscaled load/store for the scaled load/store opcode,
   /// if there is a corresponding unscaled variant available.
   static Optional<unsigned> getUnscaledLdSt(unsigned Opc);
 
+  /// Scaling factor for (scaled or unscaled) load or store.
+  static int getMemScale(unsigned Opc);
+  static int getMemScale(const MachineInstr &MI) {
+    return getMemScale(MI.getOpcode());
+  }
+
 
   /// Returns the index for the immediate for a given instruction.
   static unsigned getLoadStoreImmIdx(unsigned Opc);
 
   /// Return true if pairing the given load or store may be paired with another.
   static bool isPairableLdStInst(const MachineInstr &MI);
 
   /// Return the opcode that set flags when possible.  The caller is

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

case AArch64::STGPi:
     MinOffset = -64;
     MaxOffset = 63;
     break;
   }
 
   return true;
 }
 
-static unsigned getOffsetStride(unsigned Opc) {
+// Scaling factor for unscaled load or store.
+int AArch64InstrInfo::getMemScale(unsigned Opc) {
   switch (Opc) {
   default:
-    return 0;
-  case AArch64::LDURQi:
-  case AArch64::STURQi:
-    return 16;
-  case AArch64::LDURXi:
-  case AArch64::LDURDi:
-  case AArch64::STURXi:
-  case AArch64::STURDi:
-    return 8;
-  case AArch64::LDURWi:
+    llvm_unreachable("Opcode has unknown scale!");
+  case AArch64::LDRBBui:
+  case AArch64::LDURBBi:
+  case AArch64::LDRSBWui:
+  case AArch64::LDURSBWi:
+  case AArch64::STRBBui:
+  case AArch64::STURBBi:
+    return 1;
+  case AArch64::LDRHHui:
+  case AArch64::LDURHHi:
+  case AArch64::LDRSHWui:
+  case AArch64::LDURSHWi:
+  case AArch64::STRHHui:
+  case AArch64::STURHHi:
+    return 2;
+  case AArch64::LDRSui:
   case AArch64::LDURSi:
+  case AArch64::LDRSWui:
   case AArch64::LDURSWi:
-  case AArch64::STURWi:
+  case AArch64::LDRWui:
+  case AArch64::LDURWi:
+  case AArch64::STRSui:
   case AArch64::STURSi:
+  case AArch64::STRWui:
+  case AArch64::STURWi:
+  case AArch64::LDPSi:
+  case AArch64::LDPSWi:
+  case AArch64::LDPWi:
+  case AArch64::STPSi:
+  case AArch64::STPWi:
     return 4;
+  case AArch64::LDRDui:
+  case AArch64::LDURDi:
+  case AArch64::LDRXui:
+  case AArch64::LDURXi:
+  case AArch64::STRDui:
+  case AArch64::STURDi:
+  case AArch64::STRXui:
+  case AArch64::STURXi:
+  case AArch64::LDPDi:
+  case AArch64::LDPXi:
+  case AArch64::STPDi:
+  case AArch64::STPXi:
+    return 8;
+  case AArch64::LDRQui:
+  case AArch64::LDURQi:
+  case AArch64::STRQui:
+  case AArch64::STURQi:
+  case AArch64::LDPQi:
+  case AArch64::STPQi:
+  case AArch64::STGOffset:
+  case AArch64::STZGOffset:
+  case AArch64::ST2GOffset:
+  case AArch64::STZ2GOffset:
+  case AArch64::STGPi:
+    return 16;
   }
 }
 
 // Scale the unscaled offsets.  Returns false if the unscaled offset can't be
 // scaled.
 static bool scaleOffset(unsigned Opc, int64_t &Offset) {
-  unsigned OffsetStride = getOffsetStride(Opc);
-  if (OffsetStride == 0)
-    return false;
+  int Scale = AArch64InstrInfo::getMemScale(Opc);
+
   // If the byte-offset isn't a multiple of the stride, we can't scale this
   // offset.
-  if (Offset % OffsetStride != 0)
+  if (Offset % Scale != 0)
     return false;
 
   // Convert the byte-offset used by unscaled into an "element" offset used
   // by the scaled pair load/store instructions.
-  Offset /= OffsetStride;
-  return true;
-}
-
-// Unscale the scaled offsets. Returns false if the scaled offset can't be
-// unscaled.
-static bool unscaleOffset(unsigned Opc, int64_t &Offset) {
-  unsigned OffsetStride = getOffsetStride(Opc);
-  if (OffsetStride == 0)
-    return false;
-
-  // Convert the "element" offset used by scaled pair load/store instructions
-  // into the byte-offset used by unscaled.
-  Offset *= OffsetStride;
+  Offset /= Scale;
   return true;
 }
 
 static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
   if (FirstOpc == SecondOpc)
     return true;
   // We can also pair sign-ext and zero-ext instructions.
   switch (FirstOpc) {
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
                             int64_t Offset1, unsigned Opcode1, int FI2,
                             int64_t Offset2, unsigned Opcode2) {
   // Accesses through fixed stack object frame indices may access a different
   // fixed stack slot. Check that the object offsets + offsets match.
   if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
     int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
     int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
     assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
-    // Get the byte-offset from the object offset.
-    if (!unscaleOffset(Opcode1, Offset1) || !unscaleOffset(Opcode2, Offset2))
+    // Convert to scaled object offsets.
+    int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
+    if (ObjectOffset1 % Scale1 != 0)
+      return false;
+    ObjectOffset1 /= Scale1;
+    int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
+    if (ObjectOffset2 % Scale2 != 0)
       return false;
+    ObjectOffset2 /= Scale2;
     ObjectOffset1 += Offset1;
     ObjectOffset2 += Offset2;
-    // Get the "element" index in the object.
-    if (!scaleOffset(Opcode1, ObjectOffset1) ||
-        !scaleOffset(Opcode2, ObjectOffset2))
-      return false;
     return ObjectOffset1 + 1 == ObjectOffset2;
   }
 
   return FI1 == FI2;
 }
 
 /// Detect opportunities for ldp/stp formation.
 ///
bool AArch64InstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1,
 
   // Pairwise instructions have a 7-bit signed offset field.
   if (Offset1 > 63 || Offset1 < -64)
     return false;
 
   // The caller should already have ordered First/SecondLdSt by offset.
   // Note: except for non-equal frame index bases
   if (BaseOp1.isFI()) {
-    assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 >= Offset2) &&
+    assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
            "Caller should have ordered offsets.");
 
     const MachineFrameInfo &MFI =
         FirstLdSt.getParent()->getParent()->getFrameInfo();
     return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
                            BaseOp2.getIndex(), Offset2, SecondOpc);
   }
 

llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp

static bool isTagStore(const MachineInstr &MI) {
   case AArch64::STGOffset:
   case AArch64::STZGOffset:
   case AArch64::ST2GOffset:
   case AArch64::STZ2GOffset:
     return true;
   }
 }
 
-// Scaling factor for unscaled load or store.
-static int getMemScale(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-  default:
-    llvm_unreachable("Opcode has unknown scale!");
-  case AArch64::LDRBBui:
-  case AArch64::LDURBBi:
-  case AArch64::LDRSBWui:
-  case AArch64::LDURSBWi:
-  case AArch64::STRBBui:
-  case AArch64::STURBBi:
-    return 1;
-  case AArch64::LDRHHui:
-  case AArch64::LDURHHi:
-  case AArch64::LDRSHWui:
-  case AArch64::LDURSHWi:
-  case AArch64::STRHHui:
-  case AArch64::STURHHi:
-    return 2;
-  case AArch64::LDRSui:
-  case AArch64::LDURSi:
-  case AArch64::LDRSWui:
-  case AArch64::LDURSWi:
-  case AArch64::LDRWui:
-  case AArch64::LDURWi:
-  case AArch64::STRSui:
-  case AArch64::STURSi:
-  case AArch64::STRWui:
-  case AArch64::STURWi:
-  case AArch64::LDPSi:
-  case AArch64::LDPSWi:
-  case AArch64::LDPWi:
-  case AArch64::STPSi:
-  case AArch64::STPWi:
-    return 4;
-  case AArch64::LDRDui:
-  case AArch64::LDURDi:
-  case AArch64::LDRXui:
-  case AArch64::LDURXi:
-  case AArch64::STRDui:
-  case AArch64::STURDi:
-  case AArch64::STRXui:
-  case AArch64::STURXi:
-  case AArch64::LDPDi:
-  case AArch64::LDPXi:
-  case AArch64::STPDi:
-  case AArch64::STPXi:
-    return 8;
-  case AArch64::LDRQui:
-  case AArch64::LDURQi:
-  case AArch64::STRQui:
-  case AArch64::STURQi:
-  case AArch64::LDPQi:
-  case AArch64::STPQi:
-  case AArch64::STGOffset:
-  case AArch64::STZGOffset:
-  case AArch64::ST2GOffset:
-  case AArch64::STZ2GOffset:
-  case AArch64::STGPi:
-    return 16;
-  }
-}
-
 static unsigned getMatchingNonSExtOpcode(unsigned Opc,
                                          bool *IsValidLdStrOpc = nullptr) {
   if (IsValidLdStrOpc)
     *IsValidLdStrOpc = true;
   switch (Opc) {
   default:
     if (IsValidLdStrOpc)
       *IsValidLdStrOpc = false;

 // Returns the scale and offset range of pre/post indexed variants of MI.
 static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
                                        int &MinOffset, int &MaxOffset) {
   bool IsPaired = isPairedLdSt(MI);
   bool IsTagStore = isTagStore(MI);
   // ST*G and all paired ldst have the same scale in pre/post-indexed variants
   // as in the "unsigned offset" variant.
   // All other pre/post indexed ldst instructions are unscaled.
-  Scale = (IsTagStore || IsPaired) ? getMemScale(MI) : 1;
+  Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
 
   if (IsPaired) {
     MinOffset = -64;
     MaxOffset = 63;
   } else {
     MinOffset = -256;
     MaxOffset = 255;
   }
static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI) {
   unsigned Idx = isPairedLdSt(MI) ? 3 : 2;
   return MI.getOperand(Idx);
 }
 
 static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
                                   MachineInstr &StoreInst,
                                   const AArch64InstrInfo *TII) {
   assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
-  int LoadSize = getMemScale(LoadInst);
-  int StoreSize = getMemScale(StoreInst);
+  int LoadSize = TII->getMemScale(LoadInst);
+  int StoreSize = TII->getMemScale(StoreInst);
   int UnscaledStOffset = TII->isUnscaledLdSt(StoreInst)
                              ? getLdStOffsetOp(StoreInst).getImm()
                              : getLdStOffsetOp(StoreInst).getImm() * StoreSize;
   int UnscaledLdOffset = TII->isUnscaledLdSt(LoadInst)
                              ? getLdStOffsetOp(LoadInst).getImm()
                              : getLdStOffsetOp(LoadInst).getImm() * LoadSize;
   return (UnscaledStOffset <= UnscaledLdOffset) &&
          (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
   // to skip one further. Either way we merge will invalidate the iterator,
   // and we don't need to scan the new instruction, as it's a pairwise
   // instruction, which we're not considering for further action anyway.
   if (NextI == MergeMI)
     ++NextI;
 
   unsigned Opc = I->getOpcode();
   bool IsScaled = !TII->isUnscaledLdSt(Opc);
-  int OffsetStride = IsScaled ? 1 : getMemScale(*I);
+  int OffsetStride = IsScaled ? 1 : TII->getMemScale(*I);
 
   bool MergeForward = Flags.getMergeForward();
   // Insert our new paired instruction after whichever of the paired
   // instructions MergeForward indicates.
   MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
   // Also based on MergeForward is from where we copy the base register operand
   // so we get the flags compatible with the input code.
   const MachineOperand &BaseRegOp =
AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
   // instruction, which we're not considering for further action anyway.
   if (NextI == Paired)
     ++NextI;
 
   int SExtIdx = Flags.getSExtIdx();
   unsigned Opc =
       SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode());
   bool IsUnscaled = TII->isUnscaledLdSt(Opc);
-  int OffsetStride = IsUnscaled ? getMemScale(*I) : 1;
+  int OffsetStride = IsUnscaled ? TII->getMemScale(*I) : 1;
 
   bool MergeForward = Flags.getMergeForward();
   // Insert our new paired instruction after whichever of the paired
   // instructions MergeForward indicates.
   MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
   // Also based on MergeForward is from where we copy the base register operand
   // so we get the flags compatible with the input code.
   const MachineOperand &BaseRegOp =
       MergeForward ? getLdStBaseOp(*Paired) : getLdStBaseOp(*I);
 
   int Offset = getLdStOffsetOp(*I).getImm();
   int PairedOffset = getLdStOffsetOp(*Paired).getImm();
   bool PairedIsUnscaled = TII->isUnscaledLdSt(Paired->getOpcode());
   if (IsUnscaled != PairedIsUnscaled) {
     // We're trying to pair instructions that differ in how they are scaled.  If
     // I is scaled then scale the offset of Paired accordingly.  Otherwise, do
     // the opposite (i.e., make Paired's offset unscaled).
-    int MemSize = getMemScale(*Paired);
+    int MemSize = TII->getMemScale(*Paired);
     if (PairedIsUnscaled) {
       // If the unscaled offset isn't a multiple of the MemSize, we can't
       // pair the operations together.
-      assert(!(PairedOffset % getMemScale(*Paired)) &&
+      assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
              "Offset should be a multiple of the stride!");
       PairedOffset /= MemSize;
     } else {
       PairedOffset *= MemSize;
     }
   }
 
   // Which register is Rt and which is Rt2 depends on the offset order.
   MachineInstr *RtMI, *Rt2MI;
   if (Offset == PairedOffset + OffsetStride) {
     RtMI = &*Paired;
     Rt2MI = &*I;
     // Here we swapped the assumption made for SExtIdx.
     // I.e., we turn ldp I, Paired into ldp Paired, I.
     // Update the index accordingly.
     if (SExtIdx != -1)
       SExtIdx = (SExtIdx + 1) % 2;
   } else {
     RtMI = &*I;
     Rt2MI = &*Paired;
   }
   int OffsetImm = getLdStOffsetOp(*RtMI).getImm();
   // Scale the immediate offset, if necessary.
   if (TII->isUnscaledLdSt(RtMI->getOpcode())) {
-    assert(!(OffsetImm % getMemScale(*RtMI)) &&
+    assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
            "Unscaled offset cannot be scaled.");
-    OffsetImm /= getMemScale(*RtMI);
+    OffsetImm /= TII->getMemScale(*RtMI);
   }
 
   // Construct the new instruction.
   MachineInstrBuilder MIB;
   DebugLoc DL = I->getDebugLoc();
   MachineBasicBlock *MBB = I->getParent();
   MachineOperand RegOp0 = getLdStRegOp(*RtMI);
   MachineOperand RegOp1 = getLdStRegOp(*Rt2MI);

 }
 
 MachineBasicBlock::iterator
 AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
                                           MachineBasicBlock::iterator StoreI) {
   MachineBasicBlock::iterator NextI = LoadI;
   ++NextI;
 
-  int LoadSize = getMemScale(*LoadI);
-  int StoreSize = getMemScale(*StoreI);
+  int LoadSize = TII->getMemScale(*LoadI);
+  int StoreSize = TII->getMemScale(*StoreI);
   Register LdRt = getLdStRegOp(*LoadI).getReg();
   const MachineOperand &StMO = getLdStRegOp(*StoreI);
   Register StRt = getLdStRegOp(*StoreI).getReg();
   bool IsStoreXReg = TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
 
   assert((IsStoreXReg ||
           TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
          "Unexpected RegClass");
AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
   MachineInstr &FirstMI = *I;
   ++MBBI;
 
   bool MayLoad = FirstMI.mayLoad();
   bool IsUnscaled = TII->isUnscaledLdSt(FirstMI);
   Register Reg = getLdStRegOp(FirstMI).getReg();
   Register BaseReg = getLdStBaseOp(FirstMI).getReg();
   int Offset = getLdStOffsetOp(FirstMI).getImm();
-  int OffsetStride = IsUnscaled ? getMemScale(FirstMI) : 1;
+  int OffsetStride = IsUnscaled ? TII->getMemScale(FirstMI) : 1;
   bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
 
   // Track which register units have been modified and used between the first
   // insn (inclusive) and the second insn.
   ModifiedRegUnits.clear();
   UsedRegUnits.clear();
 
   // Remember any instructions that read/write memory between FirstMI and MI.
if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
       // a relocation.
       Register MIBaseReg = getLdStBaseOp(MI).getReg();
       int MIOffset = getLdStOffsetOp(MI).getImm();
       bool MIIsUnscaled = TII->isUnscaledLdSt(MI);
       if (IsUnscaled != MIIsUnscaled) {
         // We're trying to pair instructions that differ in how they are scaled.
         // If FirstMI is scaled then scale the offset of MI accordingly.
         // Otherwise, do the opposite (i.e., make MI's offset unscaled).
-        int MemSize = getMemScale(MI);
+        int MemSize = TII->getMemScale(MI);
         if (MIIsUnscaled) {
           // If the unscaled offset isn't a multiple of the MemSize, we can't
           // pair the operations together: bail and keep looking.
           if (MIOffset % MemSize) {
             LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
                                               UsedRegUnits, TRI);
             MemInsns.push_back(&MI);
             continue;

 
 MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
     MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
   MachineBasicBlock::iterator E = I->getParent()->end();
   MachineInstr &MemMI = *I;
   MachineBasicBlock::iterator MBBI = I;
 
   Register BaseReg = getLdStBaseOp(MemMI).getReg();
-  int MIUnscaledOffset = getLdStOffsetOp(MemMI).getImm() * getMemScale(MemMI);
+  int MIUnscaledOffset = getLdStOffsetOp(MemMI).getImm() * TII->getMemScale(MemMI);
 
   // Scan forward looking for post-index opportunities.  Updating instructions
   // can't be formed if the memory instruction doesn't have the offset we're
   // looking for.
   if (MIUnscaledOffset != UnscaledOffset)
     return E;
 
   // If the base register overlaps a source/destination register, we can't
bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
   if (!TII->isCandidateToMergeOrPair(MI))
     return false;
 
   // Early exit if the offset is not possible to match. (6 bits of positive
   // range, plus allow an extra one in case we find a later insn that matches
   // with Offset-1)
   bool IsUnscaled = TII->isUnscaledLdSt(MI);
   int Offset = getLdStOffsetOp(MI).getImm();
-  int OffsetStride = IsUnscaled ? getMemScale(MI) : 1;
+  int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
   // Allow one more for offset.
   if (Offset > 0)
     Offset -= OffsetStride;
   if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
     return false;
 
   // Look ahead up to LdStLimit instructions for a pairable instruction.
   LdStPairFlags Flags;
if (Update != E) {
     // Merge the update into the ld/st.
     MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
     return true;
   }
 
   // The immediate in the load/store is scaled by the size of the memory
   // operation. The immediate in the add we're looking for,
   // however, is not, so adjust here.
-  int UnscaledOffset = getLdStOffsetOp(MI).getImm() * getMemScale(MI);
+  int UnscaledOffset = getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
 
   // Look forward to try to find a pre-index instruction. For example,
   // ldr x1, [x0, #64]
   // add x0, x0, #64
   //   merged into:
   // ldr x1, [x0, #64]!
   Update = findMatchingUpdateInsnForward(MBBI, UnscaledOffset, UpdateLimit);
   if (Update != E) {

llvm/test/CodeGen/AArch64/arm64-memset-inline.ll

; CHECK-NEXT:  bl something
   %cast = bitcast [20 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 20, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_26_stack() {
 ; CHECK-LABEL: bzero_26_stack:
-; CHECK:       stp xzr, xzr, [sp, #8]
-; CHECK-NEXT:  str xzr, [sp]
+; CHECK:       stp xzr, xzr, [sp]
 ; CHECK-NEXT:  strh wzr, [sp, #24]
+; CHECK-NEXT:  str xzr, [sp, #16]
 ; CHECK-NEXT:  bl something
   %buf = alloca [26 x i8], align 1
   %cast = bitcast [26 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 26, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
; CHECK-NEXT:  bl something
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 12, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_16_stack() {
 ; CHECK-LABEL: memset_16_stack:
 ; CHECK:       mov x8, #-6148914691236517206
+; CHECK-NEXT:  str x8, [sp, #-32]!
 ; CHECK-NEXT:  mov x0, sp
 ; CHECK-NEXT:  stp x8, x30, [sp, #8]
-; CHECK-NEXT:  str x8, [sp]
 ; CHECK-NEXT:  bl something
   %buf = alloca [16 x i8], align 1
   %cast = bitcast [16 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 16, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 

llvm/test/CodeGen/AArch64/cluster-frame-index.mir

 #RUN: llc -mtriple=aarch64-- -mcpu=cyclone -run-pass machine-scheduler -o - %s | FileCheck %s
-...
 ---
 name:            merge_stack
 # CHECK-LABEL: name: merge_stack
 tracksRegLiveness: true
 stack:
-  - { id: 0, size: 64, alignment: 8 }
+  - { id: 0, size: 16, alignment: 8 }
 body:             |
   bb.0:
     liveins: $w0, $w1
 
     %0:gpr32 = COPY $w0
     %1:gpr32 = COPY $w1
     undef %3.sub_32:gpr64 = ORRWrs $wzr, %0, 0
     STRXui %3, %stack.0, 0 :: (store 8)
     undef %5.sub_32:gpr64 = ORRWrs $wzr, %1, 0
     STRXui %5, %stack.0, 1 :: (store 8)
     RET_ReallyLR
 
     ; CHECK: COPY
     ; CHECK-NEXT: COPY
     ; CHECK-NEXT: ORRWrs
     ; CHECK-NEXT: ORRWrs
     ; CHECK-NEXT: STRXui
     ; CHECK-NEXT: STRXui
     ; CHECK-NEXT: RET
+...
+---
+name:            merge_fixedstack
+# CHECK-LABEL: name: merge_fixedstack
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, size: 16, alignment: 8, offset: -16 }
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    %0:gpr32 = COPY $w0
+    %1:gpr32 = COPY $w1
+    undef %3.sub_32:gpr64 = ORRWrs $wzr, %0, 0
+    STRXui %3, %fixed-stack.0, 0 :: (store 8)
+    undef %5.sub_32:gpr64 = ORRWrs $wzr, %1, 0
+    STRXui %5, %fixed-stack.0, 1 :: (store 8)
+    RET_ReallyLR
+
+    ; CHECK: COPY
+    ; CHECK-NEXT: COPY
+    ; CHECK-NEXT: ORRWrs
+    ; CHECK-NEXT: ORRWrs
+    ; CHECK-NEXT: STRXui
+    ; CHECK-NEXT: STRXui
+    ; CHECK-NEXT: RET
+...

llvm/test/CodeGen/AArch64/expand-select.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O3 %s -o - | FileCheck %s
 
 define void @foo(i32 %In1, <2 x i128> %In2, <2 x i128> %In3, <2 x i128> *%Out) {
 ; CHECK-LABEL: foo:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w9, w0, #0x1
+; CHECK-NEXT:    and w8, w0, #0x1
 ; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    ldp x10, x8, [sp, #8]
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    ldr x9, [sp]
+; CHECK-NEXT:    ldp x10, x9, [sp, #8]
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    ldr x8, [sp]
 ; CHECK-NEXT:    cmeq v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    fmov w11, s0
 ; CHECK-NEXT:    tst w11, #0x1
 ; CHECK-NEXT:    csel x11, x2, x6, ne
 ; CHECK-NEXT:    csel x12, x3, x7, ne
-; CHECK-NEXT:    csel x9, x4, x9, ne
+; CHECK-NEXT:    csel x8, x4, x8, ne
 ; CHECK-NEXT:    csel x10, x5, x10, ne
-; CHECK-NEXT:    stp x9, x10, [x8, #16]
-; CHECK-NEXT:    stp x11, x12, [x8]
+; CHECK-NEXT:    stp x8, x10, [x9, #16]
+; CHECK-NEXT:    stp x11, x12, [x9]
 ; CHECK-NEXT:    ret
   %cond = and i32 %In1, 1
   %cbool = icmp eq i32 %cond, 0
   %res = select i1 %cbool, <2 x i128> %In2, <2 x i128> %In3
   store <2 x i128> %res, <2 x i128> *%Out
 
   ret void
 }
 
 ; Check case when scalar size is not power of 2.
 define void @bar(i32 %In1, <2 x i96> %In2, <2 x i96> %In3, <2 x i96> *%Out) {
 ; CHECK-LABEL: bar:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w10, w0, #0x1
+; CHECK-NEXT:    and w9, w0, #0x1
 ; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    fmov s1, w10
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    cmeq v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ldp x11, x8, [sp, #8]
-; CHECK-NEXT:    ldr x9, [sp]
+; CHECK-NEXT:    ldr x10, [sp]
 ; CHECK-NEXT:    dup v1.4s, v0.s[0]
-; CHECK-NEXT:    mov x10, v1.d[1]
-; CHECK-NEXT:    lsr x10, x10, #32
-; CHECK-NEXT:    tst w10, #0x1
-; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    lsr x9, x9, #32
+; CHECK-NEXT:    tst w9, #0x1
+; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    csel x11, x5, x11, ne
-; CHECK-NEXT:    csel x9, x4, x9, ne
-; CHECK-NEXT:    tst w10, #0x1
-; CHECK-NEXT:    csel x10, x3, x7, ne
+; CHECK-NEXT:    csel x10, x4, x10, ne
+; CHECK-NEXT:    tst w9, #0x1
+; CHECK-NEXT:    csel x9, x3, x7, ne
 ; CHECK-NEXT:    csel x12, x2, x6, ne
-; CHECK-NEXT:    stur x9, [x8, #12]
+; CHECK-NEXT:    stur x10, [x8, #12]
 ; CHECK-NEXT:    str x12, [x8]
-; CHECK-NEXT:    str w10, [x8, #8]
+; CHECK-NEXT:    str w9, [x8, #8]
 ; CHECK-NEXT:    str w11, [x8, #20]
 ; CHECK-NEXT:    ret
   %cond = and i32 %In1, 1
   %cbool = icmp eq i32 %cond, 0
   %res = select i1 %cbool, <2 x i96> %In2, <2 x i96> %In3
   store <2 x i96> %res, <2 x i96> *%Out
 
   ret void
 }

llvm/test/CodeGen/AArch64/tailcall_misched_graph.ll

 ; SDAG:  STRXui %{{.*}}, %fixed-stack.0
 ; SDAG:  STRXui [[VRB]]{{[^,]*}}, %fixed-stack.1
 ; GISEL:  STRXui [[VRB]]{{[^,]*}}, %fixed-stack.1
 ; GISEL:  STRXui %{{.*}}, %fixed-stack.0
 
 ; Make sure that there is an dependence edge between fi#-2 and fi#-4.
 ; Without this edge the scheduler would be free to move the store accross the load.
 
-; COMMON: SU({{.*}}):   [[VRB]]:gpr64 = LDRXui %fixed-stack.2
-; COMMON-NOT: SU
+; COMMON: {{^SU(.*)}}:   [[VRB]]:gpr64 = LDRXui %fixed-stack.2
+; COMMON-NOT: {{^SU(.*)}}:
 ; COMMON:  Successors:
 ; COMMON:   SU([[DEPSTOREB:.*]]): Ord  Latency=0
 ; COMMON:   SU([[DEPSTOREA:.*]]): Ord  Latency=0
 
 ; GlobalISel outputs DEPSTOREB before DEPSTOREA, but the dependency relationship
 ; still holds.
 ; SDAG: SU([[DEPSTOREA]]):   STRXui %{{.*}}, %fixed-stack.0
 ; SDAG: SU([[DEPSTOREB]]):   STRXui %{{.*}}, %fixed-stack.1
 
 ; GISEL: SU([[DEPSTOREB]]):   STRXui %{{.*}}, %fixed-stack.0
 ; GISEL: SU([[DEPSTOREA]]):   STRXui %{{.*}}, %fixed-stack.1