Index: lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
===================================================================
--- lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -205,6 +205,66 @@
   }
 }
 
+// Given a scaled load/store opcode, return the equivalent unscaled opcode.
+static unsigned getUnscaledFromScaled(unsigned Opc) {
+  switch (Opc) {
+  default:
+    llvm_unreachable("Unexpected scaled opcode!");
+  case AArch64::STRSui:
+    return AArch64::STURSi;
+  case AArch64::STRDui:
+    return AArch64::STURDi;
+  case AArch64::STRQui:
+    return AArch64::STURQi;
+  case AArch64::STRWui:
+    return AArch64::STURWi;
+  case AArch64::STRXui:
+    return AArch64::STURXi;
+  case AArch64::LDRSui:
+    return AArch64::LDURSi;
+  case AArch64::LDRDui:
+    return AArch64::LDURDi;
+  case AArch64::LDRQui:
+    return AArch64::LDURQi;
+  case AArch64::LDRWui:
+    return AArch64::LDURWi;
+  case AArch64::LDRXui:
+    return AArch64::LDURXi;
+  case AArch64::LDRSWui:
+    return AArch64::LDURSWi;
+  }
+}
+
+// Given an unscaled load/store opcode, return the equivalent scaled opcode.
+static unsigned getScaledFromUnscaled(unsigned Opc) {
+  switch (Opc) {
+  default:
+    llvm_unreachable("Unexpected unscaled opcode!");
+  case AArch64::STURSi:
+    return AArch64::STRSui;
+  case AArch64::STURDi:
+    return AArch64::STRDui;
+  case AArch64::STURQi:
+    return AArch64::STRQui;
+  case AArch64::STURWi:
+    return AArch64::STRWui;
+  case AArch64::STURXi:
+    return AArch64::STRXui;
+  case AArch64::LDURSi:
+    return AArch64::LDRSui;
+  case AArch64::LDURDi:
+    return AArch64::LDRDui;
+  case AArch64::LDURQi:
+    return AArch64::LDRQui;
+  case AArch64::LDURWi:
+    return AArch64::LDRWui;
+  case AArch64::LDURXi:
+    return AArch64::LDRXui;
+  case AArch64::LDURSWi:
+    return AArch64::LDRSWui;
+  }
+}
+
 static unsigned getMatchingNonSExtOpcode(unsigned Opc,
                                          bool *IsValidLdStrOpc = nullptr) {
   if (IsValidLdStrOpc)
@@ -382,10 +442,22 @@
   const MachineOperand &BaseRegOp =
       MergeForward ? getLdStBaseOp(Paired) : getLdStBaseOp(I);
 
+  int Offset = getLdStOffsetOp(I).getImm();
+  int PairedOffset = getLdStOffsetOp(Paired).getImm();
+  bool PairedIsUnscaled = isUnscaledLdSt(Paired->getOpcode());
+
+  // We're trying to pair instructions that differ in how they are scaled.
+  // If FirstMI is scaled then scale the offset of MI accordingly.
+  // Otherwise, do the opposite (i.e., make Paired's offset unscaled).
+  if (IsUnscaled != PairedIsUnscaled) {
+    int MemSize = getMemSize(Paired);
+    PairedOffset =
+        PairedIsUnscaled ? PairedOffset / MemSize : PairedOffset * MemSize;
+  }
+
   // Which register is Rt and which is Rt2 depends on the offset order.
   MachineInstr *RtMI, *Rt2MI;
-  if (getLdStOffsetOp(I).getImm() ==
-      getLdStOffsetOp(Paired).getImm() + OffsetStride) {
+  if (Offset == PairedOffset + OffsetStride) {
     RtMI = Paired;
     Rt2MI = I;
     // Here we swapped the assumption made for SExtIdx.
@@ -397,10 +469,10 @@
     RtMI = I;
     Rt2MI = Paired;
   }
-  // Handle Unscaled
+  // Scale the immediate offset, if necessary.
   int OffsetImm = getLdStOffsetOp(RtMI).getImm();
-  if (IsUnscaled && EnableAArch64UnscaledMemOp)
-    OffsetImm /= OffsetStride;
+  if (isUnscaledLdSt(RtMI->getOpcode()) && EnableAArch64UnscaledMemOp)
+    OffsetImm /= getMemSize(RtMI);
 
   // Construct the new instruction.
   MachineInstrBuilder MIB = BuildMI(*I->getParent(), InsertionPoint,
@@ -531,6 +603,36 @@
   return false;
 }
 
+static bool canMergeOpc(unsigned Opc, unsigned PairOpc, LdStPairFlags &Flags) {
+  bool CanMergeOpc = Opc == PairOpc;
+  // Opcodes match nothing more to check.
+  if (CanMergeOpc)
+    return true;
+
+  // Try to match a signed-extended load/store with a zero-extended load/store.
+  Flags.setSExtIdx(-1);
+  bool IsValidLdStrOpc;
+  unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc);
+  assert(IsValidLdStrOpc &&
+         "Given Opc should be a Load or Store with an immediate");
+  // Opc will be the first instruction in the pair.
+  CanMergeOpc = NonSExtOpc == getMatchingNonSExtOpcode(PairOpc);
+  if (CanMergeOpc) {
+    Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0);
+    return true;
+  }
+
+  // Try to match an unscaled load/store with a scaled load/store.
+  bool IsUnscaled = isUnscaledLdSt(Opc);
+  unsigned NewOpc =
+      IsUnscaled ? getScaledFromUnscaled(Opc) : getUnscaledFromScaled(Opc);
+  CanMergeOpc = NewOpc == PairOpc;
+  if (CanMergeOpc)
+    return true;
+
+  return false;
+}
+
 /// findMatchingInsn - Scan the instructions looking for a load/store that can
 /// be combined with the current instruction into a load/store pair.
 MachineBasicBlock::iterator
@@ -581,19 +683,8 @@
     // Now that we know this is a real instruction, count it.
     ++Count;
 
-    bool CanMergeOpc = Opc == MI->getOpcode();
-    Flags.setSExtIdx(-1);
-    if (!CanMergeOpc) {
-      bool IsValidLdStrOpc;
-      unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc);
-      assert(IsValidLdStrOpc &&
-             "Given Opc should be a Load or Store with an immediate");
-      // Opc will be the first instruction in the pair.
-      Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0);
-      CanMergeOpc = NonSExtOpc == getMatchingNonSExtOpcode(MI->getOpcode());
-    }
-
-    if (CanMergeOpc && getLdStOffsetOp(MI).isImm()) {
+    if (canMergeOpc(Opc, MI->getOpcode(), Flags) &&
+        getLdStOffsetOp(MI).isImm()) {
       assert(MI->mayLoadOrStore() && "Expected memory operation.");
       // If we've found another instruction with the same opcode, check to see
       // if the base and offset are compatible with our starting instruction.
@@ -607,6 +698,16 @@
       // final offset must be in range.
       unsigned MIBaseReg = getLdStBaseOp(MI).getReg();
       int MIOffset = getLdStOffsetOp(MI).getImm();
+
+      // We're trying to pair instructions that differ in how they are scaled.
+      // If FirstMI is scaled then scale the offset of MI accordingly.
+      // Otherwise, do the opposite (i.e., make MI's offset unscaled).
+      bool MIIsUnscaled = isUnscaledLdSt(MI);
+      if (IsUnscaled != MIIsUnscaled) {
+        int MemSize = getMemSize(MI);
+        MIOffset = MIIsUnscaled ? MIOffset / MemSize : MIOffset * MemSize;
+      }
+
       if (BaseReg == MIBaseReg && ((Offset == MIOffset + OffsetStride) ||
                                    (Offset + OffsetStride == MIOffset))) {
         int MinOffset = Offset < MIOffset ? Offset : MIOffset;
@@ -617,8 +718,7 @@
           return E;
         // If the resultant immediate offset of merging these instructions
         // is out of range for a pairwise instruction, bail and keep looking.
-        bool MIIsUnscaled = isUnscaledLdSt(MI);
-        if (!inBoundsForPair(MIIsUnscaled, MinOffset, OffsetStride)) {
+        if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
           trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
           MemInsns.push_back(MI);
           continue;