[ARM] Distribute post-inc for Thumb2 sign/zero extending loads/stores

[dmgreen]

Summary

This adds sign/zero extending scalar loads/stores to the MVE instructions added in D77813, allowing us to pick up more post-inc situations. These are comparatively simple, compared to LDR/STR (which may be better turned into an LDRD/LDM), but still require some additions over MVE instructions. Because there are i12 and i8 variants of the offset loads/stores dealing with different signs, we may need to convert an i12 address to a i8 negative instruction.

[samparker]

LGTM. Have you tested whether this approach is faster than doing the pre-indexing LSR method? I certainly like the idea of this transform instead of being beholden to the filtering gods.

[dmgreen]

Now includes a quick codesize metric, to try and detect cases where a t2LDRi12 can be shrunk to tLDRi.

[dmgreen]

I think we are still at the behests of LSR's cost modeling I'm afraid. This can just do slightly better at fixing up the results afterwards, it can slightly improve things in case ISel comes up with something unoptimal.

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

inline bool isLegalAddressImm(unsigned Opcode, int Imm,
   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
   switch (AddrMode) {
   case ARMII::AddrModeT2_i7:
     return std::abs(Imm) < (((1 << 7) * 1) - 1);
   case ARMII::AddrModeT2_i7s2:
     return std::abs(Imm) < (((1 << 7) * 2) - 1) && Imm % 2 == 0;
   case ARMII::AddrModeT2_i7s4:
     return std::abs(Imm) < (((1 << 7) * 4) - 1) && Imm % 4 == 0;
+  case ARMII::AddrModeT2_i8:
+    return std::abs(Imm) < (((1 << 8) * 1) - 1);
+  case ARMII::AddrModeT2_i12:
+    return Imm >= 0 && Imm < (((1 << 12) * 1) - 1);
   default:
     llvm_unreachable("Unhandled Addressing mode");
   }
 }
 
 // Return true if the given intrinsic is a gather or scatter
 inline bool isGatherScatter(IntrinsicInst *IntInst) {
   if (IntInst == nullptr)

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

case ARM::VLDRD:
     return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
   case ARM::VSTRS:
     return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
   case ARM::VSTRD:
     return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
   case ARM::t2LDRi8:
   case ARM::t2LDRi12:
     return ARM::t2LDR_POST;
+  case ARM::t2LDRBi8:
+  case ARM::t2LDRBi12:
+    return ARM::t2LDRB_POST;
+  case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBi12:
+    return ARM::t2LDRSB_POST;
+  case ARM::t2LDRHi8:
+  case ARM::t2LDRHi12:
+    return ARM::t2LDRH_POST;
+  case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHi12:
+    return ARM::t2LDRSH_POST;
   case ARM::t2STRi8:
   case ARM::t2STRi12:
     return ARM::t2STR_POST;
+  case ARM::t2STRBi8:
+  case ARM::t2STRBi12:
+    return ARM::t2STRB_POST;
+  case ARM::t2STRHi8:
+  case ARM::t2STRHi12:
+    return ARM::t2STRH_POST;
 
   case ARM::MVE_VLDRBS16:
     return ARM::MVE_VLDRBS16_post;
   case ARM::MVE_VLDRBS32:
     return ARM::MVE_VLDRBS32_post;
   case ARM::MVE_VLDRBU16:
     return ARM::MVE_VLDRBU16_post;
   case ARM::MVE_VLDRBU32:
static int getBaseOperandIndex(MachineInstr &MI) {
   case ARM::MVE_VLDRHU16:
   case ARM::MVE_VLDRWU32:
   case ARM::MVE_VSTRB16:
   case ARM::MVE_VSTRB32:
   case ARM::MVE_VSTRH32:
   case ARM::MVE_VSTRBU8:
   case ARM::MVE_VSTRHU16:
   case ARM::MVE_VSTRWU32:
+  case ARM::t2LDRHi8:
+  case ARM::t2LDRHi12:
+  case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHi12:
+  case ARM::t2LDRBi8:
+  case ARM::t2LDRBi12:
+  case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBi12:
+  case ARM::t2STRBi8:
+  case ARM::t2STRBi12:
+  case ARM::t2STRHi8:
+  case ARM::t2STRHi12:
     return 1;
   }
   return -1;
 }
 
+// Given a memory access Opcode, check that the give Imm would be a valid Offset
+// for this instruction (same as isLegalAddressImm), Or if the instruction
+// could be easily converted to one where that was valid. For example converting
+// t2LDRi12 to t2LDRi8 for negative offsets. Works in conjunction with
+// AdjustBaseAndOffset below.
+static bool isLegalOrConvertableAddressImm(unsigned Opcode, int Imm,
+                                           const TargetInstrInfo *TII,
+                                           int &CodesizeEstimate) {
+  if (isLegalAddressImm(Opcode, Imm, TII))
+    return true;
+
+  // We can convert AddrModeT2_i12 to AddrModeT2_i8.
+  const MCInstrDesc &Desc = TII->get(Opcode);
+  unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
+  switch (AddrMode) {
+  case ARMII::AddrModeT2_i12:
+    CodesizeEstimate += 1;
+    return std::abs(Imm) < (((1 << 8) * 1) - 1);
+  }
+  return false;
+}
+
+// Given an MI adjust its address BaseReg to use NewBaseReg and address offset
+// by -Offset. This can either happen in-place or be a replacement as MI is
+// converted to another instruction type.
+static void AdjustBaseAndOffset(MachineInstr *MI, Register NewBaseReg,
+                                int Offset, const TargetInstrInfo *TII) {
+  unsigned BaseOp = getBaseOperandIndex(*MI);
+  MI->getOperand(BaseOp).setReg(NewBaseReg);
+  int OldOffset = MI->getOperand(BaseOp + 1).getImm();
+  if (isLegalAddressImm(MI->getOpcode(), OldOffset - Offset, TII))
+    MI->getOperand(BaseOp + 1).setImm(OldOffset - Offset);
+  else {
+    unsigned ConvOpcode;
+    switch (MI->getOpcode()) {
+    case ARM::t2LDRHi12:
+      ConvOpcode = ARM::t2LDRHi8;
+      break;
+    case ARM::t2LDRSHi12:
+      ConvOpcode = ARM::t2LDRSHi8;
+      break;
+    case ARM::t2LDRBi12:
+      ConvOpcode = ARM::t2LDRBi8;
+      break;
+    case ARM::t2LDRSBi12:
+      ConvOpcode = ARM::t2LDRSBi8;
+      break;
+    case ARM::t2STRHi12:
+      ConvOpcode = ARM::t2STRHi8;
+      break;
+    case ARM::t2STRBi12:
+      ConvOpcode = ARM::t2STRBi8;
+      break;
+    default:
+      llvm_unreachable("Unhandled convertable opcode");
+    }
+    assert(isLegalAddressImm(ConvOpcode, OldOffset - Offset, TII) &&
+           "Illegal Address Immediate after convert!");
+
+    const MCInstrDesc &MCID = TII->get(ConvOpcode);
+    BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+        .add(MI->getOperand(0))
+        .add(MI->getOperand(1))
+        .addImm(OldOffset - Offset)
+        .add(MI->getOperand(3))
+        .add(MI->getOperand(4))
+        .cloneMemRefs(*MI);
+    MI->eraseFromParent();
+  }
+}
+
 static MachineInstr *createPostIncLoadStore(MachineInstr *MI, int Offset,
                                             Register NewReg,
                                             const TargetInstrInfo *TII,
                                             const TargetRegisterInfo *TRI) {
   MachineFunction *MF = MI->getMF();
   MachineRegisterInfo &MRI = MF->getRegInfo();
 
   unsigned NewOpcode = getPostIndexedLoadStoreOpcode(
       MI->getOpcode(), Offset > 0 ? ARM_AM::add : ARM_AM::sub);
 
   const MCInstrDesc &MCID = TII->get(NewOpcode);
   // Constrain the def register class
   const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF);
   MRI.constrainRegClass(NewReg, TRC);
   // And do the same for the base operand
   TRC = TII->getRegClass(MCID, 2, TRI, *MF);
   MRI.constrainRegClass(MI->getOperand(1).getReg(), TRC);
 
+  unsigned AddrMode = (MCID.TSFlags & ARMII::AddrModeMask);
+  switch (AddrMode) {
+  case ARMII::AddrModeT2_i7:
+  case ARMII::AddrModeT2_i7s2:
+  case ARMII::AddrModeT2_i7s4:
+    // Any MVE load/store
+    return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+        .addReg(NewReg, RegState::Define)
+        .add(MI->getOperand(0))
+        .add(MI->getOperand(1))
+        .addImm(Offset)
+        .add(MI->getOperand(3))
+        .add(MI->getOperand(4))
+        .cloneMemRefs(*MI);
+  case ARMII::AddrModeT2_i8:
+    if (MI->mayLoad()) {
+      return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+          .add(MI->getOperand(0))
+          .addReg(NewReg, RegState::Define)
+          .add(MI->getOperand(1))
+          .addImm(Offset)
+          .add(MI->getOperand(3))
+          .add(MI->getOperand(4))
+          .cloneMemRefs(*MI);
+    } else {
       return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
           .addReg(NewReg, RegState::Define)
           .add(MI->getOperand(0))
           .add(MI->getOperand(1))
           .addImm(Offset)
           .add(MI->getOperand(3))
           .add(MI->getOperand(4))
           .cloneMemRefs(*MI);
     }
+  default:
+    llvm_unreachable("Unhandled createPostIncLoadStore");
+  }
+}
 
 // Given a Base Register, optimise the load/store uses to attempt to create more
 // post-inc accesses. We do this by taking zero offset loads/stores with an add,
 // and convert them to a postinc load/store of the same type. Any subsequent
 // accesses will be adjusted to use and account for the post-inc value.
 // For example:
 // LDR #0            LDR_POSTINC #16
 // LDR #4            LDR #-12
 // LDR #8            LDR #-8
 // LDR #12           LDR #-4
 // ADD #16
 bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
   // We are looking for:
   // One zero offset load/store that can become postinc
   MachineInstr *BaseAccess = nullptr;
   // An increment that can be folded in
   MachineInstr *Increment = nullptr;
   // Other accesses after BaseAccess that will need to be updated to use the
-  // postinc value
+  // postinc value.
   SmallPtrSet<MachineInstr *, 8> OtherAccesses;
   for (auto &Use : MRI->use_nodbg_instructions(Base)) {
     if (!Increment && getAddSubImmediate(Use) != 0) {
       Increment = &Use;
       continue;
     }
 
     int BaseOp = getBaseOperandIndex(Use);
if (!isLegalAddressImm(NewPostIncOpcode, IncrementOffset, TII)) {
     LLVM_DEBUG(dbgs() << "  Illegal addressing mode immediate on postinc\n");
     return false;
   }
 
   // And make sure that the negative value of increment can be added to all
   // other offsets after the BaseAccess. We rely on either
   // dominates(BaseAccess, OtherAccess) or dominates(OtherAccess, BaseAccess)
   // to keep things simple.
+  // This also adds a simple codesize metric, to detect if an instruction (like
+  // t2LDRBi12) which can often be shrunk to a thumb1 instruction (tLDRBi)
+  // cannot because it is converted to something else (t2LDRBi8). We start this
+  // at -1 for the gain from removing the increment.
   SmallPtrSet<MachineInstr *, 4> SuccessorAccesses;
+  int CodesizeEstimate = -1;
   for (auto *Use : OtherAccesses) {
     if (DT->dominates(BaseAccess, Use)) {
       SuccessorAccesses.insert(Use);
       unsigned BaseOp = getBaseOperandIndex(*Use);
-      if (!isLegalAddressImm(
-              Use->getOpcode(),
-              Use->getOperand(BaseOp + 1).getImm() - IncrementOffset, TII)) {
+      if (!isLegalOrConvertableAddressImm(Use->getOpcode(),
+                                          Use->getOperand(BaseOp + 1).getImm() -
+                                              IncrementOffset,
+                                          TII, CodesizeEstimate)) {
         LLVM_DEBUG(dbgs() << "  Illegal addressing mode immediate on use\n");
         return false;
       }
     } else if (!DT->dominates(Use, BaseAccess)) {
       LLVM_DEBUG(
           dbgs() << "  Unknown dominance relation between Base and Use\n");
       return false;
     }
   }
+  if (STI->hasMinSize() && CodesizeEstimate > 0) {
+    LLVM_DEBUG(dbgs() << "  Expected to grow instructions under minsize\n");
+    return false;
+  }
 
   // Replace BaseAccess with a post inc
   LLVM_DEBUG(dbgs() << "Changing: "; BaseAccess->dump());
   LLVM_DEBUG(dbgs() << "  And   : "; Increment->dump());
   Register NewBaseReg = Increment->getOperand(0).getReg();
   MachineInstr *BaseAccessPost =
       createPostIncLoadStore(BaseAccess, IncrementOffset, NewBaseReg, TII, TRI);
   BaseAccess->eraseFromParent();
   Increment->eraseFromParent();
   (void)BaseAccessPost;
   LLVM_DEBUG(dbgs() << "  To    : "; BaseAccessPost->dump());
 
   for (auto *Use : SuccessorAccesses) {
     LLVM_DEBUG(dbgs() << "Changing: "; Use->dump());
-    unsigned BaseOp = getBaseOperandIndex(*Use);
-    Use->getOperand(BaseOp).setReg(NewBaseReg);
-    int OldOffset = Use->getOperand(BaseOp + 1).getImm();
-    Use->getOperand(BaseOp + 1).setImm(OldOffset - IncrementOffset);
+    AdjustBaseAndOffset(Use, NewBaseReg, IncrementOffset, TII);
     LLVM_DEBUG(dbgs() << "  To    : "; Use->dump());
   }
 
   // Remove the kill flag from all uses of NewBaseReg, in case any old uses
   // remain.
   for (MachineOperand &Op : MRI->use_nodbg_operands(NewBaseReg))
     Op.setIsKill(false);
   return true;

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    adds r3, r1, #4
 ; CHECK-NEXT:    adds r2, r0, #4
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB11_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh.w r4, [r3, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r2, #2]
-; CHECK-NEXT:    ldrsh r5, [r3, #-2]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov s4, r4
-; CHECK-NEXT:    ldrsh.w r4, [r3]
+; CHECK-NEXT:    ldrsh r4, [r3], #8
 ; CHECK-NEXT:    vcvt.f16.s32 s4, s4
-; CHECK-NEXT:    vmov s8, r5
+; CHECK-NEXT:    ldrsh r5, [r3, #-10]
 ; CHECK-NEXT:    vmul.f16 s2, s2, s4
-; CHECK-NEXT:    vldr.16 s4, [r2]
 ; CHECK-NEXT:    vmov s6, r4
-; CHECK-NEXT:    ldrsh r4, [r3, #-4]
+; CHECK-NEXT:    vldr.16 s4, [r2]
 ; CHECK-NEXT:    vcvt.f16.s32 s6, s6
-; CHECK-NEXT:    vcvt.f16.s32 s8, s8
+; CHECK-NEXT:    ldrsh r4, [r3, #-12]
 ; CHECK-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NEXT:    vmov s8, r5
 ; CHECK-NEXT:    vldr.16 s6, [r2, #-2]
+; CHECK-NEXT:    vcvt.f16.s32 s8, s8
 ; CHECK-NEXT:    vmov s10, r4
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
 ; CHECK-NEXT:    vmul.f16 s6, s6, s8
 ; CHECK-NEXT:    vldr.16 s8, [r2, #-4]
 ; CHECK-NEXT:    vcvt.f16.s32 s10, s10
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
 ; CHECK-NEXT:    vmul.f16 s8, s8, s10
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
 ; CHECK-NEXT:    le lr, .LBB11_5
 ; CHECK-NEXT:  .LBB11_6: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r6, .LBB11_9
 ; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r0, r0, r12, lsl #1

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll

 ; CHECK-NEXT:  .LBB5_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r8, [r5, #-3]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    ldrb r7, [r6, #-1]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-8]
 ; CHECK-NEXT:    ldrb r8, [r5, #-2]
-; CHECK-NEXT:    ldrb r7, [r6]
+; CHECK-NEXT:    ldrb r7, [r6], #4
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-4]
 ; CHECK-NEXT:    ldrb r8, [r5, #-1]
-; CHECK-NEXT:    ldrb r7, [r6, #1]
+; CHECK-NEXT:    ldrb r7, [r6, #-3]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4]
-; CHECK-NEXT:    ldrb.w r8, [r5]
-; CHECK-NEXT:    adds r5, #4
-; CHECK-NEXT:    ldrb r7, [r6, #2]
-; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    ldrb r8, [r5], #4
+; CHECK-NEXT:    ldrb r7, [r6, #-2]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #4]
 ; CHECK-NEXT:    adds r4, #16
 ; CHECK-NEXT:    le lr, .LBB5_7
 ; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r9, .LBB5_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add r0, r12

 ; CHECK-NEXT:  .LBB7_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r8, [r5, #-3]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    ldrb r7, [r6, #-1]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-8]
 ; CHECK-NEXT:    ldrb r8, [r5, #-2]
-; CHECK-NEXT:    ldrb r7, [r6]
+; CHECK-NEXT:    ldrb r7, [r6], #4
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-4]
 ; CHECK-NEXT:    ldrb r8, [r5, #-1]
-; CHECK-NEXT:    ldrb r7, [r6, #1]
+; CHECK-NEXT:    ldrb r7, [r6, #-3]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4]
-; CHECK-NEXT:    ldrb.w r8, [r5]
-; CHECK-NEXT:    adds r5, #4
-; CHECK-NEXT:    ldrb r7, [r6, #2]
-; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    ldrb r8, [r5], #4
+; CHECK-NEXT:    ldrb r7, [r6, #-2]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #4]
 ; CHECK-NEXT:    adds r4, #16
 ; CHECK-NEXT:    le lr, .LBB7_7
 ; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r9, .LBB7_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add r0, r12

llvm/test/CodeGen/Thumb2/mve-float16regloops.ll

 ; CHECK-NEXT:  @ %bb.5: @ %for.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
 ; CHECK-NEXT:    ldr.w lr, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:  .LBB16_6: @ %for.body
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh r0, [r6]
+; CHECK-NEXT:    ldrh r0, [r6], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r5]
 ; CHECK-NEXT:    adds r1, r5, #2
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #2]
+; CHECK-NEXT:    ldrh r0, [r6, #-14]
 ; CHECK-NEXT:    adds r1, r5, #6
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #4]
+; CHECK-NEXT:    ldrh r0, [r6, #-12]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #4]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #6]
+; CHECK-NEXT:    ldrh r0, [r6, #-10]
 ; CHECK-NEXT:    add.w r1, r5, #10
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #8]
+; CHECK-NEXT:    ldrh r0, [r6, #-8]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #8]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #10]
-; CHECK-NEXT:    ldrh r1, [r6, #14]
+; CHECK-NEXT:    ldrh r0, [r6, #-6]
+; CHECK-NEXT:    ldrh r1, [r6, #-2]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #12]
+; CHECK-NEXT:    ldrh r0, [r6, #-4]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
-; CHECK-NEXT:    adds r6, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    adds r5, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, r1
 ; CHECK-NEXT:    le lr, .LBB16_6
 ; CHECK-NEXT:    b .LBB16_8
 ; CHECK-NEXT:  .LBB16_7: @ in Loop: Header=BB16_4 Depth=1

llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll

 ; CHECK-NEXT:    ldr.w r8, [sp, #36]
 ; CHECK-NEXT:    mov r6, r12
 ; CHECK-NEXT:    mov r5, r7
 ; CHECK-NEXT:    and lr, r2, #3
 ; CHECK-NEXT:    lsrl r6, r5, #6
 ; CHECK-NEXT:    wls lr, lr, .LBB1_7
 ; CHECK-NEXT:  .LBB1_5: @ %while.body11
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrsh.w r5, [r0, #2]
-; CHECK-NEXT:    ldrsh.w r6, [r1]
-; CHECK-NEXT:    ldrsh.w r9, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    ldrsh.w r2, [r1, #2]
-; CHECK-NEXT:    adds r1, #4
-; CHECK-NEXT:    smlalbb r4, r11, r6, r5
+; CHECK-NEXT:    ldrsh r9, [r0], #4
+; CHECK-NEXT:    ldrsh r6, [r1], #4
+; CHECK-NEXT:    ldrsh r5, [r0, #-2]
+; CHECK-NEXT:    ldrsh r2, [r1, #-2]
 ; CHECK-NEXT:    smlalbb r12, r7, r6, r9
+; CHECK-NEXT:    smlalbb r4, r11, r6, r5
 ; CHECK-NEXT:    muls r5, r2, r5
 ; CHECK-NEXT:    smlalbb r4, r11, r2, r9
 ; CHECK-NEXT:    subs.w r12, r12, r5
 ; CHECK-NEXT:    sbc.w r7, r7, r5, asr #31
 ; CHECK-NEXT:    le lr, .LBB1_5
 ; CHECK-NEXT:  @ %bb.6: @ %while.end34.loopexit
 ; CHECK-NEXT:    lsrl r12, r7, #6
 ; CHECK-NEXT:    lsrl r4, r11, #6

llvm/test/CodeGen/Thumb2/postinc-distribute.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -run-pass arm-prera-ldst-opt %s -o - | FileCheck %s
 
 --- |
   define i32* @t2LDRi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2LDRHi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2LDRSHi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2LDRBi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2LDRSBi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2STRi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2STRHi12(i32* %x, i32 %y) { unreachable }
   define i32* @t2STRBi12(i32* %x, i32 %y) { unreachable }
 
   define i32* @storedadd(i32* %x, i32 %y) { unreachable }
+  define i32* @minsize2(i32* %x, i32 %y) minsize optsize { unreachable }
+  define i32* @minsize3(i32* %x, i32 %y) minsize optsize { unreachable }
 
 ...
 ---
 name:            t2LDRi12
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gprnopc, preferred-register: '' }
   - { id: 1, class: rgpr, preferred-register: '' }
liveins:
   - { reg: '$r0', virtual-reg: '%0' }
 body:             |
   bb.0:
     liveins: $r0
 
     ; CHECK-LABEL: name: t2LDRHi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRHi12_:%[0-9]+]]:rgpr = t2LDRHi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRH_POST:%[0-9]+]]:rgpr, [[t2LDRH_POST1:%[0-9]+]]:rgpr = t2LDRH_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRH_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRHi12 %0, 0, 14, $noreg :: (load 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
 ...
 ---
 name:            t2LDRSHi12
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gprnopc, preferred-register: '' }
   - { id: 1, class: rgpr, preferred-register: '' }
   - { id: 2, class: rgpr, preferred-register: '' }
 liveins:
   - { reg: '$r0', virtual-reg: '%0' }
 body:             |
   bb.0:
     liveins: $r0
 
     ; CHECK-LABEL: name: t2LDRSHi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRSHi12_:%[0-9]+]]:rgpr = t2LDRSHi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRSH_POST:%[0-9]+]]:rgpr, [[t2LDRSH_POST1:%[0-9]+]]:rgpr = t2LDRSH_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRSH_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRSHi12 %0, 0, 14, $noreg :: (load 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
 ...
 ---
 name:            t2LDRBi12
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gprnopc, preferred-register: '' }
   - { id: 1, class: rgpr, preferred-register: '' }
   - { id: 2, class: rgpr, preferred-register: '' }
 liveins:
   - { reg: '$r0', virtual-reg: '%0' }
 body:             |
   bb.0:
     liveins: $r0
 
     ; CHECK-LABEL: name: t2LDRBi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRB_POST:%[0-9]+]]:rgpr, [[t2LDRB_POST1:%[0-9]+]]:rgpr = t2LDRB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRB_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
 ...
 ---
 name:            t2LDRSBi12
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gprnopc, preferred-register: '' }
   - { id: 1, class: rgpr, preferred-register: '' }
   - { id: 2, class: rgpr, preferred-register: '' }
 liveins:
   - { reg: '$r0', virtual-reg: '%0' }
 body:             |
   bb.0:
     liveins: $r0
 
     ; CHECK-LABEL: name: t2LDRSBi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRSBi12_:%[0-9]+]]:rgpr = t2LDRSBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRSB_POST:%[0-9]+]]:rgpr, [[t2LDRSB_POST1:%[0-9]+]]:rgpr = t2LDRSB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRSB_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRSBi12 %0, 0, 14, $noreg :: (load 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
 ...

 body:             |
   bb.0:
     liveins: $r0, $r1
 
     ; CHECK-LABEL: name: t2STRHi12
     ; CHECK: liveins: $r0, $r1
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
-    ; CHECK: t2STRHi12 [[COPY1]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: early-clobber %2:rgpr = t2STRH_POST [[COPY1]], [[COPY]], 32, 14 /* CC::al */, $noreg :: (store 4)
+    ; CHECK: $r0 = COPY %2
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = COPY $r1
     t2STRHi12 %1:rgpr, %0, 0, 14, $noreg :: (store 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $r1
 
     ; CHECK-LABEL: name: t2STRBi12
     ; CHECK: liveins: $r0, $r1
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
-    ; CHECK: t2STRBi12 [[COPY1]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: early-clobber %2:rgpr = t2STRB_POST [[COPY1]], [[COPY]], 32, 14 /* CC::al */, $noreg :: (store 4)
+    ; CHECK: $r0 = COPY %2
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = COPY $r1
     t2STRBi12 %1:rgpr, %0, 0, 14, $noreg :: (store 4, align 4)
     %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
bb.0:
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
     t2STRi12 %1, %0, 0, 14, $noreg :: (store 4, align 4)
     $r0 = COPY %1
     tBX_RET 14, $noreg, implicit $r0
 
 ...
+---
+name:            minsize2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gprnopc, preferred-register: '' }
+  - { id: 1, class: rgpr, preferred-register: '' }
+  - { id: 2, class: rgpr, preferred-register: '' }
+  - { id: 3, class: rgpr, preferred-register: '' }
+liveins:
+  - { reg: '$r0', virtual-reg: '%0' }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    ; CHECK-LABEL: name: minsize2
+    ; CHECK: liveins: $r0
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[t2LDRB_POST:%[0-9]+]]:rgpr, [[t2LDRB_POST1:%[0-9]+]]:rgpr = t2LDRB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi8_:%[0-9]+]]:rgpr = t2LDRBi8 [[t2LDRB_POST1]], -30, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRB_POST1]]
+    ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+    %0:gprnopc = COPY $r0
+    %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
+    %3:rgpr = t2LDRBi12 %0, 2, 14, $noreg :: (load 4, align 4)
+    %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
+    $r0 = COPY %2
+    tBX_RET 14, $noreg, implicit $r0
+
+...
+---
+name:            minsize3
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gprnopc, preferred-register: '' }
+  - { id: 1, class: rgpr, preferred-register: '' }
+  - { id: 2, class: rgpr, preferred-register: '' }
+  - { id: 3, class: rgpr, preferred-register: '' }
+  - { id: 4, class: rgpr, preferred-register: '' }
+liveins:
+  - { reg: '$r0', virtual-reg: '%0' }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    ; CHECK-LABEL: name: minsize3
+    ; CHECK: liveins: $r0
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi12_1:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 2, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi12_2:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 4, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+    %0:gprnopc = COPY $r0
+    %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
+    %3:rgpr = t2LDRBi12 %0, 2, 14, $noreg :: (load 4, align 4)
+    %4:rgpr = t2LDRBi12 %0, 4, 14, $noreg :: (load 4, align 4)
+    %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
+    $r0 = COPY %2
+    tBX_RET 14, $noreg, implicit $r0
+
+...