[ARM] Expand distributing increments to also handle existing pre/post inc instructions.

[dmgreen]

Summary

This extends the distributing postinc code in load/store optimizer to also handle the case where there is an existing pre/post inc instuction, where subsequent instructions can be modified to use the adjusted increment from the increment. This can save us having to keep the old register live past the increment instruction.

[samparker]

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

2607

The function name doesn't appear to match that opcodes that it's checking?

2819–2821

I'm still a bit concerned with this loop.... Can't we get in a pickle if there are two or more instructions within the loop which use BaseReg with a zero offset? OtherAccesses relies on BaseAccess only being assigned once, right? Don't we also need to ensure that BaseReg isn't assigned between BaseAccess and OtherAccesses?

[dmgreen]

Rebase and rename backwards function.

[dmgreen]

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

2607

Oh. Whoops!

2819–2821

If I understand what you are saying properly.. We are in SSA form still, in this pre-ra pass. We shouldn't be reassigning BaseReg.

But I may have missed the point and it's been a while since I wrote this. Let me know.

[samparker]

Thanks, LGTM

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

2819–2821

Ah! Yes, I had completely missed the 'ARMPreAllocLoadStoreOpt::'

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

static int getBaseOperandIndex(MachineInstr &MI) {
   case ARM::t2LDRBi12:
   case ARM::t2LDRSBi8:
   case ARM::t2LDRSBi12:
   case ARM::t2STRBi8:
   case ARM::t2STRBi12:
   case ARM::t2STRHi8:
   case ARM::t2STRHi12:
     return 1;
+  case ARM::MVE_VLDRBS16_post:
+  case ARM::MVE_VLDRBS32_post:
+  case ARM::MVE_VLDRBU16_post:
+  case ARM::MVE_VLDRBU32_post:
+  case ARM::MVE_VLDRHS32_post:
+  case ARM::MVE_VLDRHU32_post:
+  case ARM::MVE_VLDRBU8_post:
+  case ARM::MVE_VLDRHU16_post:
+  case ARM::MVE_VLDRWU32_post:
+  case ARM::MVE_VSTRB16_post:
+  case ARM::MVE_VSTRB32_post:
+  case ARM::MVE_VSTRH32_post:
+  case ARM::MVE_VSTRBU8_post:
+  case ARM::MVE_VSTRHU16_post:
+  case ARM::MVE_VSTRWU32_post:
+  case ARM::MVE_VLDRBS16_pre:
+  case ARM::MVE_VLDRBS32_pre:
+  case ARM::MVE_VLDRBU16_pre:
+  case ARM::MVE_VLDRBU32_pre:
+  case ARM::MVE_VLDRHS32_pre:
+  case ARM::MVE_VLDRHU32_pre:
+  case ARM::MVE_VLDRBU8_pre:
+  case ARM::MVE_VLDRHU16_pre:
+  case ARM::MVE_VLDRWU32_pre:
+  case ARM::MVE_VSTRB16_pre:
+  case ARM::MVE_VSTRB32_pre:
+  case ARM::MVE_VSTRH32_pre:
+  case ARM::MVE_VSTRBU8_pre:
+  case ARM::MVE_VSTRHU16_pre:
+  case ARM::MVE_VSTRWU32_pre:
+    return 2;
   }
   return -1;
 }
 
+static bool isPostIndex(MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  case ARM::MVE_VLDRBS16_post:
+  case ARM::MVE_VLDRBS32_post:
+  case ARM::MVE_VLDRBU16_post:
+  case ARM::MVE_VLDRBU32_post:
+  case ARM::MVE_VLDRHS32_post:
+  case ARM::MVE_VLDRHU32_post:
+  case ARM::MVE_VLDRBU8_post:
+  case ARM::MVE_VLDRHU16_post:
+  case ARM::MVE_VLDRWU32_post:
+  case ARM::MVE_VSTRB16_post:
+  case ARM::MVE_VSTRB32_post:
+  case ARM::MVE_VSTRH32_post:
+  case ARM::MVE_VSTRBU8_post:
+  case ARM::MVE_VSTRHU16_post:
+  case ARM::MVE_VSTRWU32_post:
+    return true;
+  }
+  return false;
+}
+
+static bool isPreIndex(MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  case ARM::MVE_VLDRBS16_pre:
+  case ARM::MVE_VLDRBS32_pre:
+  case ARM::MVE_VLDRBU16_pre:
+  case ARM::MVE_VLDRBU32_pre:
+  case ARM::MVE_VLDRHS32_pre:
+  case ARM::MVE_VLDRHU32_pre:
+  case ARM::MVE_VLDRBU8_pre:
+  case ARM::MVE_VLDRHU16_pre:
+  case ARM::MVE_VLDRWU32_pre:
+  case ARM::MVE_VSTRB16_pre:
+  case ARM::MVE_VSTRB32_pre:
+  case ARM::MVE_VSTRH32_pre:
+  case ARM::MVE_VSTRBU8_pre:
+  case ARM::MVE_VSTRHU16_pre:
+  case ARM::MVE_VSTRWU32_pre:
+    return true;
+  }
+  return false;
+}
+
 // Given a memory access Opcode, check that the give Imm would be a valid Offset
 // for this instruction (same as isLegalAddressImm), Or if the instruction
 // could be easily converted to one where that was valid. For example converting
 // t2LDRi12 to t2LDRi8 for negative offsets. Works in conjunction with
 // AdjustBaseAndOffset below.
 static bool isLegalOrConvertableAddressImm(unsigned Opcode, int Imm,
                                            const TargetInstrInfo *TII,
                                            int &CodesizeEstimate) {
if (MI->mayLoad()) {
           .cloneMemRefs(*MI);
     }
   default:
     llvm_unreachable("Unhandled createPostIncLoadStore");
   }
 }
 
 // Given a Base Register, optimise the load/store uses to attempt to create more
-// post-inc accesses. We do this by taking zero offset loads/stores with an add,
-// and convert them to a postinc load/store of the same type. Any subsequent
-// accesses will be adjusted to use and account for the post-inc value.
+// post-inc accesses and less register moves. We do this by taking zero offset
+// loads/stores with an add, and convert them to a postinc load/store of the
+// same type. Any subsequent accesses will be adjusted to use and account for
+// the post-inc value.
 // For example:
 // LDR #0            LDR_POSTINC #16
 // LDR #4            LDR #-12
 // LDR #8            LDR #-8
 // LDR #12           LDR #-4
 // ADD #16
+//
+// At the same time if we do not find an increment but do find an existing
+// pre/post inc instruction, we can still adjust the offsets of subsequent
+// instructions to save the register move that would otherwise be needed for the
+// in-place increment.
 bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
   // We are looking for:
   // One zero offset load/store that can become postinc
   MachineInstr *BaseAccess = nullptr;
+  MachineInstr *PrePostInc = nullptr;
   // An increment that can be folded in
   MachineInstr *Increment = nullptr;
   // Other accesses after BaseAccess that will need to be updated to use the
   // postinc value.
   SmallPtrSet<MachineInstr *, 8> OtherAccesses;
   for (auto &Use : MRI->use_nodbg_instructions(Base)) {
     if (!Increment && getAddSubImmediate(Use) != 0) {
       Increment = &Use;
       continue;
     }
 
     int BaseOp = getBaseOperandIndex(Use);
     if (BaseOp == -1)
       return false;
 
     if (!Use.getOperand(BaseOp).isReg() ||
         Use.getOperand(BaseOp).getReg() != Base)
       return false;
-    if (Use.getOperand(BaseOp + 1).getImm() == 0)
+    if (isPreIndex(Use) || isPostIndex(Use))
+      PrePostInc = &Use;
+    else if (Use.getOperand(BaseOp + 1).getImm() == 0)
       BaseAccess = &Use;
     else
       OtherAccesses.insert(&Use);
   }
 
-  if (!BaseAccess || !Increment ||
-      BaseAccess->getParent() != Increment->getParent())
+  int IncrementOffset;
+  Register NewBaseReg;
+  if (BaseAccess && Increment) {
+    if (PrePostInc || BaseAccess->getParent() != Increment->getParent())
       return false;
     Register PredReg;
     if (Increment->definesRegister(ARM::CPSR) ||
         getInstrPredicate(*Increment, PredReg) != ARMCC::AL)
       return false;
 
     LLVM_DEBUG(dbgs() << "\nAttempting to distribute increments on VirtualReg "
                       << Base.virtRegIndex() << "\n");
 
     // Make sure that Increment has no uses before BaseAccess.
     for (MachineInstr &Use :
         MRI->use_nodbg_instructions(Increment->getOperand(0).getReg())) {
       if (!DT->dominates(BaseAccess, &Use) || &Use == BaseAccess) {
         LLVM_DEBUG(dbgs() << "  BaseAccess doesn't dominate use of increment\n");
         return false;
       }
     }
 
     // Make sure that Increment can be folded into Base
-  int IncrementOffset = getAddSubImmediate(*Increment);
+    IncrementOffset = getAddSubImmediate(*Increment);
     unsigned NewPostIncOpcode = getPostIndexedLoadStoreOpcode(
         BaseAccess->getOpcode(), IncrementOffset > 0 ? ARM_AM::add : ARM_AM::sub);
     if (!isLegalAddressImm(NewPostIncOpcode, IncrementOffset, TII)) {
       LLVM_DEBUG(dbgs() << "  Illegal addressing mode immediate on postinc\n");
       return false;
     }
+  }
+  else if (PrePostInc) {
+    // If we already have a pre/post index load/store then set BaseAccess,
+    // IncrementOffset and NewBaseReg to the values it already produces,
+    // allowing us to update and subsequent uses of BaseOp reg with the
+    // incremented value.
+    if (Increment)
+      return false;
+
+    LLVM_DEBUG(dbgs() << "\nAttempting to distribute increments on already "
+                      << "indexed VirtualReg " << Base.virtRegIndex() << "\n");
+    int BaseOp = getBaseOperandIndex(*PrePostInc);
+    IncrementOffset = PrePostInc->getOperand(BaseOp+1).getImm();
+    BaseAccess = PrePostInc;
+    NewBaseReg = PrePostInc->getOperand(0).getReg();
+  }
+  else
+    return false;
 
   // And make sure that the negative value of increment can be added to all
   // other offsets after the BaseAccess. We rely on either
   // dominates(BaseAccess, OtherAccess) or dominates(OtherAccess, BaseAccess)
   // to keep things simple.
   // This also adds a simple codesize metric, to detect if an instruction (like
   // t2LDRBi12) which can often be shrunk to a thumb1 instruction (tLDRBi)
   // cannot because it is converted to something else (t2LDRBi8). We start this
if (DT->dominates(BaseAccess, Use)) {
       return false;
     }
   }
   if (STI->hasMinSize() && CodesizeEstimate > 0) {
     LLVM_DEBUG(dbgs() << "  Expected to grow instructions under minsize\n");
     return false;
   }
 
+  if (!PrePostInc) {
     // Replace BaseAccess with a post inc
     LLVM_DEBUG(dbgs() << "Changing: "; BaseAccess->dump());
     LLVM_DEBUG(dbgs() << "  And   : "; Increment->dump());
-  Register NewBaseReg = Increment->getOperand(0).getReg();
+    NewBaseReg = Increment->getOperand(0).getReg();
     MachineInstr *BaseAccessPost =
         createPostIncLoadStore(BaseAccess, IncrementOffset, NewBaseReg, TII, TRI);
     BaseAccess->eraseFromParent();
     Increment->eraseFromParent();
     (void)BaseAccessPost;
     LLVM_DEBUG(dbgs() << "  To    : "; BaseAccessPost->dump());
+  }
 
   for (auto *Use : SuccessorAccesses) {
     LLVM_DEBUG(dbgs() << "Changing: "; Use->dump());
     AdjustBaseAndOffset(Use, NewBaseReg, IncrementOffset, TII);
     LLVM_DEBUG(dbgs() << "  To    : "; Use->dump());
   }
 
   // Remove the kill flag from all uses of NewBaseReg, in case any old uses

llvm/test/CodeGen/Thumb2/mve-postinc-distribute.mir

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRWU32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[MVE_VLDRWU32_post:%[0-9]+]]:rgpr, [[MVE_VLDRWU32_post1:%[0-9]+]]:mqpr = MVE_VLDRWU32_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRWU32_:%[0-9]+]]:mqpr = MVE_VLDRWU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRWU32_:%[0-9]+]]:mqpr = MVE_VLDRWU32 [[MVE_VLDRWU32_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRWU32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRWU32_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRWU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHU16_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[MVE_VLDRHU16_post:%[0-9]+]]:rgpr, [[MVE_VLDRHU16_post1:%[0-9]+]]:mqpr = MVE_VLDRHU16_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHU16_:%[0-9]+]]:mqpr = MVE_VLDRHU16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHU16_:%[0-9]+]]:mqpr = MVE_VLDRHU16 [[MVE_VLDRHU16_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHU16_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRHU16_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHU16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU8_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[MVE_VLDRBU8_post:%[0-9]+]]:rgpr, [[MVE_VLDRBU8_post1:%[0-9]+]]:mqpr = MVE_VLDRBU8_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU8_:%[0-9]+]]:mqpr = MVE_VLDRBU8 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU8_:%[0-9]+]]:mqpr = MVE_VLDRBU8 [[MVE_VLDRBU8_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU8_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRBU8_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU8 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBS32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBS32_post:%[0-9]+]]:tgpr, [[MVE_VLDRBS32_post1:%[0-9]+]]:mqpr = MVE_VLDRBS32_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBS32_:%[0-9]+]]:mqpr = MVE_VLDRBS32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBS32_:%[0-9]+]]:mqpr = MVE_VLDRBS32 [[MVE_VLDRBS32_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBS32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBS32_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBS32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBU32_post:%[0-9]+]]:tgpr, [[MVE_VLDRBU32_post1:%[0-9]+]]:mqpr = MVE_VLDRBU32_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU32_:%[0-9]+]]:mqpr = MVE_VLDRBU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU32_:%[0-9]+]]:mqpr = MVE_VLDRBU32 [[MVE_VLDRBU32_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBU32_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHS32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRHS32_post:%[0-9]+]]:tgpr, [[MVE_VLDRHS32_post1:%[0-9]+]]:mqpr = MVE_VLDRHS32_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHS32_:%[0-9]+]]:mqpr = MVE_VLDRHS32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHS32_:%[0-9]+]]:mqpr = MVE_VLDRHS32 [[MVE_VLDRHS32_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHS32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRHS32_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHS32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHU32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRHU32_post:%[0-9]+]]:tgpr, [[MVE_VLDRHU32_post1:%[0-9]+]]:mqpr = MVE_VLDRHU32_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHU32_:%[0-9]+]]:mqpr = MVE_VLDRHU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHU32_:%[0-9]+]]:mqpr = MVE_VLDRHU32 [[MVE_VLDRHU32_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHU32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRHU32_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBS16_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBS16_post:%[0-9]+]]:tgpr, [[MVE_VLDRBS16_post1:%[0-9]+]]:mqpr = MVE_VLDRBS16_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBS16_:%[0-9]+]]:mqpr = MVE_VLDRBS16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBS16_:%[0-9]+]]:mqpr = MVE_VLDRBS16 [[MVE_VLDRBS16_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBS16_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBS16_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBS16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU16_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBU16_post:%[0-9]+]]:tgpr, [[MVE_VLDRBU16_post1:%[0-9]+]]:mqpr = MVE_VLDRBU16_post [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU16_:%[0-9]+]]:mqpr = MVE_VLDRBU16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU16_:%[0-9]+]]:mqpr = MVE_VLDRBU16 [[MVE_VLDRBU16_post]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU16_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBU16_post %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRWU32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRWU32_post:%[0-9]+]]:rgpr = MVE_VSTRWU32_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRWU32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRWU32 [[COPY]], [[MVE_VSTRWU32_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRWU32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRWU32_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRWU32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRHU16_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRHU16_post:%[0-9]+]]:rgpr = MVE_VSTRHU16_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRHU16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRHU16 [[COPY]], [[MVE_VSTRHU16_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRHU16_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRHU16_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRHU16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRBU8_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRBU8_post:%[0-9]+]]:rgpr = MVE_VSTRBU8_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRBU8 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRBU8 [[COPY]], [[MVE_VSTRBU8_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRBU8_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRBU8_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRBU8 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRH32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRH32_post:%[0-9]+]]:tgpr = MVE_VSTRH32_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRH32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRH32 [[COPY]], [[MVE_VSTRH32_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRH32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRH32_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRH32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRB32_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB32_post:%[0-9]+]]:tgpr = MVE_VSTRB32_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB32 [[COPY]], [[MVE_VSTRB32_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB32_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB32_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRB16_post
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB16_post:%[0-9]+]]:tgpr = MVE_VSTRB16_post [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_post]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB16_post]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB16_post %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRWU32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VLDRWU32_pre:%[0-9]+]]:rgpr, [[MVE_VLDRWU32_pre1:%[0-9]+]]:mqpr = MVE_VLDRWU32_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRWU32_:%[0-9]+]]:mqpr = MVE_VLDRWU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRWU32_:%[0-9]+]]:mqpr = MVE_VLDRWU32 [[MVE_VLDRWU32_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRWU32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:rgpr = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRWU32_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRWU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHU16_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VLDRHU16_pre:%[0-9]+]]:rgpr, [[MVE_VLDRHU16_pre1:%[0-9]+]]:mqpr = MVE_VLDRHU16_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHU16_:%[0-9]+]]:mqpr = MVE_VLDRHU16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHU16_:%[0-9]+]]:mqpr = MVE_VLDRHU16 [[MVE_VLDRHU16_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHU16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:rgpr = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRHU16_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHU16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU8_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBU8_pre:%[0-9]+]]:rgpr, [[MVE_VLDRBU8_pre1:%[0-9]+]]:mqpr = MVE_VLDRBU8_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU8_:%[0-9]+]]:mqpr = MVE_VLDRBU8 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU8_:%[0-9]+]]:mqpr = MVE_VLDRBU8 [[MVE_VLDRBU8_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU8_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:rgpr = COPY $r0
     %2:rgpr, %1:mqpr = MVE_VLDRBU8_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU8 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBS32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBS32_pre:%[0-9]+]]:tgpr, [[MVE_VLDRBS32_pre1:%[0-9]+]]:mqpr = MVE_VLDRBS32_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBS32_:%[0-9]+]]:mqpr = MVE_VLDRBS32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBS32_:%[0-9]+]]:mqpr = MVE_VLDRBS32 [[MVE_VLDRBS32_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBS32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBS32_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBS32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBU32_pre:%[0-9]+]]:tgpr, [[MVE_VLDRBU32_pre1:%[0-9]+]]:mqpr = MVE_VLDRBU32_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU32_:%[0-9]+]]:mqpr = MVE_VLDRBU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU32_:%[0-9]+]]:mqpr = MVE_VLDRBU32 [[MVE_VLDRBU32_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBU32_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHS32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRHS32_pre:%[0-9]+]]:tgpr, [[MVE_VLDRHS32_pre1:%[0-9]+]]:mqpr = MVE_VLDRHS32_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHS32_:%[0-9]+]]:mqpr = MVE_VLDRHS32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHS32_:%[0-9]+]]:mqpr = MVE_VLDRHS32 [[MVE_VLDRHS32_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHS32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRHS32_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHS32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRHU32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRHU32_pre:%[0-9]+]]:tgpr, [[MVE_VLDRHU32_pre1:%[0-9]+]]:mqpr = MVE_VLDRHU32_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRHU32_:%[0-9]+]]:mqpr = MVE_VLDRHU32 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRHU32_:%[0-9]+]]:mqpr = MVE_VLDRHU32 [[MVE_VLDRHU32_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRHU32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRHU32_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRHU32 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBS16_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBS16_pre:%[0-9]+]]:tgpr, [[MVE_VLDRBS16_pre1:%[0-9]+]]:mqpr = MVE_VLDRBS16_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBS16_:%[0-9]+]]:mqpr = MVE_VLDRBS16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBS16_:%[0-9]+]]:mqpr = MVE_VLDRBS16 [[MVE_VLDRBS16_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBS16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBS16_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBS16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 

 body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VLDRBU16_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VLDRBU16_pre:%[0-9]+]]:tgpr, [[MVE_VLDRBU16_pre1:%[0-9]+]]:mqpr = MVE_VLDRBU16_pre [[COPY]], 32, 0, $noreg :: (load 16, align 8)
-    ; CHECK: [[MVE_VLDRBU16_:%[0-9]+]]:mqpr = MVE_VLDRBU16 [[COPY]], 16, 0, $noreg :: (load 16, align 8)
+    ; CHECK: [[MVE_VLDRBU16_:%[0-9]+]]:mqpr = MVE_VLDRBU16 [[MVE_VLDRBU16_pre]], -16, 0, $noreg :: (load 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VLDRBU16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:tgpr = COPY $r0
     %2:tgpr, %1:mqpr = MVE_VLDRBU16_pre %0, 32, 0, $noreg :: (load 16, align 8)
     %1:mqpr = MVE_VLDRBU16 %0, 16, 0, $noreg :: (load 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
 
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRWU32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRWU32_pre:%[0-9]+]]:rgpr = MVE_VSTRWU32_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRWU32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRWU32 [[COPY]], [[MVE_VSTRWU32_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRWU32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRWU32_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRWU32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRHU16_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRHU16_pre:%[0-9]+]]:rgpr = MVE_VSTRHU16_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRHU16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRHU16 [[COPY]], [[MVE_VSTRHU16_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRHU16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRHU16_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRHU16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRBU8_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r0
     ; CHECK: [[MVE_VSTRBU8_pre:%[0-9]+]]:rgpr = MVE_VSTRBU8_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRBU8 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRBU8 [[COPY]], [[MVE_VSTRBU8_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRBU8_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:rgpr = COPY $r0
     %2:rgpr = MVE_VSTRBU8_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRBU8 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRH32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRH32_pre:%[0-9]+]]:tgpr = MVE_VSTRH32_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRH32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRH32 [[COPY]], [[MVE_VSTRH32_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRH32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRH32_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRH32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRB32_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB32_pre:%[0-9]+]]:tgpr = MVE_VSTRB32_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB32 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB32 [[COPY]], [[MVE_VSTRB32_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB32_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB32_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB32 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: MVE_VSTRB16_pre
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB16_pre:%[0-9]+]]:tgpr = MVE_VSTRB16_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_pre]], -16, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB16_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %2
     tBX_RET 14, $noreg, implicit $r0
body:             |
   bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: multiple2
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB16_pre:%[0-9]+]]:tgpr = MVE_VSTRB16_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], -16, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], 34, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_pre]], -16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_pre]], -48, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_pre]], 2, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB16_pre]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB16_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, -16, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, 34, 0, $noreg :: (store 16, align 8)
bb.0:
     liveins: $r0, $q0
 
     ; CHECK-LABEL: name: multiple3
     ; CHECK: liveins: $r0, $q0
     ; CHECK: [[COPY:%[0-9]+]]:mqpr = COPY $q0
     ; CHECK: [[COPY1:%[0-9]+]]:tgpr = COPY $r0
     ; CHECK: [[MVE_VSTRB16_pre:%[0-9]+]]:tgpr = MVE_VSTRB16_pre [[COPY]], [[COPY1]], 32, 0, $noreg :: (store 16, align 8)
     ; CHECK: [[MVE_VSTRB16_pre1:%[0-9]+]]:tgpr = MVE_VSTRB16_pre [[COPY]], [[COPY1]], 64, 0, $noreg :: (store 16, align 8)
-    ; CHECK: MVE_VSTRB16 [[COPY]], [[COPY1]], 16, 0, $noreg :: (store 16, align 8)
+    ; CHECK: MVE_VSTRB16 [[COPY]], [[MVE_VSTRB16_pre1]], -48, 0, $noreg :: (store 16, align 8)
     ; CHECK: $r0 = COPY [[MVE_VSTRB16_pre1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %1:mqpr = COPY $q0
     %0:tgpr = COPY $r0
     %2:tgpr = MVE_VSTRB16_pre %1, %0, 32, 0, $noreg :: (store 16, align 8)
     %3:tgpr = MVE_VSTRB16_pre %1, %0, 64, 0, $noreg :: (store 16, align 8)
     MVE_VSTRB16 %1, %0, 16, 0, $noreg :: (store 16, align 8)
     $r0 = COPY %3

llvm/test/CodeGen/Thumb2/mve-vst2.ll

 
 define void @vst2_v4i64(<4 x i64> *%src, <8 x i64> *%dst) {
 ; CHECK-LABEL: vst2_v4i64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q4, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #32]
 ; CHECK-NEXT:    vmov.f64 d6, d1
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov.f64 d10, d3
 ; CHECK-NEXT:    vmov.f32 s13, s3
 ; CHECK-NEXT:    vmov.f32 s21, s7
+; CHECK-NEXT:    vmov.f32 s2, s16
 ; CHECK-NEXT:    vmov.f32 s6, s8
 ; CHECK-NEXT:    vmov.f32 s14, s18
 ; CHECK-NEXT:    vmov.f32 s22, s10
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vmov.f32 s7, s9
-; CHECK-NEXT:    vmov.f32 s23, s11
-; CHECK-NEXT:    vstrb.8 q1, [r0], #48
-; CHECK-NEXT:    vmov.f32 s15, s19
-; CHECK-NEXT:    vstrw.32 q5, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s3, s17
-; CHECK-NEXT:    vstrw.32 q3, [r0]
+; CHECK-NEXT:    vmov.f32 s7, s9
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #32]
+; CHECK-NEXT:    vmov.f32 s15, s19
+; CHECK-NEXT:    vstrb.8 q1, [r1], #48
+; CHECK-NEXT:    vmov.f32 s23, s11
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vstrw.32 q5, [r1, #-32]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
   %s1 = getelementptr <4 x i64>, <4 x i64>* %src, i32 0
   %l1 = load <4 x i64>, <4 x i64>* %s1, align 4
   %s2 = getelementptr <4 x i64>, <4 x i64>* %src, i32 1
   %l2 = load <4 x i64>, <4 x i64>* %s2, align 4
   %s = shufflevector <4 x i64> %l1, <4 x i64> %l2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>

llvm/test/CodeGen/Thumb2/mve-vst3.ll

 ; i64
 
 define void @vst3_v2i64(<2 x i64> *%src, <6 x i64> *%dst) {
 ; CHECK-LABEL: vst3_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov.f64 d6, d5
 ; CHECK-NEXT:    vmov.f32 s13, s11
 ; CHECK-NEXT:    vmov.f32 s14, s2
 ; CHECK-NEXT:    vmov.f32 s15, s3
 ; CHECK-NEXT:    vmov.f32 s2, s6
 ; CHECK-NEXT:    vmov.f32 s3, s7
 ; CHECK-NEXT:    vmov.f32 s6, s8
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s7, s9
-; CHECK-NEXT:    vstrb.8 q1, [r0], #32
-; CHECK-NEXT:    vstrw.32 q3, [r0]
+; CHECK-NEXT:    vstrb.8 q1, [r1], #32
+; CHECK-NEXT:    vstrw.32 q3, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [r1, #-16]
 ; CHECK-NEXT:    bx lr
 entry:
   %s1 = getelementptr <2 x i64>, <2 x i64>* %src, i32 0
   %l1 = load <2 x i64>, <2 x i64>* %s1, align 4
   %s2 = getelementptr <2 x i64>, <2 x i64>* %src, i32 1
   %l2 = load <2 x i64>, <2 x i64>* %s2, align 4
   %s3 = getelementptr <2 x i64>, <2 x i64>* %src, i32 2
   %l3 = load <2 x i64>, <2 x i64>* %s3, align 4