[ARM] Introduce t2WhileLoopStartTP

[dmgreen]

Summary

This adds t2WhileLoopStartTP, similar to the t2DoLoopStartTP added in D90591. It keeps a reference to both the tripcount register and the element count register, so that the ARMLowOverheadLoops pass in the backend can pick the correct one without having to search for it from then operand of a VCTP.

[samtebbs]

llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp

1428–1431

Can getWLSTargetBB be extracted to the tail pred utils header file and used here?

[dmgreen]

Sounds good. Now with a getWhileLoopStartTargetBB used in more places.

[samtebbs]

Sweet, looks good.

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
                      const outliner::Candidate &C) const override;
 
   /// Enable outlining by default at -Oz.
   bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
 
   bool isUnspillableTerminatorImpl(const MachineInstr *MI) const override {
     return MI->getOpcode() == ARM::t2LoopEndDec ||
            MI->getOpcode() == ARM::t2DoLoopStartTP ||
-           MI->getOpcode() == ARM::t2WhileLoopStartLR;
+           MI->getOpcode() == ARM::t2WhileLoopStartLR ||
+           MI->getOpcode() == ARM::t2WhileLoopStartTP;
   }
 
 private:
   /// Returns an unused general-purpose register which can be used for
   /// constructing an outlined call if one exists. Returns 0 otherwise.
   unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const;
 
   // Adds an instruction which saves the link register on top of the stack into

 }
 
 static inline bool isJumpTableBranchOpcode(int Opc) {
   return Opc == ARM::BR_JTr || Opc == ARM::BR_JTm_i12 ||
          Opc == ARM::BR_JTm_rs || Opc == ARM::BR_JTadd || Opc == ARM::tBR_JTr ||
          Opc == ARM::t2BR_JT;
 }
 
-static inline bool isLowOverheadTerminatorOpcode(int Opc) {
-  return Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
-         Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2LoopEnd ||
-         Opc == ARM::t2LoopEndDec;
-}
-
 static inline
 bool isIndirectBranchOpcode(int Opc) {
   return Opc == ARM::BX || Opc == ARM::MOVPCRX || Opc == ARM::tBRIND;
 }
 
 static inline bool isIndirectCall(const MachineInstr &MI) {
   int Opc = MI.getOpcode();
   switch (Opc) {

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
       Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
       Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
       Opc == ARM::t2MOV_ga_pcrel)
     return outliner::InstrType::Illegal;
 
   // Be conservative with ARMv8.1 MVE instructions.
   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
       Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
-      Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2LoopDec ||
-      Opc == ARM::t2LoopEnd || Opc == ARM::t2LoopEndDec)
+      Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
+      Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
+      Opc == ARM::t2LoopEndDec)
     return outliner::InstrType::Illegal;
 
   const MCInstrDesc &MCID = MI.getDesc();
   uint64_t MIFlags = MCID.TSFlags;
   if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
     return outliner::InstrType::Illegal;
 
   // Is this a terminator for a basic block?

llvm/lib/Target/ARM/ARMBlockPlacement.cpp

 // Currently it only moves blocks to fix backwards WLS branches.
 //
 //===----------------------------------------------------------------------===//
 
 #include "ARM.h"
 #include "ARMBaseInstrInfo.h"
 #include "ARMBasicBlockInfo.h"
 #include "ARMSubtarget.h"
+#include "MVETailPredUtils.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "arm-block-placement"
 #define DEBUG_PREFIX "ARM Block Placement: "

 
 char ARMBlockPlacement::ID = 0;
 
 INITIALIZE_PASS(ARMBlockPlacement, DEBUG_TYPE, "ARM block placement", false,
                 false)
 
 static MachineInstr *findWLSInBlock(MachineBasicBlock *MBB) {
   for (auto &Terminator : MBB->terminators()) {
-    if (Terminator.getOpcode() == ARM::t2WhileLoopStartLR)
+    if (isWhileLoopStart(Terminator))
       return &Terminator;
   }
   return nullptr;
 }
 
-/// Find t2WhileLoopStartLR in the loop predecessor BB or otherwise in its only
+/// Find WhileLoopStart in the loop predecessor BB or otherwise in its only
 /// predecessor. If found, returns (BB, WLS Instr) pair, otherwise a null pair.
 static MachineInstr *findWLS(MachineLoop *ML) {
   MachineBasicBlock *Predecessor = ML->getLoopPredecessor();
   if (!Predecessor)
     return nullptr;
   MachineInstr *WlsInstr = findWLSInBlock(Predecessor);
   if (WlsInstr)
     return WlsInstr;

 /// a forward to a backward branching WLS, then move the predecessor block
 /// to before the loopExit/target.
 bool ARMBlockPlacement::fixBackwardsWLS(MachineLoop *ML) {
   MachineInstr *WlsInstr = findWLS(ML);
   if (!WlsInstr)
     return false;
 
   MachineBasicBlock *Predecessor = WlsInstr->getParent();
-  MachineBasicBlock *LoopExit = WlsInstr->getOperand(2).getMBB();
+  MachineBasicBlock *LoopExit = getWhileLoopStartTargetBB(*WlsInstr);
 
   // We don't want to move Preheader to before the function's entry block.
   if (!LoopExit->getPrevNode())
     return false;
   if (blockIsBefore(Predecessor, LoopExit))
     return false;
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Found a backwards WLS from "
                     << Predecessor->getFullName() << " to "
                     << LoopExit->getFullName() << "\n");
 
   // Make sure no forward branching WLSs to the Predecessor become backwards
   // branching. An example loop structure where the Predecessor can't be moved,
   // since bb2's WLS will become forwards once bb3 is moved before/above bb1.
   //
   // bb1:           - LoopExit
   // bb2:
   //      WLS  bb3
   // bb3:          - Predecessor
   //      WLS bb1
   // bb4:          - Header
   for (auto It = ++LoopExit->getIterator(); It != Predecessor->getIterator();
        ++It) {
     MachineBasicBlock *MBB = &*It;
     for (auto &Terminator : MBB->terminators()) {
-      if (Terminator.getOpcode() != ARM::t2WhileLoopStartLR)
+      if (!isWhileLoopStart(Terminator))
         continue;
-      MachineBasicBlock *WLSTarget = Terminator.getOperand(2).getMBB();
+      MachineBasicBlock *WLSTarget = getWhileLoopStartTargetBB(Terminator);
       // TODO: Analyse the blocks to make a decision if it would be worth
       // moving Preheader even if we'd introduce a backwards WLS
       if (WLSTarget == Predecessor) {
         LLVM_DEBUG(
             dbgs() << DEBUG_PREFIX
                    << "Can't move Predecessor"
                       "block as it would convert a WLS from forward to a "
                       "backwards branching WLS\n");

llvm/lib/Target/ARM/ARMInstrThumb2.td

def t2LE : t2LOL<(outs ), (ins lelabel_u11:$label), "le", "$label"> {
   let isTerminator = 1;
 }
 
 let Predicates = [IsThumb2, HasV8_1MMainline, HasLOB] in {
 
 // t2DoLoopStart a pseudo for DLS hardware loops. Lowered into a DLS in
 // ARMLowOverheadLoops if possible, or reverted to a Mov if not.
 def t2DoLoopStart :
-  t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$elts), 4, IIC_Br,
-  [(set GPRlr:$X, (int_start_loop_iterations rGPR:$elts))]>;
+  t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$tc), 4, IIC_Br,
+  [(set GPRlr:$X, (int_start_loop_iterations rGPR:$tc))]>;
 
 // A pseudo for a DLSTP, created in the MVETPAndVPTOptimizationPass from a
 // t2DoLoopStart if the loops is tail predicated. Holds both the element
 // count and trip count of the loop, picking the correct one during
 // ARMLowOverheadLoops when it is converted to a DLSTP or DLS as required.
 let isTerminator = 1, hasSideEffects = 1 in
 def t2DoLoopStartTP :
-  t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$elts, rGPR:$count), 4, IIC_Br, []>;
+  t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$tc, rGPR:$elts), 4, IIC_Br, []>;
 
 // Setup for a t2WhileLoopStart. A pair of t2WhileLoopSetup and t2WhileLoopStart
 // will be created post-ISel from a llvm.test.start.loop.iterations. This
 // t2WhileLoopSetup to setup LR and t2WhileLoopStart to perform the branch. Not
 // valid after reg alloc, as it should be lowered during MVETPAndVPTOptimisations
 // into a t2WhileLoopStartLR (or expanded).
 def t2WhileLoopSetup :
-  t2PseudoInst<(outs GPRlr:$lr), (ins rGPR:$elts), 4, IIC_Br, []>;
+  t2PseudoInst<(outs GPRlr:$lr), (ins rGPR:$tc), 4, IIC_Br, []>;
 
 // A pseudo to represent the decrement in a low overhead loop. A t2LoopDec and
 // t2LoopEnd together represent a LE instruction. Ideally these are converted
 // to a t2LoopEndDec which is lowered as a single instruction.
 let hasSideEffects = 0 in
 def t2LoopDec :
   t2PseudoInst<(outs GPRlr:$Rm), (ins GPRlr:$Rn, imm0_7:$size),
                4, IIC_Br, []>, Sched<[WriteBr]>;
 
 let isBranch = 1, isTerminator = 1, hasSideEffects = 1, Defs = [CPSR] in {
 // The branch in a t2WhileLoopSetup/t2WhileLoopStart pair, eventually turned
 // into a t2WhileLoopStartLR that does both the LR setup and branch.
 def t2WhileLoopStart :
     t2PseudoInst<(outs),
-                 (ins GPRlr:$elts, brtarget:$target),
+                 (ins GPRlr:$tc, brtarget:$target),
                  4, IIC_Br, []>,
                  Sched<[WriteBr]>;
 
 // WhileLoopStartLR that sets up LR and branches on zero, equivalent to WLS. It
 // is lowered in the ARMLowOverheadLoops pass providing the branches are within
 // range. WhileLoopStartLR and LoopEnd to occupy 8 bytes because they may get
 // converted into t2CMP and t2Bcc.
 def t2WhileLoopStartLR :
     t2PseudoInst<(outs GPRlr:$lr),
-                 (ins rGPR:$elts, brtarget:$target),
+                 (ins rGPR:$tc, brtarget:$target),
+                 8, IIC_Br, []>,
+                 Sched<[WriteBr]>;
+
+// Similar to a t2DoLoopStartTP, a t2WhileLoopStartTP is a pseudo for a WLSTP
+// holding both the element count and the tripcount of the loop.
+def t2WhileLoopStartTP :
+    t2PseudoInst<(outs GPRlr:$lr),
+                 (ins rGPR:$tc, rGPR:$elts, brtarget:$target),
                  8, IIC_Br, []>,
                  Sched<[WriteBr]>;
 
 // t2LoopEnd - the branch half of a t2LoopDec/t2LoopEnd pair.
 def t2LoopEnd :
-  t2PseudoInst<(outs), (ins GPRlr:$elts, brtarget:$target),
+  t2PseudoInst<(outs), (ins GPRlr:$tc, brtarget:$target),
   8, IIC_Br, []>, Sched<[WriteBr]>;
 
 // The combination of a t2LoopDec and t2LoopEnd, performing both the LR
 // decrement and branch as a single instruction. Is lowered to a LE or
 // LETP in ARMLowOverheadLoops as appropriate, or converted to t2CMP/t2Bcc
 // if the branches are out of range.
 def t2LoopEndDec :
-  t2PseudoInst<(outs GPRlr:$Rm), (ins GPRlr:$elts, brtarget:$target),
+  t2PseudoInst<(outs GPRlr:$Rm), (ins GPRlr:$tc, brtarget:$target),
   8, IIC_Br, []>, Sched<[WriteBr]>;
 
 } // end isBranch, isTerminator, hasSideEffects
 
 }
 
 } // end isNotDuplicable
 

llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp

static bool isDomainMVE(MachineInstr *MI) {
   uint64_t Domain = MI->getDesc().TSFlags & ARMII::DomainMask;
   return Domain == ARMII::DomainMVE;
 }
 
 static bool shouldInspect(MachineInstr &MI) {
   return isDomainMVE(&MI) || isVectorPredicate(&MI) || hasVPRUse(MI);
 }
 
-static bool isDo(MachineInstr *MI) {
-  return MI->getOpcode() != ARM::t2WhileLoopStartLR;
-}
-
 namespace {
 
   using InstSet = SmallPtrSetImpl<MachineInstr *>;
 
   class PostOrderLoopTraversal {
     MachineLoop &ML;
     MachineLoopInfo &MLI;
     SmallPtrSet<MachineBasicBlock*, 4> Visited;
struct LowOverheadLoop {
     // iteration count, or the number of elements if we're tail predicating.
     MachineOperand &getLoopStartOperand() {
       if (IsTailPredicationLegal())
         return TPNumElements;
       return Start->getOperand(1);
     }
 
     unsigned getStartOpcode() const {
-      bool IsDo = isDo(Start);
+      bool IsDo = isDoLoopStart(*Start);
       if (!IsTailPredicationLegal())
         return IsDo ? ARM::t2DLS : ARM::t2WLS;
 
       return VCTPOpcodeToLSTP(VCTPs.back()->getOpcode(), IsDo);
     }
 
     void dump() const {
       if (Start) dbgs() << "ARM Loops: Found Loop Start: " << *Start;
if (!ValidateLiveOuts()) {
     return false;
   }
 
   // For tail predication, we need to provide the number of elements, instead
   // of the iteration count, to the loop start instruction. The number of
   // elements is provided to the vctp instruction, so we need to check that
   // we can use this register at InsertPt.
   MachineInstr *VCTP = VCTPs.back();
-  if (Start->getOpcode() == ARM::t2DoLoopStartTP) {
+  if (Start->getOpcode() == ARM::t2DoLoopStartTP ||
+      Start->getOpcode() == ARM::t2WhileLoopStartTP) {
     TPNumElements = Start->getOperand(2);
     StartInsertPt = Start;
     StartInsertBB = Start->getParent();
   } else {
     TPNumElements = VCTP->getOperand(1);
     MCRegister NumElements = TPNumElements.getReg().asMCReg();
 
     // If the register is defined within loop, then we can't perform TP.
if (TryRemove(Def, RDA, ElementChain, Ignore)) {
                      " count: " << *MI);
           return false;
         }
       }
       ToRemove.insert(ElementChain.begin(), ElementChain.end());
     }
   }
 
-  // If we converted the LoopStart to a t2DoLoopStartTP, we can also remove any
-  // extra instructions in the preheader, which often includes a now unused MOV.
-  if (Start->getOpcode() == ARM::t2DoLoopStartTP && Preheader &&
-      !Preheader->empty() &&
+  // If we converted the LoopStart to a t2DoLoopStartTP/t2WhileLoopStartTP, we
+  // can also remove any extra instructions in the preheader, which often
+  // includes a now unused MOV.
+  if ((Start->getOpcode() == ARM::t2DoLoopStartTP ||
+       Start->getOpcode() == ARM::t2WhileLoopStartTP) &&
+      Preheader && !Preheader->empty() &&
       !RDA.hasLocalDefBefore(VCTP, VCTP->getOperand(1).getReg())) {
     if (auto *Def = RDA.getUniqueReachingMIDef(
             &Preheader->back(), VCTP->getOperand(1).getReg().asMCReg())) {
       SmallPtrSet<MachineInstr*, 2> Ignore;
       Ignore.insert(VCTPs.begin(), VCTPs.end());
       TryRemove(Def, RDA, ToRemove, Ignore);
     }
   }
auto ValidateRanges = [](MachineInstr *Start, MachineInstr *End,
     // The WLS and LE instructions have 12-bits for the label offset. WLS
     // requires a positive offset, while LE uses negative.
     if (BBUtils->getOffsetOf(End) < BBUtils->getOffsetOf(ML.getHeader()) ||
         !BBUtils->isBBInRange(End, ML.getHeader(), 4094)) {
       LLVM_DEBUG(dbgs() << "ARM Loops: LE offset is out-of-range\n");
       return false;
     }
 
-    if (Start->getOpcode() == ARM::t2WhileLoopStartLR &&
-        (BBUtils->getOffsetOf(Start) >
-             BBUtils->getOffsetOf(Start->getOperand(2).getMBB()) ||
-         !BBUtils->isBBInRange(Start, Start->getOperand(2).getMBB(), 4094))) {
+    if (isWhileLoopStart(*Start)) {
+      MachineBasicBlock *TargetBB = getWhileLoopStartTargetBB(*Start);
+      if (BBUtils->getOffsetOf(Start) > BBUtils->getOffsetOf(TargetBB) ||
+          !BBUtils->isBBInRange(Start, TargetBB, 4094)) {
         LLVM_DEBUG(dbgs() << "ARM Loops: WLS offset is out-of-range!\n");
         return false;
       }
+    }
     return true;
   };
 
   StartInsertPt = MachineBasicBlock::iterator(Start);
   StartInsertBB = Start->getParent();
   LLVM_DEBUG(dbgs() << "ARM Loops: Will insert LoopStart at "
                     << *StartInsertPt);
 

 }
 
 // WhileLoopStart holds the exit block, so produce a cmp lr, 0 and then a
 // beq that branches to the exit branch.
 // TODO: We could also try to generate a cbz if the value in LR is also in
 // another low register.
 void ARMLowOverheadLoops::RevertWhile(MachineInstr *MI) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp: " << *MI);
-  MachineBasicBlock *DestBB = MI->getOperand(2).getMBB();
+  MachineBasicBlock *DestBB = getWhileLoopStartTargetBB(*MI);
   unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ?
     ARM::tBcc : ARM::t2Bcc;
 
   RevertWhileLoopStartLR(MI, TII, BrOpc);
 }
 
 void ARMLowOverheadLoops::RevertDo(MachineInstr *MI) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to mov: " << *MI);
MachineInstr* ARMLowOverheadLoops::ExpandLoopStart(LowOverheadLoop &LoLoop) {
   if (Opc == ARM::t2DLS && Count.isReg() && Count.getReg() == ARM::LR) {
     LLVM_DEBUG(dbgs() << "ARM Loops: Didn't insert start: DLS lr, lr");
     NewStart = nullptr;
   } else {
     MachineInstrBuilder MIB =
       BuildMI(*MBB, InsertPt, Start->getDebugLoc(), TII->get(Opc));
 
     MIB.addDef(ARM::LR);
     MIB.add(Count);
-    if (!isDo(Start))
-      MIB.add(Start->getOperand(2));
+    if (isWhileLoopStart(*Start))
+      MIB.addMBB(getWhileLoopStartTargetBB(*Start));
 
     LLVM_DEBUG(dbgs() << "ARM Loops: Inserted start: " << *MIB);
     NewStart = &*MIB;
   }
 
   LoLoop.ToRemove.insert(Start);
   return NewStart;
 }
 
if (Terminator->isUnconditionalBranch() && I != Terminator) {
       if (BB->isLayoutSuccessor(Succ)) {
         LLVM_DEBUG(dbgs() << "ARM Loops: Removing branch: " << *Terminator);
         Terminator->eraseFromParent();
       }
     }
   };
 
   if (LoLoop.Revert) {
-    if (LoLoop.Start->getOpcode() == ARM::t2WhileLoopStartLR)
+    if (isWhileLoopStart(*LoLoop.Start))
       RevertWhile(LoLoop.Start);
     else
       RevertDo(LoLoop.Start);
     if (LoLoop.Dec == LoLoop.End)
       RevertLoopEndDec(LoLoop.End);
     else
       RevertLoopEnd(LoLoop.End, RevertLoopDec(LoLoop.Dec));
   } else {
for (auto &MBB : *MF) {
     }
 
     if (Starts.empty() && Decs.empty() && Ends.empty() && EndDecs.empty())
       continue;
 
     Changed = true;
 
     for (auto *Start : Starts) {
-      if (Start->getOpcode() == ARM::t2WhileLoopStartLR)
+      if (isWhileLoopStart(*Start))
         RevertWhile(Start);
       else
         RevertDo(Start);
     }
     for (auto *Dec : Decs)
       RevertLoopDec(Dec);
 
     for (auto *End : Ends)

llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp

bool MVETPAndVPTOptimisations::ConvertTailPredLoop(MachineLoop *ML,
   LLVM_DEBUG(dbgs() << "ConvertTailPredLoop on loop "
                     << ML->getHeader()->getName() << "\n");
 
   // Find some loop components including the LoopEnd/Dec/Start, and any VCTP's
   // in the loop.
   MachineInstr *LoopEnd, *LoopPhi, *LoopStart, *LoopDec;
   if (!findLoopComponents(ML, MRI, LoopStart, LoopPhi, LoopDec, LoopEnd))
     return false;
-  if (LoopDec != LoopEnd || LoopStart->getOpcode() != ARM::t2DoLoopStart)
+  if (LoopDec != LoopEnd || (LoopStart->getOpcode() != ARM::t2DoLoopStart &&
+                             LoopStart->getOpcode() != ARM::t2WhileLoopStartLR))
     return false;
 
   SmallVector<MachineInstr *, 4> VCTPs;
   for (MachineBasicBlock *BB : ML->blocks())
     for (MachineInstr &MI : *BB)
       if (isVCTP(&MI))
         VCTPs.push_back(&MI);
 
bool MVETPAndVPTOptimisations::ConvertTailPredLoop(MachineLoop *ML,
   for (MachineInstr &Use :
        MRI->use_instructions(LoopStart->getOperand(0).getReg()))
     if ((InsertPt != MBB->end() && !DT->dominates(&*InsertPt, &Use)) ||
         !DT->dominates(ML->getHeader(), Use.getParent())) {
       LLVM_DEBUG(dbgs() << "  InsertPt could not be a terminator!\n");
       return false;
     }
 
-  MachineInstrBuilder MI = BuildMI(*MBB, InsertPt, LoopStart->getDebugLoc(),
-                                   TII->get(ARM::t2DoLoopStartTP))
-                               .add(LoopStart->getOperand(0))
-                               .add(LoopStart->getOperand(1))
-                               .addReg(CountReg);
-  (void)MI;
+  unsigned NewOpc = LoopStart->getOpcode() == ARM::t2DoLoopStart
+                        ? ARM::t2DoLoopStartTP
+                        : ARM::t2WhileLoopStartTP;
+  MachineInstrBuilder MI =
+      BuildMI(*MBB, InsertPt, LoopStart->getDebugLoc(), TII->get(NewOpc))
+  if (NewOpc == ARM::t2WhileLoopStartTP)
+    MI.add(LoopStart->getOperand(2));
   LLVM_DEBUG(dbgs() << "Replacing " << *LoopStart << "  with "
                     << *MI.getInstr());
   MRI->constrainRegClass(CountReg, &ARM::rGPRRegClass);
   LoopStart->eraseFromParent();
 
   return true;
 }
 

llvm/lib/Target/ARM/MVETailPredUtils.h

 
 namespace llvm {
 
 static inline unsigned VCTPOpcodeToLSTP(unsigned Opcode, bool IsDoLoop) {
   switch (Opcode) {
   default:
     llvm_unreachable("unhandled vctp opcode");
     break;
   case ARM::MVE_VCTP8:
     return IsDoLoop ? ARM::MVE_DLSTP_8 : ARM::MVE_WLSTP_8;
   case ARM::MVE_VCTP16:
     return IsDoLoop ? ARM::MVE_DLSTP_16 : ARM::MVE_WLSTP_16;
   case ARM::MVE_VCTP32:
     return IsDoLoop ? ARM::MVE_DLSTP_32 : ARM::MVE_WLSTP_32;
   case ARM::MVE_VCTP64:
     return IsDoLoop ? ARM::MVE_DLSTP_64 : ARM::MVE_WLSTP_64;
   }
   return 0;
 }
 
 static inline unsigned getTailPredVectorWidth(unsigned Opcode) {
   switch (Opcode) {
   default:
     llvm_unreachable("unhandled vctp opcode");
   case ARM::MVE_VCTP8:
     return 16;
   case ARM::MVE_VCTP16:
     return 8;
   case ARM::MVE_VCTP32:
     return 4;
   case ARM::MVE_VCTP64:
     return 2;
   }
   return 0;
 }
 
 static inline bool isVCTP(const MachineInstr *MI) {
   switch (MI->getOpcode()) {
   default:
     break;
   case ARM::MVE_VCTP8:
   case ARM::MVE_VCTP16:
   case ARM::MVE_VCTP32:
   case ARM::MVE_VCTP64:
     return true;
   }
   return false;
 }
 
-static inline bool isLoopStart(MachineInstr &MI) {
+static inline bool isDoLoopStart(const MachineInstr &MI) {
   return MI.getOpcode() == ARM::t2DoLoopStart ||
-         MI.getOpcode() == ARM::t2DoLoopStartTP ||
-         MI.getOpcode() == ARM::t2WhileLoopStart ||
-         MI.getOpcode() == ARM::t2WhileLoopStartLR;
+         MI.getOpcode() == ARM::t2DoLoopStartTP;
+}
+
+static inline bool isWhileLoopStart(const MachineInstr &MI) {
+  return MI.getOpcode() == ARM::t2WhileLoopStart ||
+         MI.getOpcode() == ARM::t2WhileLoopStartLR ||
+         MI.getOpcode() == ARM::t2WhileLoopStartTP;
+}
+
+static inline bool isLoopStart(const MachineInstr &MI) {
+  return isDoLoopStart(MI) || isWhileLoopStart(MI);
+}
+
+// Return the TargetBB stored in a t2WhileLoopStartLR/t2WhileLoopStartTP.
+inline MachineBasicBlock *getWhileLoopStartTargetBB(const MachineInstr &MI) {
+  assert(isWhileLoopStart(MI) && "Expected WhileLoopStart!");
+  unsigned Op = MI.getOpcode() == ARM::t2WhileLoopStartTP ? 3 : 2;
+  return MI.getOperand(Op).getMBB();
 }
 
 // WhileLoopStart holds the exit block, so produce a subs Op0, Op1, 0 and then a
 // beq that branches to the exit branch.
 // If UseCmp is true, this will create a t2CMP instead of a t2SUBri, meaning the
 // value of LR into the loop will not be setup. This is used if the LR setup is
 // done via another means (via a t2DoLoopStart, for example).
 inline void RevertWhileLoopStartLR(MachineInstr *MI, const TargetInstrInfo *TII,
                                    unsigned BrOpc = ARM::t2Bcc,
                                    bool UseCmp = false) {
   MachineBasicBlock *MBB = MI->getParent();
-  assert(MI->getOpcode() == ARM::t2WhileLoopStartLR &&
-         "Only expected a t2WhileLoopStartLR in RevertWhileLoopStartLR!");
+  assert((MI->getOpcode() == ARM::t2WhileLoopStartLR ||
+          MI->getOpcode() == ARM::t2WhileLoopStartTP) &&
+         "Only expected a t2WhileLoopStartLR/TP in RevertWhileLoopStartLR!");
 
   // Subs/Cmp
   if (UseCmp) {
     MachineInstrBuilder MIB =
         BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2CMPri));
     MIB.add(MI->getOperand(1));
     MIB.addImm(0);
     MIB.addImm(ARMCC::AL);
     MIB.addReg(ARM::NoRegister);
   } else {
     MachineInstrBuilder MIB =
         BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri));
     MIB.add(MI->getOperand(0));
     MIB.add(MI->getOperand(1));
     MIB.addImm(0);
     MIB.addImm(ARMCC::AL);
     MIB.addReg(ARM::NoRegister);
     MIB.addReg(ARM::CPSR, RegState::Define);
   }
 
   // Branch
   MachineInstrBuilder MIB =
       BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
-  MIB.add(MI->getOperand(2)); // branch target
-  MIB.addImm(ARMCC::EQ);      // condition code
+  MIB.addMBB(getWhileLoopStartTargetBB(*MI)); // branch target
+  MIB.addImm(ARMCC::EQ);                      // condition code
   MIB.addReg(ARM::CPSR);
 
   MI->eraseFromParent();
 }
 
 inline void RevertDoLoopStart(MachineInstr *MI, const TargetInstrInfo *TII) {
   MachineBasicBlock *MBB = MI->getParent();
   BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::tMOVr))

llvm/test/CodeGen/Thumb2/LowOverheadLoops/memcall.ll

 ; CHECK-NEXT:    lsl.w r12, r3, #2
 ; CHECK-NEXT:    movs r7, #0
 ; CHECK-NEXT:    b .LBB0_2
 ; CHECK-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB0_4 Depth 2
 ; CHECK-NEXT:    adds r4, r1, r7
 ; CHECK-NEXT:    adds r5, r0, r7
-; CHECK-NEXT:    mov r6, r3
-; CHECK-NEXT:    wlstp.8 lr, r6, .LBB0_3
+; CHECK-NEXT:    wlstp.8 lr, r3, .LBB0_3
 ; CHECK-NEXT:    b .LBB0_4
 ; CHECK-NEXT:  .LBB0_3: @ %for.body
 ; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    add r7, r12
 ; CHECK-NEXT:    subs r2, #1
 ; CHECK-NEXT:    beq .LBB0_5
 ; CHECK-NEXT:    b .LBB0_2
 ; CHECK-NEXT:  .LBB0_4: @ Parent Loop BB0_2 Depth=1

 ; CHECK-NEXT:  .LBB1_1: @ %for.body.preheader
 ; CHECK-NEXT:    lsl.w r12, r2, #2
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    b .LBB1_2
 ; CHECK-NEXT:  .LBB1_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB1_4 Depth 2
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r3, r2
-; CHECK-NEXT:    wlstp.8 lr, r3, .LBB1_3
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB1_3
 ; CHECK-NEXT:    b .LBB1_4
 ; CHECK-NEXT:  .LBB1_3: @ %for.body
 ; CHECK-NEXT:    @ in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    add r0, r12
 ; CHECK-NEXT:    subs r1, #1
 ; CHECK-NEXT:    beq .LBB1_5
 ; CHECK-NEXT:    b .LBB1_2
 ; CHECK-NEXT:  .LBB1_4: @ Parent Loop BB1_2 Depth=1

 ; CHECK-LABEL: test_memset_preheader:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB6_5
 ; CHECK-NEXT:  @ %bb.1: @ %prehead
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mov r3, r2
-; CHECK-NEXT:    wlstp.8 lr, r3, .LBB6_3
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB6_3
 ; CHECK-NEXT:  .LBB6_2: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q0, [r12], #16
 ; CHECK-NEXT:    letp lr, .LBB6_2
 ; CHECK-NEXT:  .LBB6_3: @ %prehead
 ; CHECK-NEXT:    dls lr, r2
 ; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:  .LBB6_4: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1

llvm/test/CodeGen/Thumb2/LowOverheadLoops/wls-search-pred.mir

body:             |
   ; CHECK:   t2B %bb.1, 14 /* CC::al */, $noreg
   ; CHECK: bb.1.prehead:
   ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
   ; CHECK:   [[DEF:%[0-9]+]]:mqpr = IMPLICIT_DEF
   ; CHECK:   [[MVE_VMOVimmi32_:%[0-9]+]]:mqpr = MVE_VMOVimmi32 0, 0, $noreg, [[DEF]]
   ; CHECK:   [[t2ADDri:%[0-9]+]]:rgpr = t2ADDri [[COPY]], 15, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[t2BICri:%[0-9]+]]:rgpr = t2BICri killed [[t2ADDri]], 16, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[t2LSRri:%[0-9]+]]:gprlr = t2LSRri killed [[t2BICri]], 4, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   [[t2WhileLoopStartLR:%[0-9]+]]:gprlr = t2WhileLoopStartLR killed [[t2LSRri]], %bb.3, implicit-def $cpsr
+  ; CHECK:   [[t2WhileLoopStartTP:%[0-9]+]]:gprlr = t2WhileLoopStartTP killed [[t2LSRri]], [[COPY]], %bb.3, implicit-def $cpsr
   ; CHECK: bb.2:
   ; CHECK:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; CHECK:   [[PHI:%[0-9]+]]:rgpr = PHI [[COPY2]], %bb.1, %11, %bb.2
-  ; CHECK:   [[PHI1:%[0-9]+]]:gprlr = PHI [[t2WhileLoopStartLR]], %bb.1, %13, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:gprlr = PHI [[t2WhileLoopStartTP]], %bb.1, %13, %bb.2
   ; CHECK:   [[PHI2:%[0-9]+]]:rgpr = PHI [[COPY]], %bb.1, %15, %bb.2
   ; CHECK:   [[MVE_VCTP8_:%[0-9]+]]:vccr = MVE_VCTP8 [[PHI2]], 0, $noreg
   ; CHECK:   [[t2SUBri:%[0-9]+]]:rgpr = t2SUBri [[PHI2]], 16, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[MVE_VSTRBU8_post:%[0-9]+]]:rgpr = MVE_VSTRBU8_post [[MVE_VMOVimmi32_]], [[PHI]], 16, 1, [[MVE_VCTP8_]]
   ; CHECK:   [[t2LoopEndDec:%[0-9]+]]:gprlr = t2LoopEndDec [[PHI1]], %bb.2, implicit-def $cpsr
   ; CHECK:   t2B %bb.3, 14 /* CC::al */, $noreg
   ; CHECK: bb.3.prehead:
   ; CHECK:   successors: %bb.4(0x80000000)

llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll

 ; CHECK-NEXT:    vshl.i32 q3, q1, #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    b .LBB10_5
 ; CHECK-NEXT:  .LBB10_3: @ %for.cond5.preheader.us73.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB10_5 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r3, r0, r5, lsl #1
-; CHECK-NEXT:    mov r5, r6
-; CHECK-NEXT:    wlstp.8 lr, r5, .LBB10_4
+; CHECK-NEXT:    wlstp.8 lr, r6, .LBB10_4
 ; CHECK-NEXT:    b .LBB10_15
 ; CHECK-NEXT:  .LBB10_4: @ %for.cond1.for.cond.cleanup3_crit_edge.us
 ; CHECK-NEXT:    @ in Loop: Header=BB10_5 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    add r11, r12
 ; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    add r3, r0
 ; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill

llvm/test/CodeGen/Thumb2/mve-memtp-loop.ll

 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp.w r2, #-1
 ; CHECK-NEXT:    it gt
 ; CHECK-NEXT:    popgt {r4, pc}
 ; CHECK-NEXT:  .LBB10_1: @ %prehead
 ; CHECK-NEXT:    mov r12, r1
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r3, r2
-; CHECK-NEXT:    wlstp.8 lr, r3, .LBB10_3
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB10_3
 ; CHECK-NEXT:  .LBB10_2: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q0, [r12], #16
 ; CHECK-NEXT:    vstrb.8 q0, [r4], #16
 ; CHECK-NEXT:    letp lr, .LBB10_2
 ; CHECK-NEXT:  .LBB10_3: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r3, [r0], #1
 ; CHECK-NEXT:    subs r2, #2

 
 define void @twoloops(i32* %X, i32 %n, i32 %m) {
 ; CHECK-LABEL: twoloops:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    mov r3, r0
-; CHECK-NEXT:    mov r1, r2
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB13_2
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB13_2
 ; CHECK-NEXT:  .LBB13_1: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q0, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB13_1
 ; CHECK-NEXT:  .LBB13_2: @ %entry
 ; CHECK-NEXT:    wlstp.8 lr, r2, .LBB13_4
 ; CHECK-NEXT:  .LBB13_3: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q0, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB13_3

 ; CHECK-NEXT:    @ Child Loop BB18_6 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB18_8 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB18_10 Depth 2
 ; CHECK-NEXT:    movw r3, :lower16:arr_56
 ; CHECK-NEXT:    add.w r1, r0, #15
 ; CHECK-NEXT:    movt r3, :upper16:arr_56
 ; CHECK-NEXT:    lsr.w r12, r1, #4
 ; CHECK-NEXT:    mov r2, r3
-; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB18_5
+; CHECK-NEXT:    wlstp.8 lr, r0, .LBB18_5
 ; CHECK-NEXT:  .LBB18_4: @ Parent Loop BB18_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB18_4
 ; CHECK-NEXT:  .LBB18_5: @ %loop
 ; CHECK-NEXT:    @ in Loop: Header=BB18_3 Depth=1
 ; CHECK-NEXT:    mov r2, r3
-; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB18_7
+; CHECK-NEXT:    wlstp.8 lr, r0, .LBB18_7
 ; CHECK-NEXT:  .LBB18_6: @ Parent Loop BB18_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB18_6
 ; CHECK-NEXT:  .LBB18_7: @ %loop
 ; CHECK-NEXT:    @ in Loop: Header=BB18_3 Depth=1
 ; CHECK-NEXT:    mov r2, r3
-; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB18_9
+; CHECK-NEXT:    wlstp.8 lr, r0, .LBB18_9
 ; CHECK-NEXT:  .LBB18_8: @ Parent Loop BB18_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB18_8
 ; CHECK-NEXT:  .LBB18_9: @ %loop
 ; CHECK-NEXT:    @ in Loop: Header=BB18_3 Depth=1
 ; CHECK-NEXT:    mov r1, r0
 ; CHECK-NEXT:    subs.w lr, r12, #0

 ; CHECK-NEXT:    mov.w r1, #11
 ; CHECK-NEXT:    cinc r1, r1, ne
 ; CHECK-NEXT:    movs r0, #38
 ; CHECK-NEXT:    mul r2, r1, r0
 ; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    movw r0, :lower16:arr_22
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    movt r0, :upper16:arr_22
-; CHECK-NEXT:    str r2, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    add.w r1, r2, #15
 ; CHECK-NEXT:    lsrs r3, r1, #4
-; CHECK-NEXT:    mov r1, r2
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB19_2
+; CHECK-NEXT:    strd r3, r2, [sp] @ 8-byte Folded Spill
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB19_2
 ; CHECK-NEXT:  .LBB19_1: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q0, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB19_1
 ; CHECK-NEXT:  .LBB19_2: @ %entry
 ; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    movw r6, :lower16:arr_20
 ; CHECK-NEXT:    movt r6, :upper16:arr_20
 ; CHECK-NEXT:    add.w r3, r6, #80

 ; CHECK-NEXT:    vstrh.16 q0, [r7], #152
 ; CHECK-NEXT:    vstrh.16 q0, [r11], #152
 ; CHECK-NEXT:    strd r9, r10, [r3, #64]
 ; CHECK-NEXT:    adds r5, #38
 ; CHECK-NEXT:    adds r3, #152
 ; CHECK-NEXT:    le lr, .LBB19_3
 ; CHECK-NEXT:  @ %bb.4: @ %for.cond.cleanup6
 ; CHECK-NEXT:    movw r0, :lower16:arr_22
-; CHECK-NEXT:    ldrd r2, r1, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    movt r0, :upper16:arr_22
-; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r0, r0, #1824
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB19_6
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB19_6
 ; CHECK-NEXT:  .LBB19_5: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q1, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB19_5
 ; CHECK-NEXT:  .LBB19_6: @ %for.cond.cleanup6
 ; CHECK-NEXT:    movw r6, :lower16:arr_20
 ; CHECK-NEXT:    movw r0, #7376
 ; CHECK-NEXT:    movt r6, :upper16:arr_20
 ; CHECK-NEXT:    adds r3, r6, r0

 ; CHECK-NEXT:    vstrh.16 q0, [r12], #152
 ; CHECK-NEXT:    vstrh.16 q0, [r1], #152
 ; CHECK-NEXT:    strd r10, r11, [r3, #64]
 ; CHECK-NEXT:    adds r0, #38
 ; CHECK-NEXT:    adds r3, #152
 ; CHECK-NEXT:    le lr, .LBB19_7
 ; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup6.1
 ; CHECK-NEXT:    movw r0, :lower16:arr_22
-; CHECK-NEXT:    ldrd r2, r1, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    movt r0, :upper16:arr_22
-; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    ldr r3, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r0, r0, #3648
-; CHECK-NEXT:    wlstp.8 lr, r1, .LBB19_10
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    wlstp.8 lr, r2, .LBB19_10
 ; CHECK-NEXT:  .LBB19_9: @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vstrb.8 q1, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB19_9
 ; CHECK-NEXT:  .LBB19_10: @ %for.cond.cleanup6.1
 ; CHECK-NEXT:    movw r7, :lower16:arr_20
 ; CHECK-NEXT:    movw r0, #14672
 ; CHECK-NEXT:    movt r7, :upper16:arr_20
 ; CHECK-NEXT:    adds r3, r7, r0

 ; CHECK-NEXT:    vstrh.16 q0, [r12], #152
 ; CHECK-NEXT:    vstrh.16 q0, [r4], #152
 ; CHECK-NEXT:    strd r10, r11, [r3, #64]
 ; CHECK-NEXT:    adds r1, #38
 ; CHECK-NEXT:    adds r3, #152
 ; CHECK-NEXT:    le lr, .LBB19_11
 ; CHECK-NEXT:  @ %bb.12: @ %for.cond.cleanup6.2
 ; CHECK-NEXT:    movw r0, :lower16:arr_22
-; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT:    ldrd r2, r1, [sp] @ 8-byte Folded Reload
 ; CHECK-NEXT:    movt r0, :upper16:arr_22
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    add.w r0, r0, #5472
-; CHECK-NEXT:    wls lr, r1, .LBB19_14
+; CHECK-NEXT:    wlstp.8 lr, r1, .LBB19_14
 ; CHECK-NEXT:  .LBB19_13: @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vctp.8 r1
-; CHECK-NEXT:    subs r1, #16
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q1, [r0], #16
-; CHECK-NEXT:    le lr, .LBB19_13
+; CHECK-NEXT:    vstrb.8 q1, [r0], #16
+; CHECK-NEXT:    letp lr, .LBB19_13
 ; CHECK-NEXT:  .LBB19_14: @ %for.cond.cleanup6.2
 ; CHECK-NEXT:    movw r2, :lower16:arr_21
 ; CHECK-NEXT:    movw r1, #5508
 ; CHECK-NEXT:    movt r2, :upper16:arr_21
 ; CHECK-NEXT:    movw r7, :lower16:arr_20
 ; CHECK-NEXT:    add r2, r1
 ; CHECK-NEXT:    movw r1, #22000
 ; CHECK-NEXT:    movt r7, :upper16:arr_20