[MachineScheduler] Track physical register dependencies per-regunit

[foad]

Summary

Change the scheduler's physical register dependency tracking from
registers-and-their-aliases to regunits. This has a couple of advantages
when subregisters are used:

- The dependency tracking is more accurate and creates fewer useless
  edges in the dependency graph. An AMDGPU example, edited for clarity:

    SU(0): $vgpr1 = V_MOV_B32 $sgpr0
    SU(1): $vgpr1 = V_ADDC_U32 0, $vgpr1
    SU(2): $vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0

  There is a data dependency on $vgpr1 from SU(0) to SU(1) and from
  SU(1) to SU(2). But the old dependency tracking code also added a
  useless edge from SU(0) to SU(2) because it thought that SU(0)'s def
  of $vgpr1 aliased with SU(2)'s use of $vgpr0_vgpr1.

- On targets like AMDGPU that make heavy use of subregisters, each
  register can have a huge number of aliases - it can be quadratic in
  the size of the largest defined register tuple. There is a much lower
  bound on the number of regunits per register, so iterating over
  regunits is faster than iterating over aliases.

The LLVM compile-time tracker shows a tiny overall improvement of 0.03%
on X86. I expect a larger compile-time improvement on targets like
AMDGPU.

Recommit after fixing AggressiveAntiDepBreaker in D156880.

[arsenm]

Finally!

[arsenm]

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

225–226

liveins should really use regunits too

226–230

This can refine to check the lanemask too

251–252

Don't see why this isn't a reference anymore

[foad]

Fix DefMIDesc.

[foad]

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

225–226

Not in this patch :)

226–230

Is there a neat way to iterate regunits for a RegisterMaskPair?

251–252

That was an oversight.

[arsenm]

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

226–230

Not that I know off the top of my head. Should at least add a fixme here

[arsenm]

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

226–230

I think you need to use MCRegUnitMaskIterator

[kparzysz]

LGTM, thanks!

[foad]

Rebase on D156880.

[bjope]

llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h

91

This should perhaps be renamed as RegUnit2SUnitsMap now when when spare set key is a RegUnit and not a Reg.

Caused me some head-ache downstream until I found out that

for (Reg2SUnitsMap::iterator I = DAG->Uses.find(*Subreg);
     I != DAG->Uses.end(); ++I) {

probably isn't working any longer unless I make some changes to use a RegUnit in that find call.

[foad]

llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h

91

Good idea. Done in d9c0166f12f0231df0893a397326efb93935ed26.

llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h

VReg2SUnitOperIdx(unsigned VReg, LaneBitmask LaneMask,
       : VReg2SUnit(VReg, LaneMask, SU), OperandIndex(OperandIndex) {}
   };
 
   /// Record a physical register access.
   /// For non-data-dependent uses, OpIdx == -1.
   struct PhysRegSUOper {
     SUnit *SU;
     int OpIdx;
-    unsigned Reg;
+    unsigned RegUnit;
 
-    PhysRegSUOper(SUnit *su, int op, unsigned R): SU(su), OpIdx(op), Reg(R) {}
+    PhysRegSUOper(SUnit *su, int op, unsigned R)
+        : SU(su), OpIdx(op), RegUnit(R) {}
 
-    unsigned getSparseSetIndex() const { return Reg; }
+    unsigned getSparseSetIndex() const { return RegUnit; }
   };
 
   /// Use a SparseMultiSet to track physical registers. Storage is only
   /// allocated once for the pass. It can be cleared in constant time and reused
   /// without any frees.
   using Reg2SUnitsMap =
       SparseMultiSet<PhysRegSUOper, identity<unsigned>, uint16_t>;
 
   /// Use SparseSet as a SparseMap by relying on the fact that it never
   /// compares ValueT's, only unsigned keys. This allows the set to be cleared
   /// between scheduling regions in constant time as long as ValueT does not
   /// require a destructor.
   using VReg2SUnitMap = SparseSet<VReg2SUnit, VirtReg2IndexFunctor>;
 

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

MachineInstr *ExitMI =
           ? &*skipDebugInstructionsBackward(RegionEnd, RegionBegin)
           : nullptr;
   ExitSU.setInstr(ExitMI);
   // Add dependencies on the defs and uses of the instruction.
   if (ExitMI) {
     for (const MachineOperand &MO : ExitMI->all_uses()) {
       Register Reg = MO.getReg();
       if (Reg.isPhysical()) {
-        Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg));
+        for (MCRegUnit Unit : TRI->regunits(Reg))
+          Uses.insert(PhysRegSUOper(&ExitSU, -1, Unit));
       } else if (Reg.isVirtual() && MO.readsReg()) {
         addVRegUseDeps(&ExitSU, MO.getOperandNo());
       }
     }
   }
   if (!ExitMI || (!ExitMI->isCall() && !ExitMI->isBarrier())) {
     // For others, e.g. fallthrough, conditional branch, assume the exit
     // uses all the registers that are livein to the successor blocks.
     for (const MachineBasicBlock *Succ : BB->successors()) {
       for (const auto &LI : Succ->liveins()) {
-        if (!Uses.contains(LI.PhysReg))
-          Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg));
+        // TODO: Use LI.LaneMask to refine this.
+        for (MCRegUnit Unit : TRI->regunits(LI.PhysReg)) {
+          if (!Uses.contains(Unit))
+            Uses.insert(PhysRegSUOper(&ExitSU, -1, Unit));
+        }
       }
     }
   }
 }
 
 /// MO is an operand of SU's instruction that defines a physical register. Adds
 /// data dependencies from SU to any uses of the physical register.
 void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) {
   const MachineOperand &MO = SU->getInstr()->getOperand(OperIdx);
   assert(MO.isDef() && "expect physreg def");
   Register Reg = MO.getReg();
 
   // Ask the target if address-backscheduling is desirable, and if so how much.
   const TargetSubtargetInfo &ST = MF.getSubtarget();
 
   // Only use any non-zero latency for real defs/uses, in contrast to
   // "fake" operands added by regalloc.
   const MCInstrDesc &DefMIDesc = SU->getInstr()->getDesc();
   bool ImplicitPseudoDef = (OperIdx >= DefMIDesc.getNumOperands() &&
                             !DefMIDesc.hasImplicitDefOfPhysReg(Reg));
-  for (MCRegAliasIterator Alias(Reg, TRI, true); Alias.isValid(); ++Alias) {
-    for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) {
+  for (MCRegUnit Unit : TRI->regunits(Reg)) {
+    for (Reg2SUnitsMap::iterator I = Uses.find(Unit); I != Uses.end(); ++I) {
       SUnit *UseSU = I->SU;
       if (UseSU == SU)
         continue;
 
       // Adjust the dependence latency using operand def/use information,
       // then allow the target to perform its own adjustments.
       MachineInstr *UseInstr = nullptr;
       int UseOpIdx = I->OpIdx;
       bool ImplicitPseudoUse = false;
       SDep Dep;
       if (UseOpIdx < 0) {
         Dep = SDep(SU, SDep::Artificial);
       } else {
         // Set the hasPhysRegDefs only for physreg defs that have a use within
         // the scheduling region.
         SU->hasPhysRegDefs = true;
+
         UseInstr = UseSU->getInstr();
+        Register UseReg = UseInstr->getOperand(UseOpIdx).getReg();
         const MCInstrDesc &UseMIDesc = UseInstr->getDesc();
-        ImplicitPseudoUse = (UseOpIdx >= ((int)UseMIDesc.getNumOperands()) &&
-                             !UseMIDesc.hasImplicitUseOfPhysReg(*Alias));
-        Dep = SDep(SU, SDep::Data, *Alias);
+        ImplicitPseudoUse = UseOpIdx >= ((int)UseMIDesc.getNumOperands()) &&
+                            !UseMIDesc.hasImplicitUseOfPhysReg(UseReg);
+
+        Dep = SDep(SU, SDep::Data, UseReg);
       }
       if (!ImplicitPseudoDef && !ImplicitPseudoUse) {
         Dep.setLatency(SchedModel.computeOperandLatency(SU->getInstr(), OperIdx,
                                                         UseInstr, UseOpIdx));
       } else {
         Dep.setLatency(0);
       }
       ST.adjustSchedDependency(SU, OperIdx, UseSU, UseOpIdx, Dep);
void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
 
   // Optionally add output and anti dependencies. For anti
   // dependencies we use a latency of 0 because for a multi-issue
   // target we want to allow the defining instruction to issue
   // in the same cycle as the using instruction.
   // TODO: Using a latency of 1 here for output dependencies assumes
   //       there's no cost for reusing registers.
   SDep::Kind Kind = MO.isUse() ? SDep::Anti : SDep::Output;
-  for (MCRegAliasIterator Alias(Reg, TRI, true); Alias.isValid(); ++Alias) {
-    for (Reg2SUnitsMap::iterator I = Defs.find(*Alias); I != Defs.end(); ++I) {
+  for (MCRegUnit Unit : TRI->regunits(Reg)) {
+    for (Reg2SUnitsMap::iterator I = Defs.find(Unit); I != Defs.end(); ++I) {
       SUnit *DefSU = I->SU;
       if (DefSU == &ExitSU)
         continue;
       MachineInstr *DefInstr = DefSU->getInstr();
-      if (DefSU != SU && (Kind != SDep::Output || !MO.isDead() ||
-                          !DefInstr->registerDefIsDead(*Alias))) {
-        SDep Dep(SU, Kind, /*Reg=*/*Alias);
+      MachineOperand &DefMO = DefInstr->getOperand(I->OpIdx);
+      if (DefSU != SU &&
+          (Kind != SDep::Output || !MO.isDead() || !DefMO.isDead())) {
+        SDep Dep(SU, Kind, DefMO.getReg());
         if (Kind != SDep::Anti) {
           Dep.setLatency(
               SchedModel.computeOutputLatency(MI, OperIdx, DefInstr));
         }
         ST.adjustSchedDependency(SU, OperIdx, DefSU, I->OpIdx, Dep);
         DefSU->addPred(Dep);
       }
     }
   }
 
   if (MO.isUse()) {
     SU->hasPhysRegUses = true;
     // Either insert a new Reg2SUnits entry with an empty SUnits list, or
     // retrieve the existing SUnits list for this register's uses.
     // Push this SUnit on the use list.
-    Uses.insert(PhysRegSUOper(SU, OperIdx, Reg));
+    for (MCRegUnit Unit : TRI->regunits(Reg))
+      Uses.insert(PhysRegSUOper(SU, OperIdx, Unit));
     if (RemoveKillFlags)
       MO.setIsKill(false);
   } else {
     addPhysRegDataDeps(SU, OperIdx);
 
     // Clear previous uses and defs of this register and its subregisters.
-    for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg)) {
-      Uses.eraseAll(SubReg);
+    for (MCRegUnit Unit : TRI->regunits(Reg)) {
+      Uses.eraseAll(Unit);
       if (!MO.isDead())
-        Defs.eraseAll(SubReg);
+        Defs.eraseAll(Unit);
     }
+
     if (MO.isDead() && SU->isCall) {
       // Calls will not be reordered because of chain dependencies (see
       // below). Since call operands are dead, calls may continue to be added
       // to the DefList making dependence checking quadratic in the size of
       // the block. Instead, we leave only one call at the back of the
       // DefList.
-      Reg2SUnitsMap::RangePair P = Defs.equal_range(Reg);
+      for (MCRegUnit Unit : TRI->regunits(Reg)) {
+        Reg2SUnitsMap::RangePair P = Defs.equal_range(Unit);
         Reg2SUnitsMap::iterator B = P.first;
         Reg2SUnitsMap::iterator I = P.second;
         for (bool isBegin = I == B; !isBegin; /* empty */) {
           isBegin = (--I) == B;
           if (!I->SU->isCall)
             break;
           I = Defs.erase(I);
         }
       }
+    }
 
     // Defs are pushed in the order they are visited and never reordered.
-    Defs.insert(PhysRegSUOper(SU, OperIdx, Reg));
+    for (MCRegUnit Unit : TRI->regunits(Reg))
+      Defs.insert(PhysRegSUOper(SU, OperIdx, Unit));
   }
 }
 
 LaneBitmask ScheduleDAGInstrs::getLaneMaskForMO(const MachineOperand &MO) const
 {
   Register Reg = MO.getReg();
   // No point in tracking lanemasks if we don't have interesting subregisters.
   const TargetRegisterClass &RC = *MRI.getRegClass(Reg);

llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll

 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mov_b32_e32 v2, s1
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, s0, v1
 ; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[1:2], v[1:2]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    s_movk_i32 s0, 0x7fff
 ; VI-NEXT:    flat_load_ushort v3, v[0:1]
-; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:    s_movk_i32 s0, 0x7fff
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_bfi_b32 v2, s0, v3, v2
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_copysign_out_f16_mag_f16_sign_f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34

llvm/test/CodeGen/AMDGPU/load-global-i16.ll

 ; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(7)
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v34, 0xffff, v3
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xffff, v2
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v1
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v1
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v38, 0xffff, v1
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v36, 0xffff, v0
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v29
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v2, 0xffff, v29
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff, v28
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v7
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6

llvm/test/CodeGen/AMDGPU/schedule-physregdeps.mir

 # RUN: llc -march=amdgcn -run-pass machine-scheduler -verify-machineinstrs %s -o - -debug-only=machine-scheduler 2>&1 | FileCheck %s
 # REQUIRES: asserts
 
 # CHECK:     SU(0):   $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec
 # CHECK:       Successors:
 # CHECK-NEXT:    SU(2): Out  Latency=1
-# CHECK-NEXT:    SU(4): Out  Latency=1
 # CHECK-NEXT:    SU(2): Data Latency=1 Reg=$vgpr0
-# CHECK-NEXT:    SU(4): Data Latency=1 Reg=$vgpr0_vgpr1
 # CHECK:     SU(1):   $vgpr1 = V_MOV_B32_e32 $sgpr0, implicit $exec
 # CHECK:       Successors:
 # CHECK-NEXT:    SU(3): Out  Latency=1
-# CHECK-NEXT:    SU(4): Out  Latency=1
 # CHECK-NEXT:    SU(3): Data Latency=1 Reg=$vgpr1
-# CHECK-NEXT:    SU(4): Data Latency=1 Reg=$vgpr0_vgpr1
 # CHECK:     SU(2):   $vgpr0 = V_ADD_CO_U32_e32 $sgpr2, $vgpr0, implicit-def $vcc, implicit $exec
 # CHECK:       Predecessors:
 # CHECK-NEXT:    SU(0): Out  Latency=1
 # CHECK-NEXT:    SU(0): Data Latency=1 Reg=$vgpr0
 # CHECK:       Successors:
 # CHECK-NEXT:    SU(4): Out  Latency=1
 # CHECK-NEXT:    SU(4): Data Latency=1 Reg=$vgpr0_vgpr1
 # CHECK-NEXT:    SU(3): Out  Latency=1
 # CHECK-NEXT:    SU(3): Data Latency=1 Reg=$vcc
-# CHECK-NEXT:    SU(4): Anti Latency=0
 # CHECK:     SU(3):   $vgpr1 = V_ADDC_U32_e32 0, $vgpr1, implicit-def dead $vcc, implicit $vcc, implicit $exec
 # CHECK:       Predecessors:
 # CHECK-NEXT:    SU(2): Out  Latency=1
 # CHECK-NEXT:    SU(2): Data Latency=1 Reg=$vcc
 # CHECK-NEXT:    SU(1): Out  Latency=1
 # CHECK-NEXT:    SU(1): Data Latency=1 Reg=$vgpr1
 # CHECK:       Successors:
 # CHECK-NEXT:    SU(4): Out  Latency=1
 # CHECK-NEXT:    SU(4): Data Latency=1 Reg=$vgpr0_vgpr1
-# CHECK-NEXT:    SU(4): Anti Latency=0
 # CHECK:     SU(4):   $vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
 # CHECK:       Predecessors:
 # CHECK-NEXT:    SU(3): Out  Latency=1
 # CHECK-NEXT:    SU(3): Data Latency=1 Reg=$vgpr0_vgpr1
-# CHECK-NEXT:    SU(3): Anti Latency=0
 # CHECK-NEXT:    SU(2): Out  Latency=1
 # CHECK-NEXT:    SU(2): Data Latency=1 Reg=$vgpr0_vgpr1
-# CHECK-NEXT:    SU(2): Anti Latency=0
-# CHECK-NEXT:    SU(1): Out  Latency=1
-# CHECK-NEXT:    SU(1): Data Latency=1 Reg=$vgpr0_vgpr1
-# CHECK-NEXT:    SU(0): Out  Latency=1
-# CHECK-NEXT:    SU(0): Data Latency=1 Reg=$vgpr0_vgpr1
 # CHECK:       Successors:
 # CHECK-NEXT:    ExitSU: Ord  Latency=3 Artificial
 
 ---
 name: test
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2
     $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec
     $vgpr1 = V_MOV_B32_e32 $sgpr0, implicit $exec
     $vgpr0 = V_ADD_CO_U32_e32 $sgpr2, $vgpr0, implicit-def $vcc, implicit $exec
     $vgpr1 = V_ADDC_U32_e32 0, $vgpr1, implicit-def dead $vcc, implicit $vcc, implicit $exec
     $vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
 ...

llvm/test/CodeGen/Hexagon/autohvx/fp-to-int.ll

 ; CHECK-NEXT:     v11.w = vsub(v11.w,v1.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r3 = ##2147483647
 ; CHECK-NEXT:     r7 = #64
 ; CHECK-NEXT:     v8.w = vasr(v8.w,r6)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v23 = vsplat(r3)
+; CHECK-NEXT:     v22 = vsplat(r3)
 ; CHECK-NEXT:     v7.w = vasr(v7.w,r6)
-; CHECK-NEXT:     v20.w = vsub(v9.w,v1.w)
+; CHECK-NEXT:     v19.w = vsub(v9.w,v1.w)
 ; CHECK-NEXT:     v8.w = vsub(v10.w,v8.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v21.w = vasl(v6.w,r2)
-; CHECK-NEXT:     v28 = vmux(q1,v1,v23)
-; CHECK-NEXT:     v26 = vmux(q0,v1,v23)
+; CHECK-NEXT:     v20.w = vasl(v6.w,r2)
+; CHECK-NEXT:     v27 = vmux(q1,v1,v22)
+; CHECK-NEXT:     v25 = vmux(q0,v1,v22)
 ; CHECK-NEXT:     v7.w = vsub(v10.w,v7.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v5.w = vasl(v4.w,r2)
 ; CHECK-NEXT:     v8.w = vmin(v8.w,v13.w)
-; CHECK-NEXT:     v9 = vor(v21,v1)
-; CHECK-NEXT:     v22.w = vmin(v7.w,v13.w)
+; CHECK-NEXT:     v9 = vor(v20,v1)
+; CHECK-NEXT:     v21.w = vmin(v7.w,v13.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v4.w = vasr(v20.w,r6)
+; CHECK-NEXT:     v4.w = vasr(v19.w,r6)
 ; CHECK-NEXT:     q3 = vcmp.gt(v8.w,v12.w)
 ; CHECK-NEXT:     v5 = vor(v5,v1)
-; CHECK-NEXT:     q2 = vcmp.gt(v22.w,v12.w)
+; CHECK-NEXT:     q2 = vcmp.gt(v21.w,v12.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v11.w = vasr(v11.w,r6)
 ; CHECK-NEXT:     v4.w = vsub(v10.w,v4.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v3.w = vasl(v2.w,r2)
 ; CHECK-NEXT:     v10.w = vsub(v10.w,v11.w)
 ; CHECK-NEXT:     v4.w = vmin(v4.w,v13.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v24.w = vasl(v0.w,r2)
+; CHECK-NEXT:     v23.w = vasl(v0.w,r2)
 ; CHECK-NEXT:     v3 = vor(v3,v1)
 ; CHECK-NEXT:     v10.w = vmin(v10.w,v13.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v8.w = vlsr(v9.w,v8.w)
-; CHECK-NEXT:     v6 = vor(v24,v1)
+; CHECK-NEXT:     v6 = vor(v23,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v5.w = vlsr(v5.w,v22.w)
-; CHECK-NEXT:     v27.w = vsub(v12.w,v8.w)
+; CHECK-NEXT:     v5.w = vlsr(v5.w,v21.w)
+; CHECK-NEXT:     v26.w = vsub(v12.w,v8.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v3.w = vlsr(v3.w,v4.w)
-; CHECK-NEXT:     v25.w = vsub(v12.w,v5.w)
-; CHECK-NEXT:     v8 = vmux(q1,v27,v8)
+; CHECK-NEXT:     v24.w = vsub(v12.w,v5.w)
+; CHECK-NEXT:     v8 = vmux(q1,v26,v8)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.w = vlsr(v6.w,v10.w)
-; CHECK-NEXT:     v5 = vmux(q0,v25,v5)
+; CHECK-NEXT:     v5 = vmux(q0,v24,v5)
 ; CHECK-NEXT:     q0 = vcmp.gt(v12.w,v2.w)
-; CHECK-NEXT:     v29.w = vsub(v12.w,v3.w)
+; CHECK-NEXT:     v28.w = vsub(v12.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v2 = vmux(q3,v8,v28)
+; CHECK-NEXT:     v2 = vmux(q3,v8,v27)
+; CHECK-NEXT:     v29.w = vsub(v12.w,v6.w)
 ; CHECK-NEXT:     q3 = vcmp.gt(v12.w,v0.w)
-; CHECK-NEXT:     v30.w = vsub(v12.w,v6.w)
-; CHECK-NEXT:     v5 = vmux(q2,v5,v26)
+; CHECK-NEXT:     v5 = vmux(q2,v5,v25)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0 = vmux(q0,v1,v23)
-; CHECK-NEXT:     v3 = vmux(q0,v29,v3)
+; CHECK-NEXT:     v30 = vmux(q0,v1,v22)
+; CHECK-NEXT:     v3 = vmux(q0,v28,v3)
 ; CHECK-NEXT:     q2 = vcmp.gt(v4.w,v12.w)
-; CHECK-NEXT:     v31 = vmux(q3,v30,v6)
+; CHECK-NEXT:     v31 = vmux(q3,v29,v6)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v2.h = vpack(v2.w,v5.w):sat
-; CHECK-NEXT:     v1 = vmux(q3,v1,v23)
+; CHECK-NEXT:     v1 = vmux(q3,v1,v22)
 ; CHECK-NEXT:     q3 = vcmp.gt(v10.w,v12.w)
-; CHECK-NEXT:     v0 = vmux(q2,v3,v0)
+; CHECK-NEXT:     v0 = vmux(q2,v3,v30)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1 = vmux(q3,v31,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v3.h = vpack(v1.w,v0.w):sat
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {

 ; CHECK-NEXT:     q3 = vcmp.gt(v13.w,v2.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r4 = #32
 ; CHECK-NEXT:     v11.w = vasl(v1.w,r4)
 ; CHECK-NEXT:     v12.w = vsub(v12.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v23 = vsplat(r4)
+; CHECK-NEXT:     v21 = vsplat(r4)
 ; CHECK-NEXT:     v8.w = vasr(v8.w,r6)
 ; CHECK-NEXT:     v11.w = vsub(v11.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v9.w = vasr(v9.w,r6)
 ; CHECK-NEXT:     v8.w = vsub(v14.w,v8.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.w = vasl(v5.w,r2)
 ; CHECK-NEXT:     v9.w = vsub(v14.w,v9.w)
-; CHECK-NEXT:     v8.w = vmin(v8.w,v23.w)
+; CHECK-NEXT:     v8.w = vmin(v8.w,v21.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v7.w = vasl(v2.w,r2)
 ; CHECK-NEXT:     v6 = vor(v6,v3)
-; CHECK-NEXT:     v9.w = vmin(v9.w,v23.w)
+; CHECK-NEXT:     v9.w = vmin(v9.w,v21.w)
 ; CHECK-NEXT:     q1 = vcmp.gt(v13.w,v8.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v22.w = vasr(v11.w,r6)
+; CHECK-NEXT:     v20.w = vasr(v11.w,r6)
 ; CHECK-NEXT:     v7 = vor(v7,v3)
 ; CHECK-NEXT:     q2 = vcmp.gt(v13.w,v9.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v12.w = vasr(v12.w,r6)
-; CHECK-NEXT:     v5.w = vsub(v14.w,v22.w)
+; CHECK-NEXT:     v5.w = vsub(v14.w,v20.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v4.w = vasl(v1.w,r2)
-; CHECK-NEXT:     v24.w = vsub(v14.w,v12.w)
-; CHECK-NEXT:     v5.w = vmin(v5.w,v23.w)
+; CHECK-NEXT:     v22.w = vsub(v14.w,v12.w)
+; CHECK-NEXT:     v5.w = vmin(v5.w,v21.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r2 = ##2147483647
 ; CHECK-NEXT:     v10.w = vasl(v0.w,r2)
 ; CHECK-NEXT:     v4 = vor(v4,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v25 = vsplat(r2)
+; CHECK-NEXT:     v23 = vsplat(r2)
 ; CHECK-NEXT:     v6.w = vlsr(v6.w,v8.w)
 ; CHECK-NEXT:     v3 = vor(v10,v3)
-; CHECK-NEXT:     v10.w = vmin(v24.w,v23.w)
+; CHECK-NEXT:     v10.w = vmin(v22.w,v21.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v7.w = vlsr(v7.w,v9.w)
-; CHECK-NEXT:     v27 = vmux(q1,v25,v6)
+; CHECK-NEXT:     v25 = vmux(q1,v23,v6)
 ; CHECK-NEXT:     q1 = vcmp.gt(v13.w,v5.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v26.w = vlsr(v4.w,v5.w)
-; CHECK-NEXT:     v28 = vmux(q2,v25,v7)
+; CHECK-NEXT:     v24.w = vlsr(v4.w,v5.w)
+; CHECK-NEXT:     v26 = vmux(q2,v23,v7)
 ; CHECK-NEXT:     q2 = vcmp.gt(v13.w,v10.w)
-; CHECK-NEXT:     v4 = vmux(q0,v13,v27)
+; CHECK-NEXT:     v4 = vmux(q0,v13,v25)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v3.w = vlsr(v3.w,v10.w)
-; CHECK-NEXT:     v29 = vmux(q3,v13,v28)
-; CHECK-NEXT:     v2 = vmux(q1,v25,v26)
+; CHECK-NEXT:     v27 = vmux(q3,v13,v26)
+; CHECK-NEXT:     v2 = vmux(q1,v23,v24)
 ; CHECK-NEXT:     q1 = vcmp.gt(v13.w,v1.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
+; CHECK-NEXT:     v28 = vmux(q2,v23,v3)
 ; CHECK-NEXT:     q3 = vcmp.gt(v13.w,v0.w)
-; CHECK-NEXT:     v1 = vmux(q2,v25,v3)
-; CHECK-NEXT:     v0 = vmux(q1,v13,v2)
+; CHECK-NEXT:     v29 = vmux(q1,v13,v2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v30.uh = vpack(v29.w,v4.w):sat
-; CHECK-NEXT:     v1 = vmux(q3,v13,v1)
+; CHECK-NEXT:     v30.uh = vpack(v27.w,v4.w):sat
+; CHECK-NEXT:     v1 = vmux(q3,v13,v28)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v31.uh = vpack(v1.w,v0.w):sat
+; CHECK-NEXT:     v31.uh = vpack(v1.w,v29.w):sat
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.uh = vpack(v1.w,v0.w):sat
+; CHECK-NEXT:     v0.uh = vpack(v1.w,v29.w):sat
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1.ub = vpack(v31.h,v30.h):sat
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v0.ub = vpack(v31.h,v0.h):sat
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {

llvm/test/CodeGen/Hexagon/autohvx/int-to-fp.ll

 ; CHECK-NEXT:     v7 = vxor(v7,v7)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r4 = #159
 ; CHECK-NEXT:     v5.uw = vcl0(v2.uw)
 ; CHECK-NEXT:     v6.w = vadd(v6.w,v4.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v27 = vsplat(r4)
+; CHECK-NEXT:     v24 = vsplat(r4)
 ; CHECK-NEXT:     r5 = ##-2147483648
 ; CHECK-NEXT:     v5.w = vadd(v5.w,v4.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v13 = vsplat(r5)
 ; CHECK-NEXT:     v3.w = vasl(v3.w,v6.w)
 ; CHECK-NEXT:     q0 = vcmp.gt(v7.w,v1.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v2.w = vasl(v2.w,v5.w)
-; CHECK-NEXT:     v26 = vmux(q0,v13,v7)
+; CHECK-NEXT:     v23 = vmux(q0,v13,v7)
 ; CHECK-NEXT:     v10.w = vadd(v3.w,v8.w)
 ; CHECK-NEXT:     v11 = vand(v3,v9)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v9 = vand(v2,v9)
 ; CHECK-NEXT:     q1 = vcmp.eq(v11.w,v7.w)
 ; CHECK-NEXT:     v8.w = vadd(v2.w,v8.w)
 ; CHECK-NEXT:     q2 = vcmp.gt(v3.uw,v10.uw)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v12.uw = vlsr(v3.uw,r3)
+; CHECK-NEXT:     v18.uw = vlsr(v10.uw,r3)
 ; CHECK-NEXT:     q3 = vcmp.eq(v9.w,v7.w)
-; CHECK-NEXT:     v22 = vmux(q1,v7,v4)
+; CHECK-NEXT:     v19 = vmux(q1,v7,v4)
 ; CHECK-NEXT:     q1 = vcmp.gt(v2.uw,v8.uw)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v3.uw = vlsr(v10.uw,r3)
-; CHECK-NEXT:     v24 = vmux(q3,v7,v4)
-; CHECK-NEXT:     v23 = vmux(q2,v4,v7)
-; CHECK-NEXT:     v4 = vmux(q1,v4,v7)
+; CHECK-NEXT:     v8.uw = vlsr(v8.uw,r3)
+; CHECK-NEXT:     v9.w = vadd(v18.w,v19.w)
+; CHECK-NEXT:     v21 = vmux(q3,v7,v4)
+; CHECK-NEXT:     v20 = vmux(q2,v4,v7)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v8.uw = vlsr(v8.uw,r3)
-; CHECK-NEXT:     v9.w = vadd(v3.w,v22.w)
-; CHECK-NEXT:     v6.w = vsub(v23.w,v6.w)
-; CHECK-NEXT:     v4.w = vsub(v4.w,v5.w)
+; CHECK-NEXT:     v12.uw = vlsr(v3.uw,r3)
+; CHECK-NEXT:     v4 = vmux(q1,v4,v7)
+; CHECK-NEXT:     v22.w = vadd(v8.w,v21.w)
+; CHECK-NEXT:     v6.w = vsub(v20.w,v6.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = #23
 ; CHECK-NEXT:     v2.uw = vlsr(v2.uw,r3)
-; CHECK-NEXT:     v25.w = vadd(v8.w,v24.w)
-; CHECK-NEXT:     q3 = vcmp.eq(v12.w,v3.w)
-; CHECK-NEXT:     v6.w = vadd(v6.w,v27.w)
+; CHECK-NEXT:     v4.w = vsub(v4.w,v5.w)
+; CHECK-NEXT:     q3 = vcmp.eq(v12.w,v18.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r3 = #23
-; CHECK-NEXT:     v3.uw = vlsr(v3.uw,r2)
+; CHECK-NEXT:     v3.uw = vlsr(v18.uw,r2)
+; CHECK-NEXT:     v4.w = vadd(v4.w,v24.w)
 ; CHECK-NEXT:     q2 = vcmp.eq(v2.w,v8.w)
-; CHECK-NEXT:     v4.w = vadd(v4.w,v27.w)
+; CHECK-NEXT:     v6.w = vadd(v6.w,v24.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v9.uw = vlsr(v9.uw,r2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v28.uw = vlsr(v25.uw,r2)
+; CHECK-NEXT:     v25.uw = vlsr(v22.uw,r2)
 ; CHECK-NEXT:     v3 = vmux(q3,v9,v3)
 ; CHECK-NEXT:     q3 = vcmp.gt(v7.w,v0.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v2.uw = vlsr(v8.uw,r2)
-; CHECK-NEXT:     v30 = vmux(q3,v13,v7)
-; CHECK-NEXT:     v3 = vor(v26,v3)
+; CHECK-NEXT:     v26.uw = vlsr(v8.uw,r2)
+; CHECK-NEXT:     v28 = vmux(q3,v13,v7)
 ; CHECK-NEXT:     q3 = vcmp.eq(v0.w,v7.w)
+; CHECK-NEXT:     v3 = vor(v23,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v29.w = vasl(v6.w,r3)
-; CHECK-NEXT:     v2 = vmux(q2,v28,v2)
+; CHECK-NEXT:     v30.w = vasl(v4.w,r3)
+; CHECK-NEXT:     v2 = vmux(q2,v25,v26)
 ; CHECK-NEXT:     q2 = vcmp.eq(v1.w,v7.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v2.w = vasl(v4.w,r3)
-; CHECK-NEXT:     v31 = vor(v30,v2)
-; CHECK-NEXT:     v3 = vor(v3,v29)
+; CHECK-NEXT:     v27.w = vasl(v6.w,r3)
+; CHECK-NEXT:     v29 = vor(v28,v2)
+; CHECK-NEXT:     v31 = vmux(q2,v7,v31)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1 = vor(v31,v2)
-; CHECK-NEXT:     v3 = vmux(q2,v7,v3)
+; CHECK-NEXT:     v1 = vor(v29,v30)
+; CHECK-NEXT:     v3 = vor(v3,v27)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
+; CHECK-NEXT:     v30.qf32 = vadd(v31.sf,v7.sf)
 ; CHECK-NEXT:     v0 = vmux(q3,v7,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v2.qf32 = vadd(v3.sf,v7.sf)
+; CHECK-NEXT:     v31.qf32 = vadd(v0.sf,v7.sf)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v3.qf32 = vadd(v0.sf,v7.sf)
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.hf = v3:2.qf32
+; CHECK-NEXT:     v0.hf = v31:30.qf32
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v0.h = vdeal(v0.h)
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:     vmem(r1+#0) = v0.new
 ; CHECK-NEXT:    }
   %v0 = load <64 x i32>, ptr %a0, align 128
   %v1 = sitofp <64 x i32> %v0 to <64 x half>

 ; CHECK-NEXT:     v3 = vxor(v3,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r5 = #159
 ; CHECK-NEXT:     r4 = ##-2147483648
 ; CHECK-NEXT:     v4.w = vadd(v4.w,v2.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v28 = vsplat(r5)
-; CHECK-NEXT:     v29 = vsplat(r4)
+; CHECK-NEXT:     v26 = vsplat(r5)
+; CHECK-NEXT:     v27 = vsplat(r4)
 ; CHECK-NEXT:     q3 = vcmp.gt(v3.w,v0.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r3 = #23
+; CHECK-NEXT:     v31.qf32 = vadd(v3.sf,v3.sf)
+; CHECK-NEXT:     v29 = vmux(q3,v27,v3)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1.w = vasl(v1.w,v4.w)
-; CHECK-NEXT:     v31 = vmux(q3,v29,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v5.w = vadd(v1.w,v5.w)
 ; CHECK-NEXT:     v6 = vand(v1,v6)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v7.uw = vlsr(v1.uw,r2)
 ; CHECK-NEXT:     q0 = vcmp.eq(v6.w,v3.w)
 ; CHECK-NEXT:     q1 = vcmp.gt(v1.uw,v5.uw)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r2 = #64
-; CHECK-NEXT:     v1.uw = vlsr(v5.uw,r2)
-; CHECK-NEXT:     v27 = vmux(q0,v3,v2)
+; CHECK-NEXT:     v24.uw = vlsr(v5.uw,r2)
+; CHECK-NEXT:     v25 = vmux(q0,v3,v2)
 ; CHECK-NEXT:     v2 = vmux(q1,v2,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     q3 = vsetq(r2)
-; CHECK-NEXT:     v5.w = vadd(v1.w,v27.w)
+; CHECK-NEXT:     v5.w = vadd(v24.w,v25.w)
 ; CHECK-NEXT:     v2.w = vsub(v2.w,v4.w)
-; CHECK-NEXT:     q2 = vcmp.eq(v7.w,v1.w)
+; CHECK-NEXT:     q2 = vcmp.eq(v7.w,v24.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1.uw = vlsr(v1.uw,r6)
-; CHECK-NEXT:     v2.w = vadd(v2.w,v28.w)
+; CHECK-NEXT:     v1.uw = vlsr(v24.uw,r6)
+; CHECK-NEXT:     v2.w = vadd(v2.w,v26.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v30.uw = vlsr(v5.uw,r6)
+; CHECK-NEXT:     v28.uw = vlsr(v5.uw,r6)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v2.w = vasl(v2.w,r3)
-; CHECK-NEXT:     v1 = vmux(q2,v30,v1)
+; CHECK-NEXT:     v1 = vmux(q2,v28,v1)
 ; CHECK-NEXT:     q2 = vcmp.eq(v0.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1 = vor(v31,v1)
+; CHECK-NEXT:     v1 = vor(v29,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1.qf32 = vadd(v3.sf,v3.sf)
-; CHECK-NEXT:     v0 = vor(v1,v2)
+; CHECK-NEXT:     v30 = vor(v1,v2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0 = vmux(q2,v3,v0)
+; CHECK-NEXT:     v30 = vmux(q2,v3,v30)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.qf32 = vadd(v0.sf,v3.sf)
+; CHECK-NEXT:     v30.qf32 = vadd(v30.sf,v3.sf)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.hf = v1:0.qf32
+; CHECK-NEXT:     v0.hf = v31:30.qf32
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v0.h = vdeal(v0.h)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:     if (q3) vmem(r1+#0) = v0
 ; CHECK-NEXT:    }

 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.w = vadd(v8.w,v6.w)
 ; CHECK-NEXT:     v7 = vand(v8,v7)
 ; CHECK-NEXT:     q0 = vcmp.gt(v5.uw,v11.uw)
 ; CHECK-NEXT:     q1 = vcmp.eq(v13.w,v9.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v28.uw = vlsr(v11.uw,r3)
 ; CHECK-NEXT:     q3 = vcmp.gt(v8.uw,v6.uw)
 ; CHECK-NEXT:     q2 = vcmp.eq(v7.w,v9.w)
-; CHECK-NEXT:     v30 = vmux(q0,v4,v9)
+; CHECK-NEXT:     v28 = vmux(q0,v4,v9)
+; CHECK-NEXT:     v27 = vmux(q1,v9,v4)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.uw = vlsr(v6.uw,r3)
-; CHECK-NEXT:     v29 = vmux(q1,v9,v4)
-; CHECK-NEXT:     v31 = vmux(q3,v4,v9)
+; CHECK-NEXT:     v29 = vmux(q3,v4,v9)
 ; CHECK-NEXT:     v4 = vmux(q2,v9,v4)
+; CHECK-NEXT:     q2 = vcmp.eq(v2.w,v9.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v3.w = vsub(v30.w,v3.w)
-; CHECK-NEXT:     v7.w = vadd(v28.w,v29.w)
-; CHECK-NEXT:     v1.w = vsub(v31.w,v1.w)
+; CHECK-NEXT:     v26.uw = vlsr(v11.uw,r3)
+; CHECK-NEXT:     v1.w = vsub(v29.w,v1.w)
 ; CHECK-NEXT:     v4.w = vadd(v6.w,v4.w)
+; CHECK-NEXT:     v31 = vmux(q2,v9,v31)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v12.uw = vlsr(v5.uw,r3)
-; CHECK-NEXT:     v3.w = vadd(v3.w,v10.w)
+; CHECK-NEXT:     v14.uw = vlsr(v8.uw,r3)
+; CHECK-NEXT:     v7.w = vadd(v26.w,v27.w)
 ; CHECK-NEXT:     v1.w = vadd(v1.w,v10.w)
-; CHECK-NEXT:     q2 = vcmp.eq(v2.w,v9.w)
+; CHECK-NEXT:     v3.w = vsub(v28.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r3 = #23
-; CHECK-NEXT:     v14.uw = vlsr(v8.uw,r3)
-; CHECK-NEXT:     q3 = vcmp.eq(v12.w,v28.w)
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     v5.uw = vlsr(v28.uw,r2)
+; CHECK-NEXT:     v12.uw = vlsr(v5.uw,r3)
 ; CHECK-NEXT:     q1 = vcmp.eq(v14.w,v6.w)
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     v7.uw = vlsr(v7.uw,r2)
+; CHECK-NEXT:     v3.w = vadd(v3.w,v10.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v4.uw = vlsr(v4.uw,r2)
-; CHECK-NEXT:     v5 = vmux(q3,v7,v5)
-; CHECK-NEXT:     q3 = vcmp.eq(v0.w,v9.w)
+; CHECK-NEXT:     q3 = vcmp.eq(v12.w,v26.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.uw = vlsr(v6.uw,r2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v3.w = vasl(v3.w,r3)
-; CHECK-NEXT:     v2 = vmux(q1,v4,v6)
+; CHECK-NEXT:     v5.uw = vlsr(v26.uw,r2)
+; CHECK-NEXT:     v30 = vmux(q1,v4,v6)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1.w = vasl(v1.w,r3)
-; CHECK-NEXT:     v3 = vor(v5,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1 = vor(v2,v1)
-; CHECK-NEXT:     v3 = vmux(q2,v9,v3)
+; CHECK-NEXT:     v7.uw = vlsr(v7.uw,r2)
+; CHECK-NEXT:     v1 = vor(v30,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
+; CHECK-NEXT:     v30.qf32 = vadd(v31.sf,v9.sf)
+; CHECK-NEXT:     v5 = vmux(q3,v7,v5)
+; CHECK-NEXT:     q3 = vcmp.eq(v0.w,v9.w)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v3.w = vasl(v3.w,r3)
 ; CHECK-NEXT:     v0 = vmux(q3,v9,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v2.qf32 = vadd(v3.sf,v9.sf)
+; CHECK-NEXT:     v3 = vor(v5,v3)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v3.qf32 = vadd(v0.sf,v9.sf)
+; CHECK-NEXT:     v31.qf32 = vadd(v0.sf,v9.sf)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.hf = v3:2.qf32
+; CHECK-NEXT:     v0.hf = v31:30.qf32
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v0.h = vdeal(v0.h)
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:     vmem(r1+#0) = v0.new
 ; CHECK-NEXT:    }
   %v0 = load <64 x i32>, ptr %a0, align 128
   %v1 = uitofp <64 x i32> %v0 to <64 x half>

 ; CHECK-NEXT:     v4 = vsplat(r6)
 ; CHECK-NEXT:     r5 = #8
 ; CHECK-NEXT:     r4 = #159
 ; CHECK-NEXT:     v1.w = vadd(v1.w,v3.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v7 = vsplat(r4)
 ; CHECK-NEXT:     r3 = #23
+; CHECK-NEXT:     v31.qf32 = vadd(v2.sf,v2.sf)
 ; CHECK-NEXT:     q2 = vcmp.eq(v0.w,v2.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.w = vasl(v0.w,v1.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v4.w = vadd(v6.w,v4.w)
 ; CHECK-NEXT:     v5 = vand(v6,v5)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v6.uw = vlsr(v6.uw,r5)
 ; CHECK-NEXT:     q0 = vcmp.eq(v5.w,v2.w)
 ; CHECK-NEXT:     q1 = vcmp.gt(v6.uw,v4.uw)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v4.uw = vlsr(v4.uw,r5)
 ; CHECK-NEXT:     v5 = vmux(q0,v2,v3)
 ; CHECK-NEXT:     v3 = vmux(q1,v3,v2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1.w = vsub(v3.w,v1.w)
-; CHECK-NEXT:     v30.w = vadd(v4.w,v5.w)
+; CHECK-NEXT:     v28.w = vadd(v4.w,v5.w)
 ; CHECK-NEXT:     q1 = vcmp.eq(v6.w,v4.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v31.uw = vlsr(v4.uw,r2)
+; CHECK-NEXT:     v29.uw = vlsr(v4.uw,r2)
 ; CHECK-NEXT:     v1.w = vadd(v1.w,v7.w)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r2 = #64
-; CHECK-NEXT:     v3.uw = vlsr(v30.uw,r2)
+; CHECK-NEXT:     v3.uw = vlsr(v28.uw,r2)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v1.w = vasl(v1.w,r3)
 ; CHECK-NEXT:     q3 = vsetq(r2)
-; CHECK-NEXT:     v3 = vmux(q1,v3,v31)
+; CHECK-NEXT:     v3 = vmux(q1,v3,v29)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v1.qf32 = vadd(v2.sf,v2.sf)
-; CHECK-NEXT:     v0 = vor(v3,v1)
+; CHECK-NEXT:     v30 = vor(v3,v1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0 = vmux(q2,v2,v0)
+; CHECK-NEXT:     v30 = vmux(q2,v2,v30)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.qf32 = vadd(v0.sf,v2.sf)
+; CHECK-NEXT:     v30.qf32 = vadd(v30.sf,v2.sf)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     v0.hf = v1:0.qf32
+; CHECK-NEXT:     v0.hf = v31:30.qf32
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     v0.h = vdeal(v0.h)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:     if (q3) vmem(r1+#0) = v0
 ; CHECK-NEXT:    }

llvm/test/CodeGen/Hexagon/autohvx/vmpy-parts.ll

 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     q0 = vcmp.gt(v4.w,v1.w)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v6.uw = vlsr(v2.uw,r2)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     v5 = vdelta(v1,v5)
+; V60-NEXT:     v31 = vmux(q1,v1,v4)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     v1 = vmux(q1,v1,v4)
+; V60-NEXT:     v5 = vdelta(v1,v5)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     if (q0) v1.w += v0.w
+; V60-NEXT:     if (q0) v31.w += v0.w
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v9:8.uw = vmpy(v0.uh,v5.uh)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v9:8.w = vadd(v9.uh,v8.uh)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     v31.w = vadd(v8.w,v6.w)
+; V60-NEXT:     v30.w = vadd(v8.w,v6.w)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v2.w += vasl(v8.w,r2)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     v9.w += vasr(v31.w,r2)
+; V60-NEXT:     v9.w += vasr(v30.w,r2)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v0.w = vadd(v3.w,v9.w)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
-; V60-NEXT:     v3.w = vsub(v0.w,v1.w)
+; V60-NEXT:     v3.w = vsub(v0.w,v31.w)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     v1:0 = vcombine(v3,v2)
 ; V60-NEXT:    }
 ; V60-NEXT:    {
 ; V60-NEXT:     jumpr r31
 ; V60-NEXT:    }
 ;

llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll

entry:
   %0 = tail call i64 asm sideeffect "", "={f1},0"(i64 %cc_dep1)
   ret i64 %0
 }
 
 define void @__int128_and_f(ptr noalias nocapture writeonly sret(i128) align 8 %agg.result, ptr %0) {
 ; Z15-LABEL: __int128_and_f:
 ; Z15:       # %bb.0: # %entry
 ; Z15-NEXT:    vl %v0, 0(%r3), 3
-; Z15-NEXT:    vrepg %v6, %v0, 1
 ; Z15-NEXT:    vlr %v4, %v0
+; Z15-NEXT:    vrepg %v6, %v0, 1
 ; Z15-NEXT:    #APP
 ; Z15-NEXT:    #NO_APP
 ; Z15-NEXT:    vmrhg %v0, %v4, %v6
 ; Z15-NEXT:    vst %v0, 0(%r2), 3
 ; Z15-NEXT:    br %r14
 ;
 ; Z13-LABEL: __int128_and_f:
 ; Z13:       # %bb.0: # %entry

 entry:
   %0 = tail call <2 x i32> asm sideeffect "", "={f5},0"(<2 x i32> %cc_dep1)
   ret <2 x i32> %0
 }
 
 define <4 x i32> @vec128_and_f(<4 x i32> %cc_dep1) {
 ; CHECK-LABEL: vec128_and_f:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vrepg %v3, %v24, 1
 ; CHECK-NEXT:    vlr %v1, %v24
+; CHECK-NEXT:    vrepg %v3, %v24, 1
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vmrhg %v24, %v1, %v3
 ; CHECK-NEXT:    br %r14
 entry:
   %0 = tail call <4 x i32> asm sideeffect "", "={f1},0"(<4 x i32> %cc_dep1)
   ret <4 x i32> %0
 }
 

llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll

 entry:
   %0 = tail call <2 x i32> asm sideeffect "", "=f,0"(<2 x i32> %cc_dep1)
   ret <2 x i32> %0
 }
 
 define <4 x i32> @vec128_and_f(<4 x i32> %cc_dep1) {
 ; CHECK-LABEL: vec128_and_f:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vrepg %v2, %v24, 1
 ; CHECK-NEXT:    vlr %v0, %v24
+; CHECK-NEXT:    vrepg %v2, %v24, 1
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vmrhg %v24, %v0, %v2
 ; CHECK-NEXT:    br %r14
 entry:
   %0 = tail call <4 x i32> asm sideeffect "", "=f,0"(<4 x i32> %cc_dep1)
   ret <4 x i32> %0
 }
 

llvm/test/CodeGen/Thumb2/mve-vldst4.ll

 ; CHECK-NEXT:    vins.f16 s11, s8
 ; CHECK-NEXT:    vmovx.f16 s25, s20
 ; CHECK-NEXT:    vins.f16 s24, s6
 ; CHECK-NEXT:    vmovx.f16 s6, s22
 ; CHECK-NEXT:    vmovx.f16 s1, s15
 ; CHECK-NEXT:    vmovx.f16 s8, s13
 ; CHECK-NEXT:    vins.f16 s20, s22
 ; CHECK-NEXT:    vins.f16 s16, s18
-; CHECK-NEXT:    vins.f16 s25, s6
 ; CHECK-NEXT:    vmov.f32 s2, s5
+; CHECK-NEXT:    vins.f16 s25, s6
 ; CHECK-NEXT:    vmov.f32 s3, s17
 ; CHECK-NEXT:    vins.f16 s0, s15
 ; CHECK-NEXT:    vmovx.f16 s9, s21
 ; CHECK-NEXT:    vins.f16 s8, s1
 ; CHECK-NEXT:    vmovx.f16 s1, s23
 ; CHECK-NEXT:    vins.f16 s12, s14
 ; CHECK-NEXT:    vins.f16 s21, s23
 ; CHECK-NEXT:    vmov.f32 s14, s4

llvm/test/CodeGen/Thumb2/mve-vst3.ll

 ; CHECK-NEXT:    vmovx.f16 s2, s8
 ; CHECK-NEXT:    vins.f16 s8, s0
 ; CHECK-NEXT:    vins.f16 s14, s2
 ; CHECK-NEXT:    vmovx.f16 s2, s24
 ; CHECK-NEXT:    vstrw.32 q3, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q3, [r0]
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov.f32 s16, s12
-; CHECK-NEXT:    vins.f16 s16, s24
 ; CHECK-NEXT:    vmov.f32 s0, s13
+; CHECK-NEXT:    vins.f16 s16, s24
 ; CHECK-NEXT:    vmov.16 q4[4], r2
 ; CHECK-NEXT:    vins.f16 s0, s25
 ; CHECK-NEXT:    vmov.f32 s19, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s12
 ; CHECK-NEXT:    vmov.f64 d15, d13
 ; CHECK-NEXT:    vmov.f32 s17, s13
 ; CHECK-NEXT:    vmov.f32 s24, s16
 ; CHECK-NEXT:    vmov.f64 d13, d9