[AMDGPU] Allow to use a whole register file on gfx90a for VGPRs

[rampitec]

Summary

In a kernel which does not have calls or AGPR usage we can allocate
the whole vector register budget for VGPRs and have no AGPRs as
long as VGPRs stay addressable (i.e. below 256).

[AlexVlx]

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

953

Why not just make this a manipulator (delete the trailing const) and get rid of mutable? It's just a stylistic issue so feel free to ignore.

[rampitec]

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

953

Because getReservedRegs takes const MF. So it is either mutable or I will have to use const_cast somewhere. I prefer mutable in this case as it is just lazily computed and cached rather than actually changing the MFI. The effects of this mutation are unobservable outside of the accessor.

[kerbowa]

LGTM

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

private:
   unsigned GITPtrHigh;
 
   unsigned HighBitsOf32BitAddress;
   unsigned GDSSize;
 
   // Current recorded maximum possible occupancy.
   unsigned Occupancy;
 
+  mutable Optional<bool> UsesAGPRs;
+
   MCPhysReg getNextUserSGPR() const;
 
   MCPhysReg getNextSystemSGPR() const;
 
 public:
   struct SpilledReg {
     Register VGPR;
     int Lane = -1;
if (Occupancy > Limit)
       Occupancy = Limit;
   }
 
   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
     if (Occupancy < Limit)
       Occupancy = Limit;
     limitOccupancy(MF);
   }
+
+  // \returns true if a function needs or may need AGPRs.
+  bool usesAGPRs(const MachineFunction &MF) const;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

if (i->VGPR == ReservedVGPR) {
         MBB.sortUniqueLiveIns();
       }
       this->VGPRReservedForSGPRSpill = AMDGPU::NoRegister;
       return true;
     }
   }
   return false;
 }
+
+bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const {
+  if (UsesAGPRs)
+    return *UsesAGPRs;
+
+  if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) ||
+      MF.getFrameInfo().hasCalls()) {
+    UsesAGPRs = true;
+    return true;
+  }
+
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
+    const Register Reg = Register::index2VirtReg(I);
+    const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
+    if (RC && SIRegisterInfo::isAGPRClass(RC)) {
+      UsesAGPRs = true;
+      return true;
+    }
+  }
+
+  for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
+    if (MRI.isPhysRegUsed(Reg)) {
+      UsesAGPRs = true;
+      return true;
+    }
+  }
+
+  UsesAGPRs = false;
+  return false;
+}

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
   unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
   unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
   for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) {
     unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
     reserveRegisterTuples(Reserved, Reg);
   }
 
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF);
-  // TODO: In an entry function without calls and AGPRs used it is possible
-  //       to use the whole register budget for VGPRs. Even more it shall
-  //       be possible to estimate maximum AGPR/VGPR pressure and split
-  //       register file accordingly.
-  if (ST.hasGFX90AInsts())
-    MaxNumVGPRs /= 2;
+  unsigned MaxNumAGPRs = MaxNumVGPRs;
   unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
+
+  if (ST.hasGFX90AInsts()) {
+    // In an entry function without calls and AGPRs used it is possible to use
+    // the whole register budget for VGPRs.
+
+    // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and
+    //       split register file accordingly.
+    if (MFI->usesAGPRs(MF)) {
+      MaxNumVGPRs /= 2;
+      MaxNumAGPRs = MaxNumVGPRs;
+    } else {
+      if (MaxNumVGPRs > TotalNumVGPRs) {
+        MaxNumAGPRs = MaxNumVGPRs - TotalNumVGPRs;
+        MaxNumVGPRs = TotalNumVGPRs;
+      } else
+        MaxNumAGPRs = 0;
+    }
+  }
+
   for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) {
     unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i);
     reserveRegisterTuples(Reserved, Reg);
-    Reg = AMDGPU::AGPR_32RegClass.getRegister(i);
+  }
+
+  for (unsigned i = MaxNumAGPRs; i < TotalNumVGPRs; ++i) {
+    unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i);
     reserveRegisterTuples(Reserved, Reg);
   }
 
   for (auto Reg : AMDGPU::SReg_32RegClass) {
     Reserved.set(getSubReg(Reg, AMDGPU::hi16));
     Register Low = getSubReg(Reg, AMDGPU::lo16);
     // This is to prevent BB vcc liveness errors.
     if (!AMDGPU::SGPR_LO16RegClass.contains(Low))
       Reserved.set(Low);
   }
 
   for (auto Reg : AMDGPU::AGPR_32RegClass) {
     Reserved.set(getSubReg(Reg, AMDGPU::hi16));
   }
 
   // Reserve all the rest AGPRs if there are no instructions to use it.
   if (!ST.hasMAIInsts()) {
     for (unsigned i = 0; i < MaxNumVGPRs; ++i) {
       unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i);
       reserveRegisterTuples(Reserved, Reg);
     }
   }
 
-  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-
   Register ScratchRSrcReg = MFI->getScratchRSrcReg();
   if (ScratchRSrcReg != AMDGPU::NoRegister) {
     // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need
     // to spill.
     // TODO: May need to reserve a VGPR if doing LDS spilling.
     reserveRegisterTuples(Reserved, ScratchRSrcReg);
   }
 

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll

 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_lshl_b64 s[8:9], s[2:3], s6
 ; GFX90A-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
 ; GFX90A-NEXT:    s_ashr_i32 s10, s3, 31
 ; GFX90A-NEXT:    s_add_u32 s2, s2, s10
 ; GFX90A-NEXT:    s_mov_b32 s11, s10
 ; GFX90A-NEXT:    s_addc_u32 s3, s3, s10
 ; GFX90A-NEXT:    s_xor_b64 s[12:13], s[2:3], s[10:11]
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s12
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v1, s12
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v2, s13
 ; GFX90A-NEXT:    s_mov_b32 s19, 0xcf800000
 ; GFX90A-NEXT:    s_sub_u32 s14, 0, s12
 ; GFX90A-NEXT:    s_subb_u32 s15, 0, s13
-; GFX90A-NEXT:    v_mac_f32_e32 v0, s16, v1
-; GFX90A-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s16, v2
+; GFX90A-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
-; GFX90A-NEXT:    v_mul_f32_e32 v0, s17, v0
-; GFX90A-NEXT:    v_mul_f32_e32 v1, s18, v0
-; GFX90A-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX90A-NEXT:    v_mac_f32_e32 v0, s19, v1
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX90A-NEXT:    v_mul_f32_e32 v1, s17, v1
+; GFX90A-NEXT:    v_mul_f32_e32 v2, s18, v1
+; GFX90A-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s19, v2
 ; GFX90A-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX90A-NEXT:    v_mul_hi_u32 v3, s14, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s14, v1
-; GFX90A-NEXT:    v_mul_lo_u32 v2, s15, v0
-; GFX90A-NEXT:    v_add_u32_e32 v3, v3, v5
-; GFX90A-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s14, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v0, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v4, s14, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s14, v2
+; GFX90A-NEXT:    v_mul_lo_u32 v3, s15, v1
+; GFX90A-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX90A-NEXT:    v_add_u32_e32 v3, v4, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v6, s14, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v1, v3
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v4, v1, v3
 ; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v7, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v8, v1, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v1, v6
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v8, v2, v6
+; GFX90A-NEXT:    v_mul_lo_u32 v6, v2, v6
 ; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v2, v5, vcc
-; GFX90A-NEXT:    v_add_co_u32_e64 v0, s[0:1], v0, v3
-; GFX90A-NEXT:    v_addc_co_u32_e64 v3, vcc, v1, v5, s[0:1]
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s14, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s14, v0
-; GFX90A-NEXT:    v_add_u32_e32 v6, v7, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v7, s15, v0
-; GFX90A-NEXT:    v_add_u32_e32 v6, v6, v7
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s14, v0
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v2, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v8, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v3, v2, v3
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v4, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v6, v5, vcc
+; GFX90A-NEXT:    v_add_co_u32_e64 v1, s[0:1], v1, v3
+; GFX90A-NEXT:    v_addc_co_u32_e64 v3, vcc, v2, v4, s[0:1]
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s14, v3
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s14, v1
+; GFX90A-NEXT:    v_add_u32_e32 v5, v7, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v7, s15, v1
+; GFX90A-NEXT:    v_add_u32_e32 v5, v5, v7
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s14, v1
 ; GFX90A-NEXT:    v_mul_hi_u32 v9, v3, v8
 ; GFX90A-NEXT:    v_mul_lo_u32 v10, v3, v8
-; GFX90A-NEXT:    v_mul_lo_u32 v12, v0, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v8, v0, v8
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v0, v6
+; GFX90A-NEXT:    v_mul_lo_u32 v12, v1, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v1, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v10
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v5
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v3, v3, v6
+; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v3, v3, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v8, v3
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, v2, v7, vcc
-; GFX90A-NEXT:    v_add_u32_e32 v1, v1, v5
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v6, v7, vcc
+; GFX90A-NEXT:    v_add_u32_e32 v2, v2, v4
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_ashr_i32 s14, s5, 31
-; GFX90A-NEXT:    v_addc_co_u32_e64 v1, vcc, v1, v6, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e64 v2, vcc, v2, v5, s[0:1]
 ; GFX90A-NEXT:    s_add_u32 s0, s4, s14
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v3
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v3
 ; GFX90A-NEXT:    s_mov_b32 s15, s14
 ; GFX90A-NEXT:    s_addc_u32 s1, s5, s14
-; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], s[0:1], s[14:15]
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s4, v1
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GFX90A-NEXT:    v_mul_hi_u32 v3, s4, v1
-; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s4, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v3, s4, v2
+; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, v5, v4
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s5, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v0, s5, v0
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v5, v0
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s5, v1
-; GFX90A-NEXT:    v_addc_co_u32_e32 v0, vcc, v3, v7, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v4, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s5, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, s5, v1
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v3, s12, v1
-; GFX90A-NEXT:    v_mul_hi_u32 v5, s12, v0
-; GFX90A-NEXT:    v_add_u32_e32 v3, v5, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s13, v0
-; GFX90A-NEXT:    v_add_u32_e32 v3, v3, v5
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s12, v0
-; GFX90A-NEXT:    v_sub_u32_e32 v5, s5, v3
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v4, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s5, v2
+; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v7, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v2, s5, v2
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v2
+; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, v6, v3, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v3, s12, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v4, s12, v1
+; GFX90A-NEXT:    v_add_u32_e32 v3, v4, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s13, v1
+; GFX90A-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s12, v1
+; GFX90A-NEXT:    v_sub_u32_e32 v4, s5, v3
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, s13
-; GFX90A-NEXT:    v_sub_co_u32_e32 v6, vcc, s4, v6
-; GFX90A-NEXT:    v_subb_co_u32_e64 v5, s[0:1], v5, v7, vcc
-; GFX90A-NEXT:    v_subrev_co_u32_e64 v7, s[0:1], s12, v6
-; GFX90A-NEXT:    v_subbrev_co_u32_e64 v5, s[0:1], 0, v5, s[0:1]
-; GFX90A-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v5
+; GFX90A-NEXT:    v_sub_co_u32_e32 v5, vcc, s4, v5
+; GFX90A-NEXT:    v_subb_co_u32_e64 v4, s[0:1], v4, v7, vcc
+; GFX90A-NEXT:    v_subrev_co_u32_e64 v7, s[0:1], s12, v5
+; GFX90A-NEXT:    v_subbrev_co_u32_e64 v4, s[0:1], 0, v4, s[0:1]
+; GFX90A-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v7
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v5
-; GFX90A-NEXT:    v_cndmask_b32_e64 v5, v8, v7, s[0:1]
-; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GFX90A-NEXT:    v_cndmask_b32_e64 v5, 1, 2, s[0:1]
+; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v4
+; GFX90A-NEXT:    v_cndmask_b32_e64 v4, v8, v7, s[0:1]
+; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
+; GFX90A-NEXT:    v_cndmask_b32_e64 v4, 1, 2, s[0:1]
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, s5
-; GFX90A-NEXT:    v_add_co_u32_e64 v5, s[0:1], v0, v5
+; GFX90A-NEXT:    v_add_co_u32_e64 v4, s[0:1], v1, v4
 ; GFX90A-NEXT:    v_subb_co_u32_e32 v3, vcc, v8, v3, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e64 v7, s[0:1], 0, v2, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s13, v3
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s12, v6
+; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s12, v5
 ; GFX90A-NEXT:    s_xor_b64 s[0:1], s[14:15], s[10:11]
 ; GFX90A-NEXT:    s_ashr_i32 s4, s9, 31
-; GFX90A-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v3
 ; GFX90A-NEXT:    s_add_u32 s8, s8, s4
-; GFX90A-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v3, v8, v5, vcc
 ; GFX90A-NEXT:    s_mov_b32 s5, s4
 ; GFX90A-NEXT:    s_addc_u32 s9, s9, s4
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
 ; GFX90A-NEXT:    s_xor_b64 s[8:9], s[8:9], s[4:5]
-; GFX90A-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GFX90A-NEXT:    v_cvt_f32_u32_e32 v3, s8
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v5, s9
-; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
-; GFX90A-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v4, s9
+; GFX90A-NEXT:    v_cndmask_b32_e32 v2, v2, v7, vcc
+; GFX90A-NEXT:    v_xor_b32_e32 v1, s0, v1
+; GFX90A-NEXT:    v_xor_b32_e32 v5, s1, v2
+; GFX90A-NEXT:    v_mac_f32_e32 v3, s16, v4
+; GFX90A-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX90A-NEXT:    v_subrev_co_u32_e32 v2, vcc, s0, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v7, s1
+; GFX90A-NEXT:    v_mul_f32_e32 v1, s17, v4
+; GFX90A-NEXT:    v_mul_f32_e32 v4, s18, v1
+; GFX90A-NEXT:    v_trunc_f32_e32 v4, v4
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s19, v4
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; GFX90A-NEXT:    s_sub_u32 s10, 0, s8
-; GFX90A-NEXT:    v_mac_f32_e32 v3, s16, v5
-; GFX90A-NEXT:    v_rcp_f32_e32 v3, v3
-; GFX90A-NEXT:    v_xor_b32_e32 v1, s1, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v6, s1
-; GFX90A-NEXT:    v_subrev_co_u32_e32 v0, vcc, s0, v0
-; GFX90A-NEXT:    v_mul_f32_e32 v3, s17, v3
-; GFX90A-NEXT:    v_mul_f32_e32 v5, s18, v3
-; GFX90A-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX90A-NEXT:    v_mac_f32_e32 v3, s19, v5
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GFX90A-NEXT:    v_subb_co_u32_e32 v3, vcc, v5, v7, vcc
 ; GFX90A-NEXT:    s_subb_u32 s11, 0, s9
-; GFX90A-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s10, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s10, v5
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s11, v3
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s10, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s10, v4
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s11, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v7, v7, v8
-; GFX90A-NEXT:    v_add_u32_e32 v6, v7, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v9, s10, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v3, v9
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v6
+; GFX90A-NEXT:    v_add_u32_e32 v5, v7, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v9, s10, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v8, v1, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v1, v9
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v5, v9
-; GFX90A-NEXT:    v_mul_lo_u32 v9, v5, v9
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GFX90A-NEXT:    v_mul_lo_u32 v9, v4, v9
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v9
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v5, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v4, v5
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v11, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v10, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX90A-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v2, v8, vcc
-; GFX90A-NEXT:    v_add_co_u32_e64 v3, s[0:1], v3, v6
-; GFX90A-NEXT:    v_addc_co_u32_e64 v6, vcc, v5, v7, s[0:1]
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s10, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v9, s10, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v10, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v4, v5
+; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v7, v5
+; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v6, v8, vcc
+; GFX90A-NEXT:    v_add_co_u32_e64 v1, s[0:1], v1, v5
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, vcc, v4, v7, s[0:1]
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s10, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v9, s10, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v8, v9, v8
-; GFX90A-NEXT:    v_mul_lo_u32 v9, s11, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v9, s11, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v8, v8, v9
-; GFX90A-NEXT:    v_mul_lo_u32 v10, s10, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v6, v10
-; GFX90A-NEXT:    v_mul_lo_u32 v12, v6, v10
-; GFX90A-NEXT:    v_mul_lo_u32 v14, v3, v8
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v3, v10
-; GFX90A-NEXT:    v_mul_hi_u32 v13, v3, v8
+; GFX90A-NEXT:    v_mul_lo_u32 v10, s10, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v5, v10
+; GFX90A-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GFX90A-NEXT:    v_mul_lo_u32 v14, v1, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v1, v10
+; GFX90A-NEXT:    v_mul_hi_u32 v13, v1, v8
 ; GFX90A-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v14
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
 ; GFX90A-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v12
-; GFX90A-NEXT:    v_mul_hi_u32 v9, v6, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v9, v5, v8
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v10, vcc, v13, v11, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v6, v8
-; GFX90A-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v2, v9, vcc
-; GFX90A-NEXT:    v_add_u32_e32 v5, v5, v7
+; GFX90A-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v5, v8
+; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v10, v5
+; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v6, v9, vcc
+; GFX90A-NEXT:    v_add_u32_e32 v4, v4, v7
 ; GFX90A-NEXT:    s_ashr_i32 s10, s7, 31
-; GFX90A-NEXT:    v_addc_co_u32_e64 v5, vcc, v5, v8, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e64 v4, vcc, v4, v8, s[0:1]
 ; GFX90A-NEXT:    s_add_u32 s0, s6, s10
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v6
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v5
 ; GFX90A-NEXT:    s_mov_b32 s11, s10
 ; GFX90A-NEXT:    s_addc_u32 s1, s7, s10
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX90A-NEXT:    s_xor_b64 s[6:7], s[0:1], s[10:11]
-; GFX90A-NEXT:    v_mul_lo_u32 v7, s6, v5
-; GFX90A-NEXT:    v_mul_hi_u32 v8, s6, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s6, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v7, s6, v4
+; GFX90A-NEXT:    v_mul_hi_u32 v8, s6, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s6, v4
 ; GFX90A-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v7
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v9, s7, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v3, s7, v3
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v7, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v8, s7, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v9, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, v8, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s7, v5
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v6, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s8, v2
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s8, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v9, s7, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v7, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v8, s7, v4
+; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v9, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v8, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s7, v4
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v4
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v6, v5, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s8, v4
+; GFX90A-NEXT:    v_mul_hi_u32 v6, s8, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v5, v6, v5
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s9, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v6, s9, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v5, v5, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v7, s8, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v7, s8, v1
 ; GFX90A-NEXT:    v_sub_u32_e32 v6, s7, v5
 ; GFX90A-NEXT:    v_mov_b32_e32 v8, s9
 ; GFX90A-NEXT:    v_sub_co_u32_e32 v7, vcc, s6, v7
 ; GFX90A-NEXT:    v_subb_co_u32_e64 v6, s[0:1], v6, v8, vcc
 ; GFX90A-NEXT:    v_subrev_co_u32_e64 v8, s[0:1], s8, v7
 ; GFX90A-NEXT:    v_subbrev_co_u32_e64 v6, s[0:1], 0, v6, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v6
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v8
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
 ; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v6
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v6, v9, v8, s[0:1]
 ; GFX90A-NEXT:    v_mov_b32_e32 v9, s7
 ; GFX90A-NEXT:    v_subb_co_u32_e32 v5, vcc, v9, v5, vcc
 ; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s9, v5
 ; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
 ; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s8, v7
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v6, 1, 2, s[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v5
-; GFX90A-NEXT:    v_add_co_u32_e64 v6, s[0:1], v3, v6
+; GFX90A-NEXT:    v_add_co_u32_e64 v6, s[0:1], v1, v6
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v5, v9, v7, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e64 v8, s[0:1], 0, v2, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e64 v8, s[0:1], 0, v4, s[0:1]
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
-; GFX90A-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
 ; GFX90A-NEXT:    s_xor_b64 s[0:1], s[10:11], s[4:5]
-; GFX90A-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
-; GFX90A-NEXT:    v_xor_b32_e32 v3, s0, v3
-; GFX90A-NEXT:    v_xor_b32_e32 v5, s1, v2
+; GFX90A-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX90A-NEXT:    v_xor_b32_e32 v1, s0, v1
+; GFX90A-NEXT:    v_xor_b32_e32 v5, s1, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, s1
-; GFX90A-NEXT:    v_subrev_co_u32_e32 v2, vcc, s0, v3
-; GFX90A-NEXT:    v_subb_co_u32_e32 v3, vcc, v5, v6, vcc
-; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
+; GFX90A-NEXT:    v_subrev_co_u32_e32 v4, vcc, s0, v1
+; GFX90A-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v6, vcc
+; GFX90A-NEXT:    global_store_dwordx4 v0, v[2:5], s[2:3]
 ; GFX90A-NEXT:    s_endpgm
   %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
   %r = sdiv <2 x i64> %x, %shl.y
   store <2 x i64> %r, <2 x i64> addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @srem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {

 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_lshl_b64 s[10:11], s[2:3], s6
 ; GFX90A-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
 ; GFX90A-NEXT:    s_ashr_i32 s4, s3, 31
 ; GFX90A-NEXT:    s_add_u32 s2, s2, s4
 ; GFX90A-NEXT:    s_mov_b32 s5, s4
 ; GFX90A-NEXT:    s_addc_u32 s3, s3, s4
 ; GFX90A-NEXT:    s_xor_b64 s[12:13], s[2:3], s[4:5]
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s12
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v1, s12
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v2, s13
 ; GFX90A-NEXT:    s_mov_b32 s19, 0xcf800000
 ; GFX90A-NEXT:    s_sub_u32 s2, 0, s12
 ; GFX90A-NEXT:    s_subb_u32 s3, 0, s13
-; GFX90A-NEXT:    v_mac_f32_e32 v0, s16, v1
-; GFX90A-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s16, v2
+; GFX90A-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x24
 ; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
-; GFX90A-NEXT:    v_mul_f32_e32 v0, s17, v0
-; GFX90A-NEXT:    v_mul_f32_e32 v1, s18, v0
-; GFX90A-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX90A-NEXT:    v_mac_f32_e32 v0, s19, v1
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX90A-NEXT:    v_mul_f32_e32 v1, s17, v1
+; GFX90A-NEXT:    v_mul_f32_e32 v2, s18, v1
+; GFX90A-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s19, v2
 ; GFX90A-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v2, v2
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_ashr_i32 s14, s5, 31
 ; GFX90A-NEXT:    s_mov_b32 s15, s14
-; GFX90A-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s2, v1
-; GFX90A-NEXT:    v_mul_lo_u32 v2, s3, v0
-; GFX90A-NEXT:    v_add_u32_e32 v3, v3, v5
-; GFX90A-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s2, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v0, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v4, s2, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GFX90A-NEXT:    v_mul_lo_u32 v3, s3, v1
+; GFX90A-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX90A-NEXT:    v_add_u32_e32 v3, v4, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v6, s2, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v1, v3
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v4, v1, v3
 ; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v7, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v8, v1, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v1, v6
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v8, v2, v6
+; GFX90A-NEXT:    v_mul_lo_u32 v6, v2, v6
 ; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v1, v2
-; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v2, v5, vcc
-; GFX90A-NEXT:    v_add_co_u32_e64 v0, s[0:1], v0, v3
-; GFX90A-NEXT:    v_addc_co_u32_e64 v3, vcc, v1, v5, s[0:1]
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s2, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s2, v0
-; GFX90A-NEXT:    v_add_u32_e32 v6, v7, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v7, s3, v0
-; GFX90A-NEXT:    v_add_u32_e32 v6, v6, v7
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v0
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v2, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v8, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v3, v2, v3
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v4, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v6, v5, vcc
+; GFX90A-NEXT:    v_add_co_u32_e64 v1, s[0:1], v1, v3
+; GFX90A-NEXT:    v_addc_co_u32_e64 v3, vcc, v2, v4, s[0:1]
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s2, v3
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s2, v1
+; GFX90A-NEXT:    v_add_u32_e32 v5, v7, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v7, s3, v1
+; GFX90A-NEXT:    v_add_u32_e32 v5, v5, v7
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v1
 ; GFX90A-NEXT:    v_mul_hi_u32 v9, v3, v8
 ; GFX90A-NEXT:    v_mul_lo_u32 v10, v3, v8
-; GFX90A-NEXT:    v_mul_lo_u32 v12, v0, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v8, v0, v8
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v0, v6
+; GFX90A-NEXT:    v_mul_lo_u32 v12, v1, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v1, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v10
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v5
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v3, v3, v6
+; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v3, v3, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v8, v3
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, v2, v7, vcc
-; GFX90A-NEXT:    v_add_u32_e32 v1, v1, v5
-; GFX90A-NEXT:    v_addc_co_u32_e64 v1, vcc, v1, v6, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v6, v7, vcc
+; GFX90A-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX90A-NEXT:    v_addc_co_u32_e64 v2, vcc, v2, v5, s[0:1]
 ; GFX90A-NEXT:    s_add_u32 s0, s4, s14
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v3
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v3
 ; GFX90A-NEXT:    s_addc_u32 s1, s5, s14
-; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], s[0:1], s[14:15]
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s4, v1
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GFX90A-NEXT:    v_mul_hi_u32 v3, s4, v1
-; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s4, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v3, s4, v2
+; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, v5, v4
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s5, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v0, s5, v0
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v5, v0
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s5, v1
-; GFX90A-NEXT:    v_addc_co_u32_e32 v0, vcc, v3, v7, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v4, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s5, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, s5, v1
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v1
-; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v4, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s5, v2
+; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v7, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v2, s5, v2
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v2
+; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, v6, v3, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v2, s12, v2
+; GFX90A-NEXT:    v_mul_hi_u32 v3, s12, v1
+; GFX90A-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX90A-NEXT:    v_mul_lo_u32 v3, s13, v1
+; GFX90A-NEXT:    v_add_u32_e32 v2, v2, v3
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, s12, v1
-; GFX90A-NEXT:    v_mul_hi_u32 v3, s12, v0
-; GFX90A-NEXT:    v_add_u32_e32 v1, v3, v1
-; GFX90A-NEXT:    v_mul_lo_u32 v3, s13, v0
-; GFX90A-NEXT:    v_add_u32_e32 v1, v1, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v0, s12, v0
-; GFX90A-NEXT:    v_sub_u32_e32 v3, s5, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v5, s13
-; GFX90A-NEXT:    v_sub_co_u32_e32 v0, vcc, s4, v0
-; GFX90A-NEXT:    v_subb_co_u32_e64 v3, s[0:1], v3, v5, vcc
-; GFX90A-NEXT:    v_subrev_co_u32_e64 v6, s[0:1], s12, v0
+; GFX90A-NEXT:    v_sub_u32_e32 v3, s5, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v4, s13
+; GFX90A-NEXT:    v_sub_co_u32_e32 v1, vcc, s4, v1
+; GFX90A-NEXT:    v_subb_co_u32_e64 v3, s[0:1], v3, v4, vcc
+; GFX90A-NEXT:    v_subrev_co_u32_e64 v5, s[0:1], s12, v1
 ; GFX90A-NEXT:    v_subbrev_co_u32_e64 v7, s[2:3], 0, v3, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v7
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[2:3]
-; GFX90A-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v6
-; GFX90A-NEXT:    v_subb_co_u32_e64 v3, s[0:1], v3, v5, s[0:1]
+; GFX90A-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v5
+; GFX90A-NEXT:    v_subb_co_u32_e64 v3, s[0:1], v3, v4, s[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
 ; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v7
-; GFX90A-NEXT:    v_subrev_co_u32_e64 v5, s[0:1], s12, v6
+; GFX90A-NEXT:    v_subrev_co_u32_e64 v4, s[0:1], s12, v5
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[2:3]
 ; GFX90A-NEXT:    v_subbrev_co_u32_e64 v3, s[0:1], 0, v3, s[0:1]
 ; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GFX90A-NEXT:    v_cndmask_b32_e64 v5, v6, v5, s[0:1]
-; GFX90A-NEXT:    v_mov_b32_e32 v6, s5
-; GFX90A-NEXT:    v_subb_co_u32_e32 v1, vcc, v6, v1, vcc
-; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
+; GFX90A-NEXT:    v_cndmask_b32_e64 v4, v5, v4, s[0:1]
+; GFX90A-NEXT:    v_mov_b32_e32 v5, s5
+; GFX90A-NEXT:    v_subb_co_u32_e32 v2, vcc, v5, v2, vcc
+; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s13, v2
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[0:1]
-; GFX90A-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
+; GFX90A-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s12, v1
 ; GFX90A-NEXT:    s_ashr_i32 s0, s11, 31
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
+; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v2
 ; GFX90A-NEXT:    s_add_u32 s2, s10, s0
-; GFX90A-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
 ; GFX90A-NEXT:    s_mov_b32 s1, s0
 ; GFX90A-NEXT:    s_addc_u32 s3, s11, s0
-; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], s[2:3], s[0:1]
-; GFX90A-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
-; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX90A-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v5, s5
-; GFX90A-NEXT:    v_xor_b32_e32 v0, s14, v0
-; GFX90A-NEXT:    s_sub_u32 s2, 0, s4
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v4, s5
 ; GFX90A-NEXT:    v_xor_b32_e32 v1, s14, v1
-; GFX90A-NEXT:    v_mac_f32_e32 v3, s16, v5
-; GFX90A-NEXT:    v_rcp_f32_e32 v3, v3
-; GFX90A-NEXT:    v_mov_b32_e32 v6, s14
-; GFX90A-NEXT:    v_subrev_co_u32_e32 v0, vcc, s14, v0
-; GFX90A-NEXT:    v_mul_f32_e32 v3, s17, v3
-; GFX90A-NEXT:    v_mul_f32_e32 v5, s18, v3
-; GFX90A-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX90A-NEXT:    v_mac_f32_e32 v3, s19, v5
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GFX90A-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GFX90A-NEXT:    v_xor_b32_e32 v5, s14, v2
+; GFX90A-NEXT:    v_subrev_co_u32_e32 v2, vcc, s14, v1
+; GFX90A-NEXT:    v_mac_f32_e32 v3, s16, v4
+; GFX90A-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX90A-NEXT:    v_mov_b32_e32 v7, s14
+; GFX90A-NEXT:    s_sub_u32 s2, 0, s4
+; GFX90A-NEXT:    v_subb_co_u32_e32 v3, vcc, v5, v7, vcc
+; GFX90A-NEXT:    v_mul_f32_e32 v1, s17, v4
+; GFX90A-NEXT:    v_mul_f32_e32 v4, s18, v1
+; GFX90A-NEXT:    v_trunc_f32_e32 v4, v4
+; GFX90A-NEXT:    v_mac_f32_e32 v1, s19, v4
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX90A-NEXT:    v_cvt_u32_f32_e32 v4, v4
 ; GFX90A-NEXT:    s_subb_u32 s3, 0, s5
-; GFX90A-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v7, s2, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v5
-; GFX90A-NEXT:    v_mul_lo_u32 v6, s3, v3
+; GFX90A-NEXT:    s_ashr_i32 s10, s7, 31
+; GFX90A-NEXT:    v_mul_hi_u32 v7, s2, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v4
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s3, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v7, v7, v8
-; GFX90A-NEXT:    v_add_u32_e32 v6, v7, v6
-; GFX90A-NEXT:    v_mul_lo_u32 v9, s2, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v3, v9
-; GFX90A-NEXT:    v_mul_hi_u32 v7, v3, v6
+; GFX90A-NEXT:    v_add_u32_e32 v5, v7, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v9, s2, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v8, v1, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v1, v9
+; GFX90A-NEXT:    v_mul_hi_u32 v7, v1, v5
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v5, v9
-; GFX90A-NEXT:    v_mul_lo_u32 v9, v5, v9
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v4, v9
+; GFX90A-NEXT:    v_mul_lo_u32 v9, v4, v9
 ; GFX90A-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v9
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v5, v6
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v4, v5
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v11, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v10, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX90A-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v2, v8, vcc
-; GFX90A-NEXT:    v_add_co_u32_e64 v3, s[0:1], v3, v6
-; GFX90A-NEXT:    v_addc_co_u32_e64 v6, vcc, v5, v7, s[0:1]
-; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v6
-; GFX90A-NEXT:    v_mul_hi_u32 v9, s2, v3
+; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v10, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v4, v5
+; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v7, v5
+; GFX90A-NEXT:    v_addc_co_u32_e32 v7, vcc, v6, v8, vcc
+; GFX90A-NEXT:    v_add_co_u32_e64 v1, s[0:1], v1, v5
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, vcc, v4, v7, s[0:1]
+; GFX90A-NEXT:    v_mul_lo_u32 v8, s2, v5
+; GFX90A-NEXT:    v_mul_hi_u32 v9, s2, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v8, v9, v8
-; GFX90A-NEXT:    v_mul_lo_u32 v9, s3, v3
+; GFX90A-NEXT:    v_mul_lo_u32 v9, s3, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v8, v8, v9
-; GFX90A-NEXT:    v_mul_lo_u32 v10, s2, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v11, v6, v10
-; GFX90A-NEXT:    v_mul_lo_u32 v12, v6, v10
-; GFX90A-NEXT:    v_mul_lo_u32 v14, v3, v8
-; GFX90A-NEXT:    v_mul_hi_u32 v10, v3, v10
-; GFX90A-NEXT:    v_mul_hi_u32 v13, v3, v8
+; GFX90A-NEXT:    v_mul_lo_u32 v10, s2, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v11, v5, v10
+; GFX90A-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GFX90A-NEXT:    v_mul_lo_u32 v14, v1, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v10, v1, v10
+; GFX90A-NEXT:    v_mul_hi_u32 v13, v1, v8
 ; GFX90A-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v14
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
 ; GFX90A-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v12
-; GFX90A-NEXT:    v_mul_hi_u32 v9, v6, v8
+; GFX90A-NEXT:    v_mul_hi_u32 v9, v5, v8
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v10, vcc, v13, v11, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v6, v6, v8
-; GFX90A-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v2, v9, vcc
-; GFX90A-NEXT:    v_add_u32_e32 v5, v5, v7
-; GFX90A-NEXT:    s_ashr_i32 s10, s7, 31
-; GFX90A-NEXT:    v_addc_co_u32_e64 v5, vcc, v5, v8, s[0:1]
+; GFX90A-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v5, v5, v8
+; GFX90A-NEXT:    v_add_co_u32_e32 v5, vcc, v10, v5
+; GFX90A-NEXT:    v_addc_co_u32_e32 v8, vcc, v6, v9, vcc
+; GFX90A-NEXT:    v_add_u32_e32 v4, v4, v7
+; GFX90A-NEXT:    v_addc_co_u32_e64 v4, vcc, v4, v8, s[0:1]
 ; GFX90A-NEXT:    s_add_u32 s0, s6, s10
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v6
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v5
 ; GFX90A-NEXT:    s_mov_b32 s11, s10
 ; GFX90A-NEXT:    s_addc_u32 s1, s7, s10
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX90A-NEXT:    s_xor_b64 s[6:7], s[0:1], s[10:11]
-; GFX90A-NEXT:    v_mul_lo_u32 v7, s6, v5
-; GFX90A-NEXT:    v_mul_hi_u32 v8, s6, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v6, s6, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v7, s6, v4
+; GFX90A-NEXT:    v_mul_hi_u32 v8, s6, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s6, v4
 ; GFX90A-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v7
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX90A-NEXT:    v_mul_hi_u32 v9, s7, v3
-; GFX90A-NEXT:    v_mul_lo_u32 v3, s7, v3
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v7, v3
-; GFX90A-NEXT:    v_mul_hi_u32 v8, s7, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v9, vcc
-; GFX90A-NEXT:    v_addc_co_u32_e32 v6, vcc, v8, v4, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s7, v5
-; GFX90A-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v5
-; GFX90A-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v6, vcc
-; GFX90A-NEXT:    v_mul_lo_u32 v2, s4, v2
-; GFX90A-NEXT:    v_mul_hi_u32 v5, s4, v3
-; GFX90A-NEXT:    v_add_u32_e32 v2, v5, v2
-; GFX90A-NEXT:    v_mul_lo_u32 v5, s5, v3
-; GFX90A-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX90A-NEXT:    v_mul_lo_u32 v3, s4, v3
-; GFX90A-NEXT:    v_sub_u32_e32 v5, s7, v2
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX90A-NEXT:    v_mul_hi_u32 v9, s7, v1
+; GFX90A-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v7, v1
+; GFX90A-NEXT:    v_mul_hi_u32 v8, s7, v4
+; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v9, vcc
+; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, v8, v0, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s7, v4
+; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v4
+; GFX90A-NEXT:    v_addc_co_u32_e32 v4, vcc, v6, v5, vcc
+; GFX90A-NEXT:    v_mul_lo_u32 v4, s4, v4
+; GFX90A-NEXT:    v_mul_hi_u32 v5, s4, v1
+; GFX90A-NEXT:    v_add_u32_e32 v4, v5, v4
+; GFX90A-NEXT:    v_mul_lo_u32 v5, s5, v1
+; GFX90A-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX90A-NEXT:    v_mul_lo_u32 v1, s4, v1
+; GFX90A-NEXT:    v_sub_u32_e32 v5, s7, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, s5
-; GFX90A-NEXT:    v_sub_co_u32_e32 v3, vcc, s6, v3
+; GFX90A-NEXT:    v_sub_co_u32_e32 v1, vcc, s6, v1
 ; GFX90A-NEXT:    v_subb_co_u32_e64 v5, s[0:1], v5, v6, vcc
-; GFX90A-NEXT:    v_subrev_co_u32_e64 v7, s[0:1], s4, v3
+; GFX90A-NEXT:    v_subrev_co_u32_e64 v7, s[0:1], s4, v1
 ; GFX90A-NEXT:    v_subbrev_co_u32_e64 v8, s[2:3], 0, v5, s[0:1]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[2:3], s5, v8
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
 ; GFX90A-NEXT:    v_cmp_le_u32_e64 s[2:3], s4, v7
 ; GFX90A-NEXT:    v_subb_co_u32_e64 v5, s[0:1], v5, v6, s[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[2:3]
 ; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[2:3], s5, v8
 ; GFX90A-NEXT:    v_subrev_co_u32_e64 v6, s[0:1], s4, v7
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v9, v9, v10, s[2:3]
 ; GFX90A-NEXT:    v_subbrev_co_u32_e64 v5, s[0:1], 0, v5, s[0:1]
 ; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v9
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v6, v7, v6, s[0:1]
 ; GFX90A-NEXT:    v_mov_b32_e32 v7, s7
-; GFX90A-NEXT:    v_subb_co_u32_e32 v2, vcc, v7, v2, vcc
-; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s5, v2
+; GFX90A-NEXT:    v_subb_co_u32_e32 v4, vcc, v7, v4, vcc
+; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s5, v4
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
+; GFX90A-NEXT:    v_cmp_le_u32_e32 vcc, s4, v1
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s5, v2
+; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, s5, v4
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
-; GFX90A-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
-; GFX90A-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX90A-NEXT:    v_xor_b32_e32 v3, s10, v3
-; GFX90A-NEXT:    v_xor_b32_e32 v5, s10, v2
+; GFX90A-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX90A-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX90A-NEXT:    v_xor_b32_e32 v1, s10, v1
+; GFX90A-NEXT:    v_xor_b32_e32 v5, s10, v4
 ; GFX90A-NEXT:    v_mov_b32_e32 v6, s10
-; GFX90A-NEXT:    v_subrev_co_u32_e32 v2, vcc, s10, v3
-; GFX90A-NEXT:    v_subb_co_u32_e32 v3, vcc, v5, v6, vcc
-; GFX90A-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
+; GFX90A-NEXT:    v_subrev_co_u32_e32 v4, vcc, s10, v1
+; GFX90A-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v6, vcc
+; GFX90A-NEXT:    global_store_dwordx4 v0, v[2:5], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
   %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
   %r = srem <2 x i64> %x, %shl.y
   store <2 x i64> %r, <2 x i64> addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll

 ; GFX90A: NumVgprs: 128
 ; GFX90A: NumAgprs: 128
 ; GFX90A: TotalNumVgprs: 256
 ; GFX10WGP-WAVE32: NumVgprs: 256
 ; GFX10WGP-WAVE64: NumVgprs: 256
 ; GFX10CU-WAVE32: NumVgprs: 128
 ; GFX10CU-WAVE64: NumVgprs: 128
 define amdgpu_kernel void @f512() #512 {
+  call void @foo()
   call void @use256vgprs()
   ret void
 }
 attributes #512 = { nounwind "amdgpu-flat-work-group-size"="512,512" }
 
 ; GCN-LABEL: {{^}}f1024:
 ; GFX9: NumVgprs: 64
 ; GFX90A: NumVgprs: 64
 ; GFX90A: NumAgprs: 64
 ; GFX90A: TotalNumVgprs: 128
 ; GFX10WGP-WAVE32: NumVgprs: 128
 ; GFX10WGP-WAVE64: NumVgprs: 128
 ; GFX10CU-WAVE32: NumVgprs: 64
 ; GFX10CU-WAVE64: NumVgprs: 64
 define amdgpu_kernel void @f1024() #1024 {
+  call void @foo()
   call void @use256vgprs()
   ret void
 }
+
 attributes #1024 = { nounwind "amdgpu-flat-work-group-size"="1024,1024" }
+
+declare void @foo()

llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll

 ; -enable-misched=false makes the register usage more predictable
 ; -regalloc=fast just makes the test run faster
-; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
 ; RUN: llc -march=amdgcn -mcpu=gfx90a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false --sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
 
 define internal void @use256vgprs() {
   %v0 = call i32 asm sideeffect "; def $0", "=v"()
   %v1 = call i32 asm sideeffect "; def $0", "=v"()
   %v2 = call i32 asm sideeffect "; def $0", "=v"()
   %v3 = call i32 asm sideeffect "; def $0", "=v"()
   %v4 = call i32 asm sideeffect "; def $0", "=v"()
   %v5 = call i32 asm sideeffect "; def $0", "=v"()
define internal void @use256vgprs() {
   call void asm sideeffect "; use $0", "v"(i32 %v251)
   call void asm sideeffect "; use $0", "v"(i32 %v252)
   call void asm sideeffect "; use $0", "v"(i32 %v253)
   call void asm sideeffect "; use $0", "v"(i32 %v254)
   call void asm sideeffect "; use $0", "v"(i32 %v255)
   ret void
 }
 
-; GCN-LABEL: {{^}}f256:
-; GFX9: NumVgprs: 256
+define internal void @use512vgprs() {
+  %v0 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v1 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v2 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v3 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v4 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v5 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v6 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  %v7 = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  call void @use256vgprs()
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v0)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v1)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v2)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v3)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v4)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v5)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v6)
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %v7)
+  ret void
+}
+
+define void @foo() #0 {
+  ret void
+}
+
+attributes #0 = { noinline }
+
+; GCN-LABEL: {{^}}k256_w8:
+; GFX90A: NumVgprs: 32
+; GFX90A: NumAgprs: 32
+; GFX90A: TotalNumVgprs: 64
+define amdgpu_kernel void @k256_w8() #2568 {
+  call void @foo()
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k256_w8_no_agprs:
+; GFX90A: NumVgprs: 64
+; GFX90A: NumAgprs: 0
+; GFX90A: TotalNumVgprs: 64
+define amdgpu_kernel void @k256_w8_no_agprs() #2568 {
+  call void @use256vgprs()
+  ret void
+}
+
+attributes #2568 = { nounwind "amdgpu-flat-work-group-size"="256,256" "amdgpu-waves-per-eu"="8" }
+
+; GCN-LABEL: {{^}}k256_w4:
+; GFX90A: NumVgprs: 64
+; GFX90A: NumAgprs: 64
+; GFX90A: TotalNumVgprs: 128
+define amdgpu_kernel void @k256_w4() #2564 {
+  call void @foo()
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k256_w4_no_agprs:
+; GFX90A: NumVgprs: 128
+; GFX90A: NumAgprs: 0
+; GFX90A: TotalNumVgprs: 128
+define amdgpu_kernel void @k256_w4_no_agprs() #2564 {
+  call void @use256vgprs()
+  ret void
+}
+
+attributes #2564 = { nounwind "amdgpu-flat-work-group-size"="256,256" "amdgpu-waves-per-eu"="4" }
+
+; GCN-LABEL: {{^}}k256_w2:
+; GFX90A: NumVgprs: 128
+; GFX90A: NumAgprs: 128
+; GFX90A: TotalNumVgprs: 256
+define amdgpu_kernel void @k256_w2() #2562 {
+  call void @foo()
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k256_w2_no_agprs:
 ; GFX90A: NumVgprs: 256
 ; GFX90A: NumAgprs: 0
 ; GFX90A: TotalNumVgprs: 256
-; GFX10WGP-WAVE32: NumVgprs: 256
-; GFX10WGP-WAVE64: NumVgprs: 256
-; GFX10CU-WAVE32: NumVgprs: 256
-; GFX10CU-WAVE64: NumVgprs: 256
-define amdgpu_kernel void @f256() #256 {
+define amdgpu_kernel void @k256_w2_no_agprs() #2562 {
   call void @use256vgprs()
   ret void
 }
-attributes #256 = { nounwind "amdgpu-flat-work-group-size"="256,256" }
 
-; GCN-LABEL: {{^}}f512:
-; GFX9: NumVgprs: 128
+attributes #2562 = { nounwind "amdgpu-flat-work-group-size"="256,256" "amdgpu-waves-per-eu"="2" }
+
+; GCN-LABEL: {{^}}k256_w1:
+; GFX90A: NumVgprs: 256
+; GFX90A: NumAgprs: 256
+; GFX90A: TotalNumVgprs: 512
+define amdgpu_kernel void @k256_w1() #2561 {
+  call void @foo()
+  call void @use512vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k256_w1_no_agprs:
+; GFX90A: NumVgprs: 256
+; GFX90A: NumAgprs: 256
+; GFX90A: TotalNumVgprs: 512
+define amdgpu_kernel void @k256_w1_no_agprs() #2561 {
+  call void @use512vgprs()
+  ret void
+}
+
+attributes #2561 = { nounwind "amdgpu-flat-work-group-size"="256,256" "amdgpu-waves-per-eu"="1" }
+
+; GCN-LABEL: {{^}}k512_no_agprs:
+; GFX90A: NumVgprs: 256
+; GFX90A: NumAgprs: 0
+; GFX90A: TotalNumVgprs: 256
+define amdgpu_kernel void @k512_no_agprs() #512 {
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k512_call:
+; GFX90A: NumVgprs: 128
+; GFX90A: NumAgprs: 128
+; GFX90A: TotalNumVgprs: 256
+define amdgpu_kernel void @k512_call() #512 {
+  call void @foo()
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k512_virtual_agpr:
 ; GFX90A: NumVgprs: 128
 ; GFX90A: NumAgprs: 128
 ; GFX90A: TotalNumVgprs: 256
-; GFX10WGP-WAVE32: NumVgprs: 256
-; GFX10WGP-WAVE64: NumVgprs: 256
-; GFX10CU-WAVE32: NumVgprs: 128
-; GFX10CU-WAVE64: NumVgprs: 128
-define amdgpu_kernel void @f512() #512 {
+define amdgpu_kernel void @k512_virtual_agpr() #512 {
+  %a0 = call i32 asm sideeffect "; def $0", "=a"()
   call void @use256vgprs()
   ret void
 }
+
+; GCN-LABEL: {{^}}k512_physical_agpr:
+; GFX90A: NumVgprs: 128
+; GFX90A: NumAgprs: 128
+; GFX90A: TotalNumVgprs: 256
+define amdgpu_kernel void @k512_physical_agpr() #512 {
+  call void asm sideeffect "", "~{a8}" ()
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}f512:
+; GFX90A: NumVgprs: 12{{[0-9]}}
+; GFX90A: NumAgprs: {{[1-9]}}
+define void @f512() #512 {
+  call void @use256vgprs()
+  ret void
+}
+
 attributes #512 = { nounwind "amdgpu-flat-work-group-size"="512,512" }
 
-; GCN-LABEL: {{^}}f1024:
-; GFX9: NumVgprs: 64
+; GCN-LABEL: {{^}}k1024:
+; GFX90A: NumVgprs: 128
+; GFX90A: NumAgprs: 0
+; GFX90A: TotalNumVgprs: 128
+define amdgpu_kernel void @k1024() #1024 {
+  call void @use256vgprs()
+  ret void
+}
+
+; GCN-LABEL: {{^}}k1024_call:
 ; GFX90A: NumVgprs: 64
 ; GFX90A: NumAgprs: 64
 ; GFX90A: TotalNumVgprs: 128
-; GFX10WGP-WAVE32: NumVgprs: 128
-; GFX10WGP-WAVE64: NumVgprs: 128
-; GFX10CU-WAVE32: NumVgprs: 64
-; GFX10CU-WAVE64: NumVgprs: 64
-define amdgpu_kernel void @f1024() #1024 {
+define amdgpu_kernel void @k1024_call() #1024 {
+  call void @foo()
   call void @use256vgprs()
   ret void
 }
+
 attributes #1024 = { nounwind "amdgpu-flat-work-group-size"="1024,1024" }