[AMDGPU] Select s_cselect

[piotr]

Summary

Add patterns to select s_cselect in the isel.

Handle more cases of implicit SCC accesses in si-fix-sgpr-copies
to allow new patterns to work.

[arsenm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

609

This should not be changed. We now report zero extended booleans since a few months ago

642

Should be 1

5577–5579

I'm not super comfortable with a search for the def like this, but there probably isn't a better option. I also care less about this code now that GlobalISel does this correctly.

Can you check if SCCSource is undef? I think that's the case this might crash on since there will be no def to find. The other cases should be caught by the verifier

6243–6244

Probably need to note this is really the expected SCC users to handle, since it's more restricted than checking if there's an scc use

6264

Braces

llvm/lib/Target/AMDGPU/SOPInstructions.td

470

I have a patch for GlobalISel to move select matching into patterns, which will be incompatible with this since the type of SCC is switched to i32. I guess this doesn't matter, since we're manually selecting it there already.

[piotr]

Addressed review comments.

[piotr]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

609

Thanks, I was confused by this, mainly because this global change here did not cause any issues in my tests.

I rewrote this part and moved the relevant functionality to lowerSelect.

5577–5579

Added the check.

Yes, this patch is not as neat as I would like due to the extra handling of SCC.

llvm/lib/Target/AMDGPU/SOPInstructions.td

470

Sounds good.

Btw, some s_cselect's are already generated for the global-isel at this point. It was just the old isel that did not have any support for that.

[arsenm]

Can you also add a MIR test that at least covers the undef case?

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

5578

Maybe add a search range limit?

[piotr]

Added MIR test to test undef SCC, added minor tweaks in lowerSelect and rebased.

[piotr]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

5578

Not sure - the code relies on finding SCC def, and by construction (.td pattern) the def is very close to S_CSELECT's implicit use anyway.

llvm/lib/Target/AMDGPU/SIInstrInfo.h

MachineOperand buildExtractSubRegOrImm(MachineBasicBlock::iterator MI,
                                          unsigned SubIdx,
                                          const TargetRegisterClass *SubRC) const;
 private:
   void swapOperands(MachineInstr &Inst) const;
 
   bool moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
                         MachineDominatorTree *MDT = nullptr) const;
 
+  void lowerSelect(SetVectorType &Worklist, MachineInstr &Inst,
+                   MachineDominatorTree *MDT = nullptr) const;
+
   void lowerScalarAbs(SetVectorType &Worklist,
                       MachineInstr &Inst) const;
 
   void lowerScalarXnor(SetVectorType &Worklist,
                        MachineInstr &Inst) const;
 
   void splitScalarNotBinop(SetVectorType &Worklist,
                            MachineInstr &Inst,

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

BuildMI(MBB, MI, DL, get(Opc), DestReg)
       .addReg(SrcReg, getKillRegState(KillSrc));
     return;
   }
 
   if (RC == &AMDGPU::SReg_32_XM0RegClass ||
       RC == &AMDGPU::SReg_32RegClass) {
     if (SrcReg == AMDGPU::SCC) {
       BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
           .addImm(1)
           .addImm(0);
       return;
     }
 
     if (DestReg == AMDGPU::VCC_LO) {
       if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
         BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
           .addReg(SrcReg, getKillRegState(KillSrc));
if (RC == &AMDGPU::SReg_32_XM0RegClass ||
     }
 
     BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
             .addReg(SrcReg, getKillRegState(KillSrc));
     return;
   }
 
   if (RC == &AMDGPU::SReg_64RegClass) {
+    if (SrcReg == AMDGPU::SCC) {
+      BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
+          .addImm(1)
+          .addImm(0);
+      return;
+    }
+
     if (DestReg == AMDGPU::VCC) {
       if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
         BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
           .addReg(SrcReg, getKillRegState(KillSrc));
       } else {
         // FIXME: Hack until VReg_1 removed.
         assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
         BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
if (RC == &AMDGPU::SReg_64RegClass) {
     }
 
     BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
             .addReg(SrcReg, getKillRegState(KillSrc));
     return;
   }
 
   if (DestReg == AMDGPU::SCC) {
-    assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
-    BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
+    unsigned Opcode;
+    if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
+      Opcode = AMDGPU::S_CMP_LG_U32;
+    } else {
+      assert(AMDGPU::SReg_64RegClass.contains(SrcReg));
+      Opcode = AMDGPU::S_CMP_LG_U64;
+    }
+
+    BuildMI(MBB, MI, DL, get(Opcode))
         .addReg(SrcReg, getKillRegState(KillSrc))
         .addImm(0);
+
     return;
   }
 
   if (RC == &AMDGPU::AGPR_32RegClass) {
     assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
            AMDGPU::SReg_32RegClass.contains(SrcReg) ||
            AMDGPU::AGPR_32RegClass.contains(SrcReg));
     if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
case AMDGPU::S_USUBO_PSEUDO: {
       legalizeOperands(*NewInstr, MDT);
 
       MRI.replaceRegWith(Dest0.getReg(), DestReg);
       addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI,
                                    Worklist);
       Inst.eraseFromParent();
     }
       continue;
+
+    case AMDGPU::S_CSELECT_B32:
+    case AMDGPU::S_CSELECT_B64:
+      lowerSelect(Worklist, Inst, MDT);
+      Inst.eraseFromParent();
+      continue;
     }
 
     if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
       // We cannot move this instruction to the VALU, so we should try to
       // legalize its operands instead.
       legalizeOperands(Inst, MDT);
       continue;
     }
if (ST.hasAddNoCarry()) {
 
     addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
     return true;
   }
 
   return false;
 }
 
+void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst,
+                              MachineDominatorTree *MDT) const {
+
+  MachineBasicBlock &MBB = *Inst.getParent();
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+  MachineBasicBlock::iterator MII = Inst;
+  DebugLoc DL = Inst.getDebugLoc();
+
+  MachineOperand &Dest = Inst.getOperand(0);
+  MachineOperand &Src0 = Inst.getOperand(1);
+  MachineOperand &Src1 = Inst.getOperand(2);
+  MachineOperand &Cond = Inst.getOperand(3);
+
+  Register SCCSource = Cond.getReg();
+  // Find SCC def, and if that is a copy (SCC = COPY reg) then use reg instead.
+  if (!Cond.isUndef()) {
+    for (MachineInstr &CandI :
+         make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)),
+                    Inst.getParent()->rend())) {
+      if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) !=
+          -1) {
+        if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
+          SCCSource = CandI.getOperand(1).getReg();
+        }
+        break;
+      }
+    }
+  }
+
+  // If this is a trivial select where the condition is effectively not SCC
+  // (SCCSource is a source of copy to SCC), then the select is semantically
+  // equivalent to copying SCCSource. Hence, there is no need to create
+  // V_CNDMASK, we can just use that and bail out.
+  if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) &&
+      Src1.isImm() && (Src1.getImm() == 0)) {
+    MRI.replaceRegWith(Dest.getReg(), SCCSource);
+    return;
+  }
+
+  const TargetRegisterClass *TC = ST.getWavefrontSize() == 64
+                                      ? &AMDGPU::SReg_64_XEXECRegClass
+                                      : &AMDGPU::SReg_32_XM0_XEXECRegClass;
+  Register CopySCC = MRI.createVirtualRegister(TC);
+
+  if (SCCSource == AMDGPU::SCC) {
+    // Insert a trivial select instead of creating a copy, because a copy from
+    // SCC would semantically mean just copying a single bit, but we may need
+    // the result to be a vector condition mask that needs preserving.
+    unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64
+                                                    : AMDGPU::S_CSELECT_B32;
+    auto NewSelect =
+        BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0);
+    NewSelect->getOperand(3).setIsUndef(Cond.isUndef());
+  } else {
+    BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource);
+  }
+
+  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+  auto UpdatedInst =
+      BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg)
+          .addImm(0)
+          .add(Src1) // False
+          .addImm(0)
+          .add(Src0) // True
+          .addReg(CopySCC);
+
+  MRI.replaceRegWith(Dest.getReg(), ResultReg);
+  legalizeOperands(*UpdatedInst, MDT);
+  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
+}
+
 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
                                  MachineInstr &Inst) const {
   MachineBasicBlock &MBB = *Inst.getParent();
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
   MachineBasicBlock::iterator MII = Inst;
   DebugLoc DL = Inst.getDebugLoc();
 
   MachineOperand &Dest = Inst.getOperand(0);
void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
   MachineOperand &Dest = Inst.getOperand(0);
   MRI.replaceRegWith(Dest.getReg(), ResultReg);
   addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
 }
 
 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
                                                MachineInstr &SCCDefInst,
                                                SetVectorType &Worklist) const {
+  bool SCCUsedImplicitly = false;
+
   // Ensure that def inst defines SCC, which is still live.
   assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
          !Op.isDead() && Op.getParent() == &SCCDefInst);
   SmallVector<MachineInstr *, 4> CopyToDelete;
   // This assumes that all the users of SCC are in the same block
   // as the SCC def.
   for (MachineInstr &MI : // Skip the def inst itself.
        make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)),
                   SCCDefInst.getParent()->end())) {
     // Check if SCC is used first.
     if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) {
       if (MI.isCopy()) {
         MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
         unsigned DestReg = MI.getOperand(0).getReg();
-        SmallVector<MachineInstr *, 4> Users;
+
         for (auto &User : MRI.use_nodbg_instructions(DestReg)) {
           if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) ||
               (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) {
-            Users.push_back(&User);
+            User.getOperand(4).setReg(RI.getVCC());
             Worklist.insert(&User);
+          } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) {
+            User.getOperand(5).setReg(RI.getVCC());
+            // No need to add to Worklist.
           }
         }
-        for (auto &U : Users)
-          U->getOperand(4).setReg(RI.getVCC());
         CopyToDelete.push_back(&MI);
-      } else
+      } else {
+        if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
+            MI.getOpcode() == AMDGPU::S_CSELECT_B64) {
+          // This is an implicit use of SCC and it is really expected by
+          // the SCC users to handle.
+          // We cannot preserve the edge to the user so add the explicit
+          // copy: SCC = COPY VCC.
+          // The copy will be cleaned up during the processing of the user
+          // in lowerSelect.
+          SCCUsedImplicitly = true;
+        }
+
         Worklist.insert(&MI);
       }
+    }
     // Exit if we find another SCC def.
     if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
       break;
   }
   for (auto &Copy : CopyToDelete)
     Copy->eraseFromParent();
+
+  if (SCCUsedImplicitly) {
+    BuildMI(*SCCDefInst.getParent(), std::next(SCCDefInst.getIterator()),
+            SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC)
+        .addReg(RI.getVCC());
+  }
 }
 
 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
   const MachineInstr &Inst) const {
   const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
 
   switch (Inst.getOpcode()) {
   // For target instructions, getOpRegClass just returns the virtual register

llvm/lib/Target/AMDGPU/SOPInstructions.td

def S_MAX_I32 : SOP2_32 <"s_max_i32",
   [(set i32:$sdst, (smax i32:$src0, i32:$src1))]
 >;
 def S_MAX_U32 : SOP2_32 <"s_max_u32",
   [(set i32:$sdst, (umax i32:$src0, i32:$src1))]
 >;
 } // End isCommutable = 1
 } // End Defs = [SCC]
 
+class SelectPat<SDPatternOperator select> : PatFrag <
+  (ops node:$src1, node:$src2),
+  (select SCC, $src1, $src2),
+  [{ return N->getOperand(0)->hasOneUse() && !N->isDivergent(); }]
+>;
+
+let Uses = [SCC], AddedComplexity = 20 in {
+  def S_CSELECT_B64 : SOP2_64 <"s_cselect_b64",
+    [(set i64:$sdst, (SelectPat<select> i64:$src0, i64:$src1))]
+  >;
+
+  def S_CSELECT_B32 : SOP2_32 <"s_cselect_b32",
+    [(set i32:$sdst, (SelectPat<select> i32:$src0, i32:$src1))]
+>;
 
-let Uses = [SCC] in {
-  def S_CSELECT_B32 : SOP2_32 <"s_cselect_b32">;
-  def S_CSELECT_B64 : SOP2_64 <"s_cselect_b64">;
 } // End Uses = [SCC]
 
 let Defs = [SCC] in {
 let isCommutable = 1 in {
 def S_AND_B32 : SOP2_32 <"s_and_b32",
   [(set i32:$sdst, (UniformBinFrag<and> i32:$src0, i32:$src1))]
 >;
 

llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll

 entry:
   %0 = getelementptr i32, i32 addrspace(3)* %in, i32 16385
   %1 = load i32, i32 addrspace(3)* %0
   store i32 %1, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}null_32bit_lds_ptr:
-; SI: v_cmp_ne_u32
+; SI: s_cmp_lg_u32
 ; SI-NOT: v_cmp_ne_u32
-; SI: v_cndmask_b32
+; SI: s_cselect_b32
 define amdgpu_kernel void @null_32bit_lds_ptr(i32 addrspace(1)* %out, i32 addrspace(3)* %lds) nounwind {
   %cmp = icmp ne i32 addrspace(3)* %lds, null
   %x = select i1 %cmp, i32 123, i32 456
   store i32 %x, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}mul_32bit_ptr:

llvm/test/CodeGen/AMDGPU/addrspacecast.ll

 }
 
 ; HSA-LABEL: {{^}}use_flat_to_group_addrspacecast:
 ; HSA: enable_sgpr_private_segment_buffer = 1
 ; HSA: enable_sgpr_dispatch_ptr = 0
 ; HSA: enable_sgpr_queue_ptr = 0
 
 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}}
-; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
-; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]]
-; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]]
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}}
+; CI-DAG: v_cmp_ne_u64_e64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
+; CI-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO]]:[[CMP_HI]]{{\]}}, 0
+; GFX9-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, 0
+; HSA-DAG: s_cselect_b32 s[[PTR_LO]], s[[PTR_LO]], -1
+; HSA-DAG: v_mov_b32_e32 [[CASTPTR:v[0-9]+]], s[[PTR_LO]]
 ; HSA: ds_write_b32 [[CASTPTR]], v[[K]]
 define amdgpu_kernel void @use_flat_to_group_addrspacecast(i32* %ptr) #0 {
   %ftos = addrspacecast i32* %ptr to i32 addrspace(3)*
   store volatile i32 0, i32 addrspace(3)* %ftos
   ret void
 }
 
 ; HSA-LABEL: {{^}}use_flat_to_private_addrspacecast:
 ; HSA: enable_sgpr_private_segment_buffer = 1
 ; HSA: enable_sgpr_dispatch_ptr = 0
 ; HSA: enable_sgpr_queue_ptr = 0
 
 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}}
-; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
-; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]]
-; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]]
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}}
+; CI-DAG: v_cmp_ne_u64_e64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}}
+; CI-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO]]:[[CMP_HI]]{{\]}}, 0
+; GFX9-DAG: s_cmp_lg_u64 s{{\[}}[[CMP_LO:[0-9]+]]:[[CMP_HI:[0-9]+]]{{\]}}, 0
+; HSA-DAG: s_cselect_b32 s[[PTR_LO]], s[[PTR_LO]], -1
+; HSA-DAG: v_mov_b32_e32 [[CASTPTR:v[0-9]+]], s[[PTR_LO]]
 ; HSA: buffer_store_dword v[[K]], [[CASTPTR]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}
 define amdgpu_kernel void @use_flat_to_private_addrspacecast(i32* %ptr) #0 {
   %ftos = addrspacecast i32* %ptr to i32 addrspace(5)*
   store volatile i32 0, i32 addrspace(5)* %ftos
   ret void
 }
 
 ; HSA-LABEL: {{^}}use_flat_to_global_addrspacecast:

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll

 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i32 s1, s0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
 ; GCN-NEXT:    s_sext_i32_i16 s0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {

 ; CHECK-NEXT:    store i16 [[TMP23]], i16 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s4, 16
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
-; GCN-NEXT:    s_sext_i32_i16 s3, s4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-NEXT:    s_ashr_i32 s5, s4, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s5
+; GCN-NEXT:    s_sext_i32_i16 s2, s4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT:    s_xor_b32 s2, s2, s5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s3, s3, 30
-; GCN-NEXT:    s_or_b32 s3, s3, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    s_ashr_i32 s2, s2, 30
+; GCN-NEXT:    s_or_b32 s6, s2, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[2:3], 0
+; GCN-NEXT:    s_cselect_b32 s2, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s5
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
 ; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
 }

 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_bfe_i32 s1, s0, 0x80008
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
 ; GCN-NEXT:    s_sext_i32_i8 s0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @srem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {

 ; CHECK-NEXT:    [[TMP22:%.*]] = ashr i32 [[TMP21]], 24
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP22]] to i8
 ; CHECK-NEXT:    store i8 [[TMP23]], i8 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i8:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bfe_i32 s1, s0, 0x80008
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
-; GCN-NEXT:    s_sext_i32_i8 s3, s0
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    s_bfe_i32 s0, s2, 0x80008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT:    s_sext_i32_i8 s1, s2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_lshr_b32 s3, s2, 8
+; GCN-NEXT:    s_or_b32 s6, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_lshr_b32 s2, s0, 8
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @udiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x, <4 x i32> %y) {

 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sext_i32_i16 s8, s2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_sext_i32_i16 s9, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s2, s2, 16
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    s_ashr_i32 s2, s2, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 16
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    s_sext_i32_i16 s2, s3
+; GCN-NEXT:    v_mul_f32_e32 v3, v1, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
-; GCN-NEXT:    s_sext_i32_i16 s0, s3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_mad_f32 v1, -v3, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    s_sext_i32_i16 s2, s1
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s8, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v3
+; GCN-NEXT:    s_sext_i32_i16 s0, s1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
-; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    s_ashr_i32 s0, s3, 16
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
+; GCN-NEXT:    v_mad_f32 v1, -v4, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    s_ashr_i32 s1, s1, 16
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
-; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    s_ashr_i32 s2, s3, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s0, v4
+; GCN-NEXT:    s_ashr_i32 s0, s1, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_mad_f32 v4, -v5, v0, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v5
 ; GCN-NEXT:    s_mov_b32 s0, 0xffff
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
-; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v3
+; GCN-NEXT:    v_and_b32_e32 v2, s0, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <4 x i16> %x, %y
   store <4 x i16> %r, <4 x i16> addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x, <4 x i16> %y) {

 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sext_i32_i16 s8, s2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_sext_i32_i16 s9, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s8, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
 ; GCN-NEXT:    s_ashr_i32 s2, s2, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
 ; GCN-NEXT:    s_xor_b32 s8, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    s_or_b32 s10, s8, 1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_mov_b32_e32 v4, s8
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v2|, |v1|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v3
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s2
 ; GCN-NEXT:    s_sext_i32_i16 s2, s3
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
 ; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s0, v1
 ; GCN-NEXT:    s_sext_i32_i16 s0, s1
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    s_xor_b32 s0, s0, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    s_ashr_i32 s0, s3, 16
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v2|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s0, v4
+; GCN-NEXT:    s_ashr_i32 s0, s3, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    s_ashr_i32 s2, s1, 16
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    s_ashr_i32 s8, s1, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s8
+; GCN-NEXT:    s_xor_b32 s2, s8, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GCN-NEXT:    s_ashr_i32 s2, s2, 30
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
-; GCN-NEXT:    s_xor_b32 s3, s2, s0
-; GCN-NEXT:    s_ashr_i32 s3, s3, 30
+; GCN-NEXT:    s_or_b32 s9, s2, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    s_or_b32 s3, s3, 1
-; GCN-NEXT:    v_mov_b32_e32 v6, s3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v4|, |v2|
+; GCN-NEXT:    s_cmp_lg_u64 s[2:3], 0
+; GCN-NEXT:    s_cselect_b32 s2, s9, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s2, v5
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
 ; GCN-NEXT:    s_mov_b32 s0, 0xffff
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
 ; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s8, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem <4 x i16> %x, %y

 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_bfe_i32 s1, s0, 0x30008
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
 ; GCN-NEXT:    s_bfe_i32 s0, s0, 0x30000
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
 }
 

 ; CHECK-NEXT:    [[TMP22:%.*]] = ashr i32 [[TMP21]], 29
 ; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP22]] to i3
 ; CHECK-NEXT:    store i3 [[TMP23]], i3 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: srem_i3:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bfe_i32 s1, s0, 0x30008
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
-; GCN-NEXT:    s_bfe_i32 s3, s0, 0x30000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
-; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    s_bfe_i32 s0, s2, 0x30008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT:    s_bfe_i32 s1, s2, 0x30000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_lshr_b32 s3, s2, 8
+; GCN-NEXT:    s_or_b32 s6, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_lshr_b32 s2, s0, 8
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
 }
 

 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sext_i32_i16 s9, s2
 ; GCN-NEXT:    s_sext_i32_i16 s8, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GCN-NEXT:    s_xor_b32 s8, s9, s8
-; GCN-NEXT:    s_ashr_i32 s0, s0, 16
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_ashr_i32 s8, s8, 30
-; GCN-NEXT:    s_or_b32 s8, s8, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_or_b32 s10, s8, 1
+; GCN-NEXT:    s_sext_i32_i16 s1, s1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s8, s10, 0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 16
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    s_ashr_i32 s2, s2, 16
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
 ; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
-; GCN-NEXT:    s_sext_i32_i16 s0, s1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_mad_f32 v2, -v3, v0, v2
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; GCN-NEXT:    s_sext_i32_i16 s1, s3
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v2|, |v0|
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s0, v3
+; GCN-NEXT:    s_sext_i32_i16 s0, s3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
-; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; GCN-NEXT:    v_mad_f32 v3, -v4, v0, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
-; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v4
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v1, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x, <3 x i16> %y) {
 ; CHECK-LABEL: @srem_v3i16(

 ; GCN-NEXT:    s_sext_i32_i16 s8, s2
 ; GCN-NEXT:    s_sext_i32_i16 s6, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s8
 ; GCN-NEXT:    s_xor_b32 s6, s8, s6
 ; GCN-NEXT:    s_ashr_i32 s6, s6, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s6, s6, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s6
-; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s6, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s6, v2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    v_alignbit_b32 v2, s1, v2, 16
 ; GCN-NEXT:    v_bfe_i32 v3, v2, 0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v3
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_alignbit_b32 v1, s3, v1, 16
 ; GCN-NEXT:    v_bfe_i32 v5, v1, 0, 16
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v6, v5
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v4
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    v_xor_b32_e32 v3, v5, v3
 ; GCN-NEXT:    s_sext_i32_i16 s0, s1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v6, v7
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_mad_f32 v6, -v5, v4, v6
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
 ; GCN-NEXT:    v_ashrrev_i32_e32 v3, 30, v3
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v6|, |v4|
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s0
 ; GCN-NEXT:    v_or_b32_e32 v3, 1, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
 ; GCN-NEXT:    s_sext_i32_i16 s2, s3
 ; GCN-NEXT:    v_mul_lo_u32 v2, v3, v2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s2
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v4
 ; GCN-NEXT:    s_xor_b32 s0, s2, s0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    s_or_b32 s0, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v3, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v3, -v5, v4, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v4|
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v4|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s0, s0, 0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v5
 ; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
 ; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s3, v3
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem <3 x i16> %x, %y

 ; GCN-NEXT:    s_bfe_i32 s3, s0, 0xf0000
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s3
 ; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
 ; GCN-NEXT:    s_bfe_i32 s1, s2, 0xf0000
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    s_xor_b32 s1, s1, s3
-; GCN-NEXT:    s_bfe_i32 s0, s0, 0xf000f
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_or_b32 s1, s1, 1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
-; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v5, s1
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; GCN-NEXT:    s_bfe_i32 s0, s0, 0xf000f
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s1, v4
 ; GCN-NEXT:    s_bfe_i32 s1, s2, 0xf000f
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
 ; GCN-NEXT:    s_xor_b32 s0, s1, s0
-; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 15
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
+; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v2|
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v1
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 15
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s0, v5
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v0
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v2
 ; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GCN-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, v5, v6
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_mad_f32 v5, -v1, v4, v5
+; GCN-NEXT:    v_mad_f32 v5, -v1, v2, v5
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v4|
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GCN-NEXT:    s_movk_i32 s0, 0x7fff
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_and_b32_e32 v3, s0, v3
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_and_b32_e32 v2, s0, v3
+; GCN-NEXT:    v_and_b32_e32 v3, s0, v4
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
-; GCN-NEXT:    v_and_b32_e32 v2, s0, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
 ; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    s_endpgm
   %r = sdiv <3 x i15> %x, %y

 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
 ; GCN-NEXT:    s_movk_i32 s3, 0x7fff
-; GCN-NEXT:    s_and_b32 s11, s0, s3
-; GCN-NEXT:    s_bfe_i32 s11, s11, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s11
-; GCN-NEXT:    s_and_b32 s9, s2, s3
-; GCN-NEXT:    s_bfe_i32 s9, s9, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s9
+; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    s_and_b32 s1, s0, s3
+; GCN-NEXT:    s_bfe_i32 s1, s1, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s1
+; GCN-NEXT:    s_and_b32 s8, s2, s3
+; GCN-NEXT:    s_bfe_i32 s8, s8, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s8
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_xor_b32 s9, s9, s11
-; GCN-NEXT:    s_ashr_i32 s9, s9, 30
-; GCN-NEXT:    s_or_b32 s9, s9, 1
+; GCN-NEXT:    s_xor_b32 s1, s8, s1
+; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_lshr_b32 s10, s2, 15
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_mov_b32_e32 v5, s9
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    s_bfe_u32 s12, s0, 0xf000f
-; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    s_bfe_u32 s11, s2, 0xf000f
+; GCN-NEXT:    s_lshr_b32 s12, s0, 15
+; GCN-NEXT:    s_bfe_u32 s13, s0, 0xf000f
+; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s1, v4
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
-; GCN-NEXT:    s_lshr_b32 s1, s0, 15
-; GCN-NEXT:    s_bfe_i32 s0, s12, 0xf0000
+; GCN-NEXT:    s_bfe_i32 s0, s13, 0xf0000
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
-; GCN-NEXT:    s_bfe_u32 s10, s2, 0xf000f
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
-; GCN-NEXT:    s_lshr_b32 s8, s2, 15
-; GCN-NEXT:    s_bfe_i32 s2, s10, 0xf0000
-; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    s_bfe_i32 s1, s11, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GCN-NEXT:    s_xor_b32 s0, s2, s0
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v3|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GCN-NEXT:    v_and_b32_e32 v1, s3, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
-; GCN-NEXT:    v_mov_b32_e32 v6, s0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
 ; GCN-NEXT:    v_bfe_i32 v4, v1, 0, 15
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s0, v5
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v4
 ; GCN-NEXT:    v_and_b32_e32 v0, s3, v0
 ; GCN-NEXT:    v_bfe_i32 v6, v0, 0, 15
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v7, v6
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v5
 ; GCN-NEXT:    v_xor_b32_e32 v4, v6, v4
 ; GCN-NEXT:    v_ashrrev_i32_e32 v4, 30, v4
 ; GCN-NEXT:    v_or_b32_e32 v4, 1, v4
 ; GCN-NEXT:    v_mul_f32_e32 v6, v7, v8
 ; GCN-NEXT:    v_trunc_f32_e32 v6, v6
 ; GCN-NEXT:    v_mad_f32 v7, -v6, v5, v7
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v6, v6
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v7|, |v5|
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s12
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
 ; GCN-NEXT:    v_mul_lo_u32 v1, v4, v1
 ; GCN-NEXT:    v_and_b32_e32 v2, s3, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
 ; GCN-NEXT:    v_and_b32_e32 v3, s3, v3
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
 ; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0

 ; GCN-LABEL: udiv_i64_oddk_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f176a73
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GCN-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_movk_i32 s2, 0xfee0
 ; GCN-NEXT:    s_mov_b32 s3, 0x68958c89
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_movk_i32 s8, 0x11f
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    s_movk_i32 s4, 0x11e
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_movk_i32 s9, 0x11e
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, s3
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
+; GCN-NEXT:    v_mul_lo_u32 v8, v2, s3
 ; GCN-NEXT:    s_mov_b32 s2, 0x976a7377
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
-; GCN-NEXT:    s_movk_i32 s3, 0x11f
-; GCN-NEXT:    s_mov_b32 s9, s5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
-; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s8
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s2
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_mov_b32_e32 v5, s8
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, s2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s7, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s6, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s4, v4
-; GCN-NEXT:    s_mov_b32 s2, 0x976a7376
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v6, s7
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s2, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v4
+; GCN-NEXT:    s_mov_b32 s10, 0x976a7376
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
+; GCN-NEXT:    v_mov_b32_e32 v6, s11
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = udiv i64 %x, 1235195949943
   store i64 %r, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @udiv_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @udiv_i64_pow2k_denom(

 ; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
 ; GCN-NEXT:    v_mul_hi_u32 v6, s11, v1
 ; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
 ; GCN-NEXT:    s_lshr_b64 s[2:3], s[8:9], 12
+; GCN-NEXT:    s_movk_i32 s8, 0xffe
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, s0
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mov_b32_e32 v3, s11
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s0, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_movk_i32 s0, 0xffe
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s8, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v3, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = udiv <2 x i64> %x, <i64 4096, i64 4095>
   store <2 x i64> %r, <2 x i64> addrspace(1)* %out
   ret void
 }

 ; GCN-LABEL: urem_i64_oddk_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f1761f8
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
 ; GCN-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_movk_i32 s2, 0xfee0
 ; GCN-NEXT:    s_mov_b32 s3, 0x689e0837
-; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_movk_i32 s8, 0x11f
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GCN-NEXT:    s_movk_i32 s12, 0x11f
-; GCN-NEXT:    s_mov_b32 s13, 0x9761f7c9
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    s_mov_b32 s12, 0x9761f7c9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s9, s5
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
-; GCN-NEXT:    s_movk_i32 s5, 0x11e
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    s_movk_i32 s9, 0x11e
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, s3
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
-; GCN-NEXT:    s_mov_b32 s8, s4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
-; GCN-NEXT:    s_mov_b32 s4, 0x9761f7c8
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
-; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GCN-NEXT:    s_mov_b32 s10, -1
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v8, v2, s3
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, v0, s12
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, s13
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, s13
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s13
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s8
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s12
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s12
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s12
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
-; GCN-NEXT:    v_mov_b32_e32 v3, s12
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s13, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[2:3], s5, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[2:3], s4, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s13, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s12, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v5, s7
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s5, v1
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v5
+; GCN-NEXT:    s_mov_b32 s10, 0x9761f7c8
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
+; GCN-NEXT:    v_mov_b32_e32 v5, s11
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v0
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s12, v1
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = urem i64 %x, 1235195393993
   store i64 %r, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @urem_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
 ; CHECK-LABEL: @urem_i64_pow2k_denom(

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, s3
 ; GCN-NEXT:    v_mul_hi_u32 v3, s3, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = sdiv i64 %x, 1235195

 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
 ; GCN-NEXT:    s_ashr_i32 s12, s3, 31
 ; GCN-NEXT:    s_add_u32 s2, s2, s12
 ; GCN-NEXT:    s_mov_b32 s13, s12
 ; GCN-NEXT:    s_addc_u32 s3, s3, s12
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[12:13]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[12:13]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_sub_u32 s2, 0, s14
+; GCN-NEXT:    s_subb_u32 s3, 0, s15
+; GCN-NEXT:    s_ashr_i32 s16, s11, 31
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s17, s16
 ; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v2, s4, v1
-; GCN-NEXT:    v_mul_lo_u32 v5, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s4, v0
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_mul_lo_u32 v7, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
 ; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
 ; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
 ; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
 ; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s10, s14
-; GCN-NEXT:    s_addc_u32 s1, s11, s14
+; GCN-NEXT:    s_add_u32 s0, s10, s16
+; GCN-NEXT:    s_addc_u32 s1, s11, s16
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[16:17]
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
 ; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
 ; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
 ; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s15, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s15
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[12:13]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[16:17], s[12:13]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, s9
 ; GCN-NEXT:    v_mul_hi_u32 v3, s9, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s9, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
-; GCN-NEXT:    s_movk_i32 s0, 0xffe
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    s_movk_i32 s9, 0xffe
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s8, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s8, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s8
 ; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0

 ; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
 ; GCN-LABEL: sdiv_v2i64_pow2_shl_denom:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x11
 ; GCN-NEXT:    s_mov_b32 s3, 0
 ; GCN-NEXT:    s_movk_i32 s2, 0x1000
-; GCN-NEXT:    s_mov_b32 s18, 0x4f800000
-; GCN-NEXT:    s_mov_b32 s19, 0x5f7ffffc
+; GCN-NEXT:    s_mov_b32 s20, 0x4f800000
+; GCN-NEXT:    s_mov_b32 s21, 0x5f7ffffc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshl_b64 s[12:13], s[2:3], s6
 ; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
 ; GCN-NEXT:    s_ashr_i32 s16, s3, 31
 ; GCN-NEXT:    s_add_u32 s2, s2, s16
 ; GCN-NEXT:    s_mov_b32 s17, s16
 ; GCN-NEXT:    s_addc_u32 s3, s3, s16
 ; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[16:17]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
-; GCN-NEXT:    s_mov_b32 s20, 0x2f800000
-; GCN-NEXT:    s_mov_b32 s21, 0xcf800000
+; GCN-NEXT:    s_mov_b32 s22, 0x2f800000
+; GCN-NEXT:    s_mov_b32 s23, 0xcf800000
 ; GCN-NEXT:    s_sub_u32 s6, 0, s14
-; GCN-NEXT:    v_mac_f32_e32 v0, s18, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s20, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_subb_u32 s7, 0, s15
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
-; GCN-NEXT:    v_mul_f32_e32 v0, s19, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, s20, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, s21, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s22, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_mac_f32_e32 v0, s21, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s23, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s18, s9, 31
+; GCN-NEXT:    s_add_u32 s0, s8, s18
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
 ; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GCN-NEXT:    v_mul_lo_u32 v5, s6, v0
+; GCN-NEXT:    s_mov_b32 s19, s18
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
 ; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_addc_u32 s1, s9, s18
+; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[18:19]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-NEXT:    v_add_i32_e64 v0, s[2:3], v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc

 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[2:3]
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s9, 31
-; GCN-NEXT:    s_add_u32 s0, s8, s2
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_addc_u32 s1, s9, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[2:3]
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
 ; GCN-NEXT:    v_mul_hi_u32 v5, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v7, s9, v1
 ; GCN-NEXT:    v_mul_lo_u32 v1, s9, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s9, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s9, v0
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[16:17]
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
 ; GCN-NEXT:    v_mul_lo_u32 v5, s15, v0
 ; GCN-NEXT:    v_mov_b32_e32 v7, s15
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
 ; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s9, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
-; GCN-NEXT:    v_subb_u32_e64 v5, s[0:1], v5, v7, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s14, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[0:1], 0, v5, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s15, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v8, v7, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    s_ashr_i32 s8, s13, 31
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    s_add_u32 s12, s12, s8
-; GCN-NEXT:    v_cndmask_b32_e64 v5, v10, v8, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s8, v3
+; GCN-NEXT:    v_subb_u32_e64 v5, vcc, v5, v7, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v10, v8, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v8, s9
-; GCN-NEXT:    s_mov_b32 s9, s8
-; GCN-NEXT:    s_addc_u32 s13, s13, s8
-; GCN-NEXT:    s_xor_b64 s[12:13], s[12:13], s[8:9]
+; GCN-NEXT:    s_xor_b64 s[8:9], s[18:19], s[16:17]
+; GCN-NEXT:    s_ashr_i32 s16, s13, 31
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v8, v2, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s12, s16
+; GCN-NEXT:    s_mov_b32 s17, s16
+; GCN-NEXT:    s_addc_u32 s1, s13, s16
+; GCN-NEXT:    s_xor_b64 s[12:13], s[0:1], s[16:17]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v10, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v11, s13
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
-; GCN-NEXT:    v_mac_f32_e32 v10, s18, v11
+; GCN-NEXT:    v_mac_f32_e32 v10, s20, v11
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v8, v3, vcc
 ; GCN-NEXT:    v_rcp_f32_e32 v3, v10
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GCN-NEXT:    s_sub_u32 s14, 0, s12
-; GCN-NEXT:    v_mul_f32_e32 v3, s19, v3
-; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v9, v7, s[2:3]
+; GCN-NEXT:    v_mul_f32_e32 v3, s21, v3
+; GCN-NEXT:    v_mul_f32_e32 v5, s22, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
-; GCN-NEXT:    v_mac_f32_e32 v3, s21, v5
+; GCN-NEXT:    v_mac_f32_e32 v3, s23, v5
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v9, v7, s[0:1]
+; GCN-NEXT:    s_sub_u32 s2, 0, s12
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, s14, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s14, v5
-; GCN-NEXT:    s_subb_u32 s15, 0, s13
-; GCN-NEXT:    v_mul_lo_u32 v8, s15, v3
-; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, s2, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v5
+; GCN-NEXT:    s_subb_u32 s3, 0, s13
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v3
+; GCN-NEXT:    s_ashr_i32 s14, s11, 31
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, s14, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
 ; GCN-NEXT:    v_mul_lo_u32 v8, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v2
 ; GCN-NEXT:    v_mul_hi_u32 v9, v3, v7
 ; GCN-NEXT:    v_mul_hi_u32 v11, v5, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v5, v2
-; GCN-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-NEXT:    s_mov_b32 s15, s14
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v10, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s9, v1
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v11, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
 ; GCN-NEXT:    v_add_i32_e64 v2, s[0:1], v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v6, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v5, v7, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v8, s14, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, s14, v2
-; GCN-NEXT:    v_mul_lo_u32 v10, s15, v2
+; GCN-NEXT:    v_mul_lo_u32 v8, s2, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v10, s3, v2
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; GCN-NEXT:    v_mul_lo_u32 v9, s14, v2
+; GCN-NEXT:    v_mul_lo_u32 v9, s2, v2
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
 ; GCN-NEXT:    v_mul_lo_u32 v12, v2, v8
 ; GCN-NEXT:    v_mul_hi_u32 v14, v2, v8
 ; GCN-NEXT:    v_mul_hi_u32 v13, v2, v9
 ; GCN-NEXT:    v_mul_hi_u32 v11, v3, v9
 ; GCN-NEXT:    v_mul_lo_u32 v9, v3, v9
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v8
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
 ; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v14, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, v3, v8
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, v12, v9
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v13, v11, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v10, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v6, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
 ; GCN-NEXT:    v_addc_u32_e64 v5, vcc, v5, v8, s[0:1]
 ; GCN-NEXT:    s_add_u32 s0, s10, s14
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GCN-NEXT:    s_mov_b32 s15, s14
 ; GCN-NEXT:    s_addc_u32 s1, s11, s14
 ; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s10, v3
 ; GCN-NEXT:    v_mul_hi_u32 v7, s10, v2
 ; GCN-NEXT:    v_mul_hi_u32 v9, s10, v3
 ; GCN-NEXT:    v_mul_hi_u32 v10, s11, v3
 ; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v9, s11, v2
 ; GCN-NEXT:    v_mul_hi_u32 v2, s11, v2
-; GCN-NEXT:    v_mov_b32_e32 v8, s3
+; GCN-NEXT:    v_mov_b32_e32 v8, s9
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v10, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, s12, v3
 ; GCN-NEXT:    v_mul_hi_u32 v5, s12, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
 ; GCN-NEXT:    v_mul_lo_u32 v6, s13, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_mul_lo_u32 v5, s12, v2
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
 ; GCN-NEXT:    v_sub_i32_e32 v6, vcc, s11, v4
 ; GCN-NEXT:    v_mov_b32_e32 v7, s13
-; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s10, v5
-; GCN-NEXT:    v_subb_u32_e64 v6, s[0:1], v6, v7, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s12, v5
-; GCN-NEXT:    v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v8, v7, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v2
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v2
-; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v3, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v10, v8, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v5, s[0:1], s10, v5
+; GCN-NEXT:    v_subb_u32_e64 v6, vcc, v6, v7, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v7, vcc, s12, v5
+; GCN-NEXT:    v_subbrev_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v6
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 2, v2
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 1, v2
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v3, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v10, v8, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v8, s11
-; GCN-NEXT:    v_subb_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v8, v4, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v5
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v4
 ; GCN-NEXT:    v_cndmask_b32_e32 v4, v8, v5, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v9, v7, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[8:9]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[16:17]
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v2, s0, v2
 ; GCN-NEXT:    v_xor_b32_e32 v3, s1, v3
 ; GCN-NEXT:    v_mov_b32_e32 v4, s1
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s0, v2
 ; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_mul_hi_u32 v2, s3, v0
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s3, v2
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v3, vcc
-; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v2
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, -1, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %r = srem i64 %x, 1235195

 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s14, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s14, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s14
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s14, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm

 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s16, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s16, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s17, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s16, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s8, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s17
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s16, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v7, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s17, v7
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s16, v5
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s16, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s17, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[2:3]
-; GCN-NEXT:    s_ashr_i32 s2, s15, 31
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GCN-NEXT:    s_add_u32 s8, s14, s2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v5, s[2:3], s16, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v7, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s17, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s16, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s17, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s16, v5
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v8
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v7, s9
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_addc_u32 s9, s15, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[8:9], s[2:3]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v8, s8
-; GCN-NEXT:    v_cvt_f32_u32_e32 v9, s9
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v7, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v7, v1, s[0:1]
+; GCN-NEXT:    s_ashr_i32 s0, s15, 31
+; GCN-NEXT:    s_add_u32 s8, s14, s0
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_addc_u32 s9, s15, s0
+; GCN-NEXT:    s_xor_b64 s[8:9], s[8:9], s[0:1]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v9, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v10, s9
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s17, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GCN-NEXT:    v_mac_f32_e32 v8, s18, v9
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s16, v0
-; GCN-NEXT:    v_rcp_f32_e32 v8, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s17, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc
+; GCN-NEXT:    v_mac_f32_e32 v9, s18, v10
+; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
+; GCN-NEXT:    v_rcp_f32_e32 v8, v9
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[2:3]
 ; GCN-NEXT:    v_mul_f32_e32 v3, s19, v8
 ; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
 ; GCN-NEXT:    v_trunc_f32_e32 v5, v5
 ; GCN-NEXT:    v_mac_f32_e32 v3, s21, v5
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; GCN-NEXT:    s_sub_u32 s2, 0, s8
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc

 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v10, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, s8, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, s8, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, s9, v2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v8, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_sub_i32_e64 v2, s[0:1], s10, v2
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v3
 ; GCN-NEXT:    v_mov_b32_e32 v5, s9
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s10, v2
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v6, s[0:1], s8, v2
-; GCN-NEXT:    v_subbrev_u32_e64 v7, s[2:3], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s9, v7
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s8, v6
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s8, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s9, v7
-; GCN-NEXT:    v_cndmask_b32_e64 v8, v8, v9, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v7, v4, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v6, s[2:3], s8, v2
+; GCN-NEXT:    v_subbrev_u32_e64 v7, vcc, 0, v4, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v6
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v8
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v7, v4, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v7, s11
-; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v7, v3, vcc
+; GCN-NEXT:    v_subb_u32_e64 v3, vcc, v7, v3, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v2, s14, v2
 ; GCN-NEXT:    v_xor_b32_e32 v3, s14, v3
 ; GCN-NEXT:    v_mov_b32_e32 v4, s14
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s14, v2
 ; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
   %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
   %r = srem <2 x i64> %x, %shl.y
   store <2 x i64> %r, <2 x i64> addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/ctlz.ll

 define amdgpu_kernel void @s_ctlz_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind {
 ; SI-LABEL: s_ctlz_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
+; SI-NEXT:    s_cmp_lg_u32 s2, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 32
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_ctlz_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s1, s0
-; VI-NEXT:    v_mov_b32_e32 v0, s1
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s1, 32
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_ctlz_i32:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
 ; EG-NEXT:    CF_END

 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x13
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
 ; SI-NEXT:    s_flbit_i32_b32 s1, s3
 ; SI-NEXT:    s_add_i32 s0, s0, 32
-; SI-NEXT:    s_or_b32 s2, s2, s3
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, s1
+; SI-NEXT:    s_or_b32 s1, s2, s3
+; SI-NEXT:    s_cmp_lg_u32 s1, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 64
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_ctlz_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s2, s0
-; VI-NEXT:    s_flbit_i32_b32 s3, s1
 ; VI-NEXT:    s_add_i32 s2, s2, 32
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_flbit_i32_b32 s3, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b32 s2, s2, s3
 ; VI-NEXT:    s_or_b32 s0, s0, s1
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s2, 64
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_ctlz_i64:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 9, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
 ; EG-NEXT:    CF_END

 }
 
 define amdgpu_kernel void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64 %val) nounwind {
 ; SI-LABEL: s_ctlz_i64_trunc:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_flbit_i32_b32 s0, s2
 ; SI-NEXT:    s_flbit_i32_b32 s1, s3
 ; SI-NEXT:    s_add_i32 s0, s0, 32
-; SI-NEXT:    s_or_b32 s2, s2, s3
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s2, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, s1
+; SI-NEXT:    s_or_b32 s1, s2, s3
+; SI-NEXT:    s_cmp_lg_u32 s1, 0
+; SI-NEXT:    s_cselect_b32 s0, s0, 64
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_ctlz_i64_trunc:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_flbit_i32_b32 s2, s0
-; VI-NEXT:    s_flbit_i32_b32 s3, s1
 ; VI-NEXT:    s_add_i32 s2, s2, 32
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_flbit_i32_b32 s3, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b32 s2, s2, s3
 ; VI-NEXT:    s_or_b32 s0, s0, s1
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s0, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s0, 0
+; VI-NEXT:    s_cselect_b32 s0, s2, 64
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_ctlz_i64_trunc:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 8, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
 ; EG-NEXT:    CF_END

llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll

define amdgpu_kernel void @v_ctlz_zero_undef_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
   %val = load i8, i8 addrspace(1)* %in.gep
   %ctlz = call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
   store i8 %ctlz, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_ctlz_zero_undef_i64:
 ; GCN: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}}
-; GCN-DAG: v_cmp_eq_u32_e64 vcc, s[[HI]], 0{{$}}
+; GCN-DAG: s_cmp_eq_u32 s[[HI]], 0{{$}}
 ; GCN-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]]
 ; GCN-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32
 ; GCN-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]]
-; GCN-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[ADD]]
-; GCN-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]]
-; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]]
+; GCN-DAG: s_cselect_b32 [[RES:s[0-9]+]], [[ADD]], [[FFBH_HI]]
 ; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[CTLZ:[0-9]+]], [[RES]]
 ; GCN: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
 define amdgpu_kernel void @s_ctlz_zero_undef_i64(i64 addrspace(1)* noalias %out, [8 x i32], i64 %val) nounwind {
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
   store i64 %ctlz, i64 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_ctlz_zero_undef_i64_trunc:

llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll

define amdgpu_kernel void @v_cttz_zero_undef_i16_with_select(i16 addrspace(1)* noalias %out, i16 addrspace(1)* nocapture readonly %arrayidx) nounwind {
   %cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone
   %cttz_ret = icmp ne i16 %val, 0
   %ret = select i1 %cttz_ret, i16 %cttz, i16 32
   store i16 %ret, i16 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_cttz_zero_undef_i32_with_select:
-; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, 0
+; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI-DAG: v_cmp_ne_u32_e32 vcc, 0
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
 define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
   %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone
   %cttz_ret = icmp ne i32 %val, 0
   %ret = select i1 %cttz_ret, i32 %cttz, i32 32
   store i32 %ret, i32 addrspace(1)* %out, align 4
   ret void

llvm/test/CodeGen/AMDGPU/dagcombine-select.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}select_and1:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
   %a = and i32 %y, %s
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_and2:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
   %a = and i32 %s, %y
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_and3:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_and_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_and3(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 -1, i32 0
   %a = and i32 %y, %s
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_and_v4:
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, v{{[0-9]+}},
 ; GCN-NOT: v_and_b32
 ; GCN:     store_dword
 define amdgpu_kernel void @select_and_v4(<4 x i32> addrspace(1)* %p, i32 %x, <4 x i32> %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = and <4 x i32> %s, %y
   store <4 x i32> %a, <4 x i32> addrspace(1)* %p, align 32
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_or1:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or1(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
   %a = or i32 %y, %s
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_or2:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or2(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 0, i32 -1
   %a = or i32 %s, %y
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_or3:
-; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
+; GCN:     s_cselect_b32 [[SEL:s[0-9]+]], s{{[0-9]+}},
+; GCN:     v_mov_b32_e32 [[VSEL:v[0-9]+]], [[SEL]]
 ; GCN-NOT: v_or_b32
-; GCN:     store_dword v[{{[0-9:]+}}], [[SEL]],
+; GCN:     store_dword v[{{[0-9:]+}}], [[VSEL]],
 define amdgpu_kernel void @select_or3(i32 addrspace(1)* %p, i32 %x, i32 %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, i32 -1, i32 0
   %a = or i32 %y, %s
   store i32 %a, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_or_v4:
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
 ; GCN:     v_cndmask_b32_e32 [[SEL:v[0-9]+]], -1, v{{[0-9]+}},
 ; GCN-NOT: v_or_b32
 ; GCN:     store_dword
 define amdgpu_kernel void @select_or_v4(<4 x i32> addrspace(1)* %p, i32 %x, <4 x i32> %y) {
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = or <4 x i32> %s, %y
   store <4 x i32> %a, <4 x i32> addrspace(1)* %p, align 32
   ret void
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 2, 9,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 9, 2
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 -4, i32 3
   %bo = sub i32 5, %sel
   store i32 %bo, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants_i16:

 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16_neg(i16 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i16 4, i16 3000
   %bo = sub i16 1, %sel
   store i16 %bo, i16 addrspace(1)* %p, align 2
   ret void
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants_v2i16:
-; GCN-DAG: v_mov_b32_e32 [[F:v[0-9]+]], 0x60002
-; GCN-DAG: v_mov_b32_e32 [[T:v[0-9]+]], 0x50009
-; GCN:     v_cndmask_b32_e32 v{{[0-9]+}}, [[F]], [[T]],
+; GCN-DAG: s_mov_b32 [[T:s[0-9]+]], 0x50009
+; GCN:     s_cselect_b32 s{{[0-9]+}}, [[T]], 0x60002
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v2i16(<2 x i16> addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, <2 x i16> <i16 -4, i16 2>, <2 x i16> <i16 3, i16 1>
   %bo = sub <2 x i16> <i16 5, i16 7>, %sel
   store <2 x i16> %bo, <2 x i16> addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}sel_constants_sub_constant_sel_constants_v4i32:
 ; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 2, 9,
 ; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 6, 5,
 ; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 10, 6,
 ; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 14, 7,
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(<4 x i32> addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, <4 x i32> <i32 -4, i32 2, i32 3, i32 4>, <4 x i32> <i32 3, i32 1, i32 -1, i32 -3>
   %bo = sub <4 x i32> <i32 5, i32 7, i32 9, i32 11>, %sel
   store <4 x i32> %bo, <4 x i32> addrspace(1)* %p, align 32
   ret void
 }
 
 ; GCN-LABEL: {{^}}sdiv_constant_sel_constants_i64:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 5, 0,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 0, 5
 define amdgpu_kernel void @sdiv_constant_sel_constants_i64(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 121, i64 23
   %bo = sdiv i64 120, %sel
   store i64 %bo, i64 addrspace(1)* %p, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}sdiv_constant_sel_constants_i32:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 26,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 26, 8
 define amdgpu_kernel void @sdiv_constant_sel_constants_i32(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 7, i32 23
   %bo = sdiv i32 184, %sel
   store i32 %bo, i32 addrspace(1)* %p, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}udiv_constant_sel_constants_i64:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 5, 0,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 0, 5
 define amdgpu_kernel void @udiv_constant_sel_constants_i64(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 -4, i64 23
   %bo = udiv i64 120, %sel
   store i64 %bo, i64 addrspace(1)* %p, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}srem_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 3, 33,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 33, 3
 define amdgpu_kernel void @srem_constant_sel_constants(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 34, i64 15
   %bo = srem i64 33, %sel
   store i64 %bo, i64 addrspace(1)* %p, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}urem_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 3, 33,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 33, 3
 define amdgpu_kernel void @urem_constant_sel_constants(i64 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i64 34, i64 15
   %bo = urem i64 33, %sel
   store i64 %bo, i64 addrspace(1)* %p, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}shl_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 4,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 4, 8
 define amdgpu_kernel void @shl_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = shl i32 1, %sel
   store i32 %bo, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}lshr_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 8, 16,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 16, 8
 define amdgpu_kernel void @lshr_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = lshr i32 64, %sel
   store i32 %bo, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}ashr_constant_sel_constants:
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 16, 32,
+; GCN: s_cselect_b32 s{{[0-9]+}}, 32, 16
 define amdgpu_kernel void @ashr_constant_sel_constants(i32 addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, i32 2, i32 3
   %bo = ashr i32 128, %sel
   store i32 %bo, i32 addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}fsub_constant_sel_constants:

 define amdgpu_kernel void @fsub_constant_sel_constants_f16(half addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, half -2.0, half 3.0
   %bo = fsub half -1.0, %sel
   store half %bo, half addrspace(1)* %p, align 2
   ret void
 }
 
 ; GCN-LABEL: {{^}}fsub_constant_sel_constants_v2f16:
-; GCN-DAG: v_mov_b32_e32 [[T:v[0-9]+]], 0x45003c00
-; GCN:     v_cndmask_b32_e32 v{{[0-9]+}}, -2.0, [[T]],
+; GCN:     s_cselect_b32 s{{[0-9]+}}, 0x45003c00, -2.0
 define amdgpu_kernel void @fsub_constant_sel_constants_v2f16(<2 x half> addrspace(1)* %p, i1 %cond) {
   %sel = select i1 %cond, <2 x half> <half -2.0, half -3.0>, <2 x half> <half -1.0, half 4.0>
   %bo = fsub <2 x half> <half -1.0, half 2.0>, %sel
   store <2 x half> %bo, <2 x half> addrspace(1)* %p, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}fsub_constant_sel_constants_v4f32:

llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll

 entry:
   %ext = extractelement <4 x float> <float 0.0, float 1.0, float 2.0, float 4.0>, i32 %sel
   store float %ext, float addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}int4_extelt:
 ; GCN-NOT: buffer_
-; GCN-DAG: v_cmp_eq_u32_e64 [[C1:[^,]+]], [[IDX:s[0-9]+]], 1
-; GCN-DAG: v_cmp_ne_u32_e64 [[C2:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cmp_ne_u32_e64 [[C3:[^,]+]], [[IDX]], 3
+; GCN-DAG: s_cmp_lg_u32 [[IDX:s[0-9]+]], 2
+; GCN-DAG: v_cmp_eq_u32_e64 [[C1:[^,]+]], [[IDX]], 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 3
 ; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V1:v[0-9]+]], 0, 1, [[C1]]
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V2:v[0-9]+]], 2, [[V1]], [[C2]]
-; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V3:v[0-9]+]], 4, [[V2]], [[C3]]
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V2:v[0-9]+]], 2, [[V1]], vcc
+; GCN-DAG: v_cndmask_b32_e{{32|64}} [[V3:v[0-9]+]], 4, [[V2]], vcc
 ; GCN: store_dword v[{{[0-9:]+}}], [[V3]]
 define amdgpu_kernel void @int4_extelt(i32 addrspace(1)* %out, i32 %sel) {
 entry:
   %ext = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 4>, i32 %sel
   store i32 %ext, i32 addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/extractelt-to-trunc.ll

entry:
   store i32 %elt2, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @no_extract_volatile_load_dynextract(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in, i32 %idx) {
 ; GCN-LABEL: no_extract_volatile_load_dynextract:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s12, s[0:1], 0xd
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    s_load_dword s12, s[0:1], 0xd
 ; GCN-NEXT:    s_mov_b32 s10, s2
 ; GCN-NEXT:    s_mov_b32 s11, s3
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_mov_b32 s8, s6
 ; GCN-NEXT:    s_mov_b32 s9, s7
 ; GCN-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCN-NEXT:    s_mov_b32 s0, s4
-; GCN-NEXT:    s_mov_b32 s1, s5
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 1
+; GCN-NEXT:    s_cmp_eq_u32 s12, 1
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_cmp_eq_u32 s12, 2
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 2
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_cmp_eq_u32 s12, 3
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e64 vcc, s12, 3
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 entry:
   %vec = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in
   %eltN = extractelement <4 x i32> %vec, i32 %idx
   store i32 %eltN, i32 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/fceil64.ll

 ; SI-DAG: cndmask_b32
 ; SI-DAG: cndmask_b32
 ; SI-DAG: cmp_lt_i32
 ; SI-DAG: cndmask_b32
 ; SI-DAG: cndmask_b32
 ; SI-DAG: v_cmp_gt_f64
 ; SI-DAG: v_cmp_lg_f64
 ; SI-DAG: v_cndmask_b32
-; SI: v_cndmask_b32
+; SI: s_cselect_b32
 ; SI: v_add_f64
 ; SI: s_endpgm
 define amdgpu_kernel void @fceil_f64(double addrspace(1)* %out, double %x) {
   %y = call double @llvm.ceil.f64(double %x) nounwind readnone
   store double %y, double addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir

-# RUN: llc -march=amdgcn -run-pass=si-fix-sgpr-copies -o - %s | FileCheck --check-prefix=GCN %s
+# RUN: llc -march=amdgcn -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
 
 # GCN-LABEL: name: fix-sgpr-copies
 # GCN: V_ADD_I32_e32
 # GCN: V_ADDC_U32_e32
 ---
 name: fix-sgpr-copies
 body:               |
   bb.0:
bb.1:
     %1:sgpr_32 = IMPLICIT_DEF
     S_BRANCH %bb.2
 
   bb.2:
     %2:sgpr_32 = PHI %0, %bb.0, %1, %bb.1
     %3:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %0, %subreg.sub1
     $vgpr3 = COPY %3.sub0
 ...
+
+# Test to ensure that undef SCC gets properly propagated.
+# GCN-LABEL: name: scc_undef
+# GCN: S_CSELECT_B64 -1, 0, implicit undef $scc
+# GCN: V_CNDMASK
+---
+name: scc_undef
+tracksRegLiveness: true
+
+body:               |
+  bb.0:
+  %1:vgpr_32 = IMPLICIT_DEF
+  %2:sreg_32 = S_MOV_B32 1
+  %3:sreg_32 = COPY %1:vgpr_32
+  %4:sreg_32 = S_CSELECT_B32 killed %2:sreg_32, killed %3:sreg_32, implicit undef $scc
+---

llvm/test/CodeGen/AMDGPU/fshl.ll

 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_sub_i32 s3, 32, s2
 ; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_and_b32 s1, s2, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshl_i32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_sub_i32 s3, 32, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-NEXT:    s_and_b32 s1, s2, 31
-; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; VI-NEXT:    v_mov_b32_e32 v1, s0
-; VI-NEXT:    v_alignbit_b32 v0, s0, v0, v2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshl_i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_sub_i32 s3, 32, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    s_and_b32 s1, s2, 31
-; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v2
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshl_i32:
 ; R600:       ; %bb.0: ; %entry

 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xf
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s9
 ; SI-NEXT:    s_sub_i32 s10, 32, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s10
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s10
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s3, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_sub_i32 s1, 32, s0
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s0, s0, 31
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    v_mov_b32_e32 v2, s1
 ; SI-NEXT:    v_alignbit_b32 v0, s2, v0, v2
 ; SI-NEXT:    v_mov_b32_e32 v2, s2
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshl_v2i32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x34
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x3c
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
 ; VI-NEXT:    s_sub_i32 s8, 32, s1
-; VI-NEXT:    v_mov_b32_e32 v1, s8
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s8
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_sub_i32 s1, 32, s0
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s0, s0, 31
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mov_b32_e32 v2, s1
 ; VI-NEXT:    v_alignbit_b32 v0, s4, v0, v2
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshl_v2i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
 ; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x3c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX9-NEXT:    s_sub_i32 s8, 32, s1
-; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    s_sub_i32 s1, 32, s0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX9-NEXT:    v_alignbit_b32 v0, s4, v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshl_v2i32:
 ; R600:       ; %bb.0: ; %entry

 ; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
 ; SI-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x11
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x15
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s15
 ; SI-NEXT:    s_sub_i32 s16, 32, s3
-; SI-NEXT:    v_mov_b32_e32 v1, s16
 ; SI-NEXT:    s_and_b32 s3, s3, 31
+; SI-NEXT:    v_mov_b32_e32 v1, s16
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
 ; SI-NEXT:    v_alignbit_b32 v0, s11, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s11
 ; SI-NEXT:    s_sub_i32 s3, 32, s2
-; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s2, s2, 31
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s14
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    v_alignbit_b32 v0, s10, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s10
 ; SI-NEXT:    s_sub_i32 s2, 32, s1
-; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s13
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_alignbit_b32 v0, s9, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s9
 ; SI-NEXT:    s_sub_i32 s1, 32, s0
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; SI-NEXT:    s_and_b32 s0, s0, 31
+; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
 ; SI-NEXT:    v_mov_b32_e32 v4, s1
 ; SI-NEXT:    v_alignbit_b32 v0, s8, v0, v4
 ; SI-NEXT:    v_mov_b32_e32 v4, s8
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshl_v4i32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; VI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x44
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x54
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s11
 ; VI-NEXT:    s_sub_i32 s14, 32, s3
-; VI-NEXT:    v_mov_b32_e32 v1, s14
 ; VI-NEXT:    s_and_b32 s3, s3, 31
+; VI-NEXT:    v_mov_b32_e32 v1, s14
+; VI-NEXT:    s_cmp_eq_u32 s3, 0
 ; VI-NEXT:    v_alignbit_b32 v0, s7, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    s_sub_i32 s3, 32, s2
-; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s2, s2, 31
+; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s10
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_alignbit_b32 v0, s6, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s6
 ; VI-NEXT:    s_sub_i32 s2, 32, s1
-; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s9
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_sub_i32 s1, 32, s0
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; VI-NEXT:    s_and_b32 s0, s0, 31
+; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_alignbit_b32 v0, s4, v0, v4
 ; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; VI-NEXT:    v_mov_b32_e32 v4, s12
 ; VI-NEXT:    v_mov_b32_e32 v5, s13
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshl_v4i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x44
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x54
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s11
 ; GFX9-NEXT:    s_sub_i32 s14, 32, s3
-; GFX9-NEXT:    v_mov_b32_e32 v1, s14
 ; GFX9-NEXT:    s_and_b32 s3, s3, 31
+; GFX9-NEXT:    v_mov_b32_e32 v1, s14
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 0
 ; GFX9-NEXT:    v_alignbit_b32 v0, s7, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    s_sub_i32 s3, 32, s2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s2, s2, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_alignbit_b32 v0, s6, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    s_sub_i32 s2, 32, s1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    v_alignbit_b32 v0, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    s_sub_i32 s1, 32, s0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_alignbit_b32 v0, s4, v0, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s13
 ; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshl_v4i32:
 ; R600:       ; %bb.0: ; %entry

llvm/test/CodeGen/AMDGPU/fshr.ll

 ; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xf
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s9
 ; SI-NEXT:    s_and_b32 s1, s1, 31
 ; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s0, s0, 31
 ; SI-NEXT:    v_alignbit_b32 v1, s3, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    v_alignbit_b32 v2, s2, v0, v2
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshr_v2i32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x34
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x3c
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
 ; VI-NEXT:    s_and_b32 s1, s1, 31
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s0, s0, 31
 ; VI-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    v_alignbit_b32 v2, s4, v0, v2
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshr_v2i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
 ; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x3c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
 ; GFX9-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    v_alignbit_b32 v2, s4, v0, v2
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshr_v2i32:
 ; R600:       ; %bb.0: ; %entry

 ; SI-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x11
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x15
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s15
 ; SI-NEXT:    s_and_b32 s3, s3, 31
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    v_alignbit_b32 v1, s11, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; SI-NEXT:    s_cmp_eq_u32 s3, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s2, s2, 31
+; SI-NEXT:    v_alignbit_b32 v1, s11, v0, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s14
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
-; SI-NEXT:    v_alignbit_b32 v1, s10, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s1, s1, 31
+; SI-NEXT:    v_alignbit_b32 v1, s10, v0, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s1, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s13
 ; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    s_and_b32 s0, s0, 31
 ; SI-NEXT:    v_alignbit_b32 v1, s9, v0, v1
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT:    s_cmp_eq_u32 s0, 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
 ; SI-NEXT:    v_mov_b32_e32 v4, s0
 ; SI-NEXT:    v_alignbit_b32 v4, s8, v0, v4
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshr_v4i32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; VI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x44
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x54
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s11
 ; VI-NEXT:    s_and_b32 s3, s3, 31
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    v_alignbit_b32 v1, s7, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; VI-NEXT:    s_cmp_eq_u32 s3, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s2, s2, 31
+; VI-NEXT:    v_alignbit_b32 v1, s7, v0, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s10
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
-; VI-NEXT:    v_alignbit_b32 v1, s6, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s1, s1, 31
+; VI-NEXT:    v_alignbit_b32 v1, s6, v0, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s9
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    s_and_b32 s0, s0, 31
 ; VI-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT:    s_cmp_eq_u32 s0, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_alignbit_b32 v4, s4, v0, v4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; VI-NEXT:    v_mov_b32_e32 v4, s12
 ; VI-NEXT:    v_mov_b32_e32 v5, s13
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshr_v4i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x44
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x54
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s11
 ; GFX9-NEXT:    s_and_b32 s3, s3, 31
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    v_alignbit_b32 v1, s7, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s2, s2, 31
+; GFX9-NEXT:    v_alignbit_b32 v1, s7, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    v_alignbit_b32 v1, s6, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
+; GFX9-NEXT:    v_alignbit_b32 v1, s6, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    s_and_b32 s0, s0, 31
 ; GFX9-NEXT:    v_alignbit_b32 v1, s5, v0, v1
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX9-NEXT:    v_alignbit_b32 v4, s4, v0, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
+; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s13
 ; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshr_v4i32:
 ; R600:       ; %bb.0: ; %entry

llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll

entry:
   %v = insertelement <4 x float> undef, float 1.000000e+00, i32 %sel
   store <4 x float> %v, <4 x float> addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}int4_inselt:
 ; GCN-NOT: v_movrel
 ; GCN-NOT: buffer_
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC1:[^,]+]], [[IDX:s[0-9]+]], 3
-; GCN-DAG: v_cndmask_b32_e32 v[[ELT_LAST:[0-9]+]], 1, v{{[0-9]+}}, [[CC1]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC2:[^,]+]], [[IDX]], 2
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}, [[CC2]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC3:[^,]+]], [[IDX]], 1
-; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}, [[CC3]]
-; GCN-DAG: v_cmp_ne_u32_e64 [[CC4:[^,]+]], [[IDX]], 0
-; GCN-DAG: v_cndmask_b32_e32 v[[ELT_FIRST:[0-9]+]], 1, v{{[0-9]+}}, [[CC4]]
-; GCN:     flat_store_dwordx4 v[{{[0-9:]+}}], v{{\[}}[[ELT_FIRST]]:[[ELT_LAST]]]
+; GCN-DAG: s_cmp_lg_u32 [[IDX:s[0-9]+]], 3
+; GCN-DAG: s_cselect_b32 s[[ELT_3:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 2
+; GCN-DAG: s_cselect_b32 s[[ELT_2:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 1
+; GCN-DAG: s_cselect_b32 s[[ELT_1:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: s_cmp_lg_u32 [[IDX]], 0
+; GCN-DAG: s_cselect_b32 s[[ELT_0:[0-9]+]], s{{[0-9]+}}, 1
+; GCN-DAG: v_mov_b32_e32 v[[VELT_0:[0-9]+]], s[[ELT_0]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_1:[0-9]+]], s[[ELT_1]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_2:[0-9]+]], s[[ELT_2]]
+; GCN-DAG: v_mov_b32_e32 v[[VELT_3:[0-9]+]], s[[ELT_3]]
+; GCN:     flat_store_dwordx4 v[{{[0-9:]+}}], v{{\[}}[[VELT_0]]:[[VELT_3]]]
 define amdgpu_kernel void @int4_inselt(<4 x i32> addrspace(1)* %out, <4 x i32> %vec, i32 %sel) {
 entry:
   %v = insertelement <4 x i32> %vec, i32 1, i32 %sel
   store <4 x i32> %v, <4 x i32> addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}float2_inselt:

llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll

 ; SI-LABEL: dynamic_insertelement_v2i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x4
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s5, s7, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s6, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v2i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s7
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s5, s7, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s4, s6, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <2 x i32> %a, i32 5, i32 %b
   store <2 x i32> %vecins, <2 x i32> addrspace(1)* %out, align 8
   ret void
 }
 
 define amdgpu_kernel void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v3i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s5, s10, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s6, s9, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s8, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s6
+; SI-NEXT:    v_mov_b32_e32 v2, s5
 ; SI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v3i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x20
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
+; VI-NEXT:    s_cmp_lg_u32 s4, 2
+; VI-NEXT:    s_cselect_b32 s5, s10, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s6, s9, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s4, s8, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <3 x i32> %a, i32 5, i32 %b
   store <3 x i32> %vecins, <3 x i32> addrspace(1)* %out, align 16
   ret void
 }
 
 define amdgpu_kernel void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, i32 %b, [8 x i32], i32 %val) nounwind {
 ; SI-LABEL: dynamic_insertelement_v4i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
 ; SI-NEXT:    s_load_dword s6, s[4:5], 0x8
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x11
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s11
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
-; SI-NEXT:    v_mov_b32_e32 v4, s4
-; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-NEXT:    s_cmp_eq_u32 s6, 3
+; SI-NEXT:    s_cselect_b32 s5, s4, s11
+; SI-NEXT:    s_cmp_eq_u32 s6, 2
+; SI-NEXT:    s_cselect_b32 s7, s4, s10
+; SI-NEXT:    s_cmp_eq_u32 s6, 1
+; SI-NEXT:    s_cselect_b32 s9, s4, s9
+; SI-NEXT:    s_cmp_eq_u32 s6, 0
+; SI-NEXT:    s_cselect_b32 s4, s4, s8
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s7
+; SI-NEXT:    v_mov_b32_e32 v3, s5
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v4i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
 ; VI-NEXT:    s_load_dword s6, s[4:5], 0x20
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x44
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s11
-; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 3
-; VI-NEXT:    v_cndmask_b32_e32 v3, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, v0, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_eq_u32_e64 vcc, s6, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; VI-NEXT:    s_cmp_eq_u32 s6, 3
+; VI-NEXT:    s_cselect_b32 s5, s4, s11
+; VI-NEXT:    s_cmp_eq_u32 s6, 2
+; VI-NEXT:    s_cselect_b32 s7, s4, s10
+; VI-NEXT:    s_cmp_eq_u32 s6, 1
+; VI-NEXT:    s_cselect_b32 s9, s4, s9
+; VI-NEXT:    s_cmp_eq_u32 s6, 0
+; VI-NEXT:    s_cselect_b32 s4, s4, s8
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    v_mov_b32_e32 v2, s7
+; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <4 x i32> %a, i32 %val, i32 %b
   store <4 x i32> %vecins, <4 x i32> addrspace(1)* %out, align 16
   ret void
 }
 
 define amdgpu_kernel void @dynamic_insertelement_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v8i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x10
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s11
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; SI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v0, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s15
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; SI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s14
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; SI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s13
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s12
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_cmp_lg_u32 s4, 3
+; SI-NEXT:    s_cselect_b32 s5, s11, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s6, s10, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s7, s9, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s8, s8, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 7
+; SI-NEXT:    s_cselect_b32 s9, s15, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 6
+; SI-NEXT:    s_cselect_b32 s10, s14, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 5
+; SI-NEXT:    s_cselect_b32 s11, s13, 5
+; SI-NEXT:    s_cmp_lg_u32 s4, 4
+; SI-NEXT:    s_cselect_b32 s4, s12, 5
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s11
+; SI-NEXT:    v_mov_b32_e32 v2, s10
+; SI-NEXT:    v_mov_b32_e32 v3, s9
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    v_mov_b32_e32 v5, s7
+; SI-NEXT:    v_mov_b32_e32 v6, s6
+; SI-NEXT:    v_mov_b32_e32 v7, s5
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v8i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x40
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s11
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; VI-NEXT:    v_cndmask_b32_e32 v3, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s10
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; VI-NEXT:    v_cndmask_b32_e32 v2, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s9
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; VI-NEXT:    v_cndmask_b32_e32 v1, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v0, s8
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; VI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s15
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; VI-NEXT:    v_cndmask_b32_e32 v7, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s14
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; VI-NEXT:    v_cndmask_b32_e32 v6, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s13
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; VI-NEXT:    v_cndmask_b32_e32 v5, 5, v4, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, s12
-; VI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; VI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    s_cmp_lg_u32 s4, 3
+; VI-NEXT:    s_cselect_b32 s5, s11, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 2
+; VI-NEXT:    s_cselect_b32 s6, s10, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cselect_b32 s7, s9, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 0
+; VI-NEXT:    s_cselect_b32 s8, s8, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 7
+; VI-NEXT:    s_cselect_b32 s9, s15, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 6
+; VI-NEXT:    s_cselect_b32 s10, s14, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 5
+; VI-NEXT:    s_cselect_b32 s11, s13, 5
+; VI-NEXT:    s_cmp_lg_u32 s4, 4
+; VI-NEXT:    s_cselect_b32 s4, s12, 5
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s11
+; VI-NEXT:    v_mov_b32_e32 v2, s10
+; VI-NEXT:    v_mov_b32_e32 v3, s9
+; VI-NEXT:    v_mov_b32_e32 v4, s8
+; VI-NEXT:    v_mov_b32_e32 v5, s7
+; VI-NEXT:    v_mov_b32_e32 v6, s6
+; VI-NEXT:    v_mov_b32_e32 v7, s5
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x i32> %a, i32 5, i32 %b
   store <8 x i32> %vecins, <8 x i32> addrspace(1)* %out, align 32
   ret void
 }
 
 define amdgpu_kernel void @dynamic_insertelement_v16i32(<16 x i32> addrspace(1)* %out, <16 x i32> %a, i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v16i32:

 }
 
 define amdgpu_kernel void @dynamic_insertelement_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> %a, i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v16i8:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x8
+; SI-NEXT:    s_movk_i32 s7, 0xff
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_lshr_b32 s5, s11, 24
-; SI-NEXT:    v_mov_b32_e32 v0, s5
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 15
-; SI-NEXT:    s_lshr_b32 s5, s11, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s5
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 14
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    s_movk_i32 s5, 0xff
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
+; SI-NEXT:    s_cmp_lg_u32 s4, 15
+; SI-NEXT:    s_cselect_b32 s5, s5, 5
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_lshr_b32 s6, s11, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 14
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_and_b32 s6, s6, s7
+; SI-NEXT:    s_or_b32 s5, s6, s5
+; SI-NEXT:    s_lshl_b32 s5, s5, 16
 ; SI-NEXT:    s_lshr_b32 s6, s11, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s6
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 13
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, s11
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 12
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v2, s5, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    s_mov_b32 s6, 0xffff
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s10, 24
-; SI-NEXT:    v_or_b32_e32 v3, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 11
-; SI-NEXT:    s_lshr_b32 s7, s10, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 10
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
-; SI-NEXT:    s_lshr_b32 s7, s10, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 9
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, s10
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 8
-; SI-NEXT:    v_cndmask_b32_e32 v2, 5, v2, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v2, s5, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s9, 24
-; SI-NEXT:    v_or_b32_e32 v2, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 7
-; SI-NEXT:    s_lshr_b32 s7, s9, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 6
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
-; SI-NEXT:    s_lshr_b32 s7, s9, 8
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v1, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 5
-; SI-NEXT:    v_cndmask_b32_e32 v1, 5, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s9
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 4
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_and_b32_e32 v4, s5, v4
-; SI-NEXT:    v_or_b32_e32 v1, v4, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v1
-; SI-NEXT:    s_lshr_b32 s7, s8, 24
-; SI-NEXT:    v_or_b32_e32 v1, v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 3
-; SI-NEXT:    s_lshr_b32 s7, s8, 16
-; SI-NEXT:    v_cndmask_b32_e32 v0, 5, v0, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 2
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v4, s5, v4
-; SI-NEXT:    s_lshr_b32 s7, s8, 8
-; SI-NEXT:    v_or_b32_e32 v0, v4, v0
-; SI-NEXT:    v_mov_b32_e32 v4, s7
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 1
-; SI-NEXT:    v_cndmask_b32_e32 v4, 5, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v5, s8
-; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
-; SI-NEXT:    v_cndmask_b32_e32 v5, 5, v5, vcc
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT:    v_and_b32_e32 v5, s5, v5
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_and_b32_e32 v4, s6, v4
-; SI-NEXT:    v_or_b32_e32 v0, v4, v0
+; SI-NEXT:    s_cmp_lg_u32 s4, 13
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_lshl_b32 s6, s6, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 12
+; SI-NEXT:    s_cselect_b32 s11, s11, 5
+; SI-NEXT:    s_and_b32 s11, s11, s7
+; SI-NEXT:    s_or_b32 s6, s11, s6
+; SI-NEXT:    s_mov_b32 s11, 0xffff
+; SI-NEXT:    s_and_b32 s6, s6, s11
+; SI-NEXT:    s_or_b32 s5, s6, s5
+; SI-NEXT:    s_lshr_b32 s6, s10, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 11
+; SI-NEXT:    s_cselect_b32 s6, s6, 5
+; SI-NEXT:    s_lshl_b32 s6, s6, 8
+; SI-NEXT:    s_lshr_b32 s12, s10, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 10
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s6, s12, s6
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
+; SI-NEXT:    s_lshr_b32 s12, s10, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 9
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 8
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_and_b32 s10, s10, s7
+; SI-NEXT:    s_or_b32 s10, s10, s12
+; SI-NEXT:    s_and_b32 s10, s10, s11
+; SI-NEXT:    s_or_b32 s6, s10, s6
+; SI-NEXT:    s_lshr_b32 s10, s9, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 7
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_lshl_b32 s10, s10, 8
+; SI-NEXT:    s_lshr_b32 s12, s9, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 6
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s10, s12, s10
+; SI-NEXT:    s_lshl_b32 s10, s10, 16
+; SI-NEXT:    s_lshr_b32 s12, s9, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 5
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 4
+; SI-NEXT:    s_cselect_b32 s9, s9, 5
+; SI-NEXT:    s_and_b32 s9, s9, s7
+; SI-NEXT:    s_or_b32 s9, s9, s12
+; SI-NEXT:    s_and_b32 s9, s9, s11
+; SI-NEXT:    s_or_b32 s9, s9, s10
+; SI-NEXT:    s_lshr_b32 s10, s8, 24
+; SI-NEXT:    s_cmp_lg_u32 s4, 3
+; SI-NEXT:    s_cselect_b32 s10, s10, 5
+; SI-NEXT:    s_lshl_b32 s10, s10, 8
+; SI-NEXT:    s_lshr_b32 s12, s8, 16
+; SI-NEXT:    s_cmp_lg_u32 s4, 2
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_and_b32 s12, s12, s7
+; SI-NEXT:    s_or_b32 s10, s12, s10
+; SI-NEXT:    s_lshl_b32 s10, s10, 16
+; SI-NEXT:    s_lshr_b32 s12, s8, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cselect_b32 s12, s12, 5
+; SI-NEXT:    s_lshl_b32 s12, s12, 8
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
+; SI-NEXT:    s_cselect_b32 s4, s8, 5
+; SI-NEXT:    s_and_b32 s4, s4, s7
+; SI-NEXT:    s_or_b32 s4, s4, s12
+; SI-NEXT:    s_and_b32 s4, s4, s11
+; SI-NEXT:    s_or_b32 s4, s4, s10
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_mov_b32_e32 v2, s6
+; SI-NEXT:    v_mov_b32_e32 v3, s5
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v16i8:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x20

llvm/test/CodeGen/AMDGPU/mad_uint24.ll

 ; SimplifyDemandedBits on the first operand of the mul instruction
 ; assuming that the pattern would be matched to a 24-bit mad.  This
 ; led to some instructions being incorrectly erased when the entire
 ; 24-bit mad pattern wasn't being matched.
 
 ; Check that the select instruction is not deleted.
 ; FUNC-LABEL: {{^}}i24_i32_i32_mad:
 ; EG: CNDE_INT
-; SI: v_cndmask
+; SI: s_cselect
 define amdgpu_kernel void @i24_i32_i32_mad(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
 entry:
   %0 = ashr i32 %a, 8
   %1 = icmp ne i32 %c, 0
   %2 = select i1 %1, i32 %0, i32 34
   %3 = mul i32 %2, %c
   %4 = add i32 %3, %d
   store i32 %4, i32 addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/sad.ll

define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) {
   %ret = add i32 %ret0, %c
 
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_sad_u32_multi_use_select_pat2:
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DAG: s_cmp_gt_u32  s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN-DAG: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) {
   %icmp0 = icmp ugt i32 %a, %b
   %sub0 = sub i32 %a, %b
   %sub1 = sub i32 %b, %a
   %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1
   store volatile i32 %ret0, i32  addrspace(5)*undef
 
define amdgpu_kernel void @v_sad_u32_i8_pat2(i8 addrspace(1)* %out) {
   %ret = add i8 %ret0, %c
 
   store i8 %ret, i8 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}s_sad_u32_i8_pat2:
 ; GCN: s_load_dword
-; GCN: s_bfe_u32
+; GCN-DAG: s_bfe_u32
 ; GCN-DAG: s_sub_i32
 ; GCN-DAG: s_and_b32
 ; GCN-DAG: s_sub_i32
 ; GCN-DAG: s_lshr_b32
-; GCN: v_add_i32_e32
+; GCN: s_add_i32
 define amdgpu_kernel void @s_sad_u32_i8_pat2(i8 addrspace(1)* %out, i8 zeroext %a, i8 zeroext %b, i8 zeroext %c) {
   %icmp0 = icmp ugt i8 %a, %b
   %sub0 = sub i8 %a, %b
   %sub1 = sub i8 %b, %a
   %ret0 = select i1 %icmp0, i8 %sub0, i8 %sub1
 
   %ret = add i8 %ret0, %c
 
   store i8 %ret, i8 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat1:
-; GCN: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; GCN: s_max_u32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN: s_cmp_le_u32 s{{[0-9]+}}, s{{[0-9]+}}
+; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
   %icmp0 = icmp ugt i32 %a, %b
   %t0 = select i1 %icmp0, i32 %a, i32 %b
 
   %icmp1 = icmp ule i32 %a, %b
   %t1 = select i1 %icmp1, i32 %a, i32 %d
 
   %ret0 = sub i32 %t0, %t1
   %ret = add i32 %ret0, %c
 
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat2:
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) {
   %icmp0 = icmp ugt i32 %a, %b
   %sub0 = sub i32 %a, %d
   %sub1 = sub i32 %b, %a
   %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1
 
   %ret = add i32 %ret0, %c
 
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/sdiv.ll

 ; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    v_mul_f32_e32 v1, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v3, -v1, v2, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i8:
 ; TONGA:       ; %bb.0:
 ; TONGA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s3, 0xf000

 ; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; TONGA-NEXT:    v_mul_f32_e32 v1, v3, v4
 ; TONGA-NEXT:    v_trunc_f32_e32 v1, v1
 ; TONGA-NEXT:    v_mad_f32 v3, -v1, v2, v3
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v1, v1
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000

 ; GCN-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v3, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i23:
 ; TONGA:       ; %bb.0:
 ; TONGA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s3, 0xf000

 ; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
 ; TONGA-NEXT:    v_mul_f32_e32 v2, v1, v4
 ; TONGA-NEXT:    v_trunc_f32_e32 v2, v2
 ; TONGA-NEXT:    v_mad_f32 v1, -v2, v3, v1
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i23:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000

 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
 ; GCN-NEXT:    v_or_b32_e32 v1, 1, v1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v0, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i24:
 ; TONGA:       ; %bb.0:
 ; TONGA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s3, 0xf000

 ; TONGA-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
 ; TONGA-NEXT:    v_or_b32_e32 v1, 1, v1
 ; TONGA-NEXT:    v_mul_f32_e32 v3, v0, v4
 ; TONGA-NEXT:    v_trunc_f32_e32 v3, v3
 ; TONGA-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v3, v0
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i24:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000

llvm/test/CodeGen/AMDGPU/sdiv64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -amdgpu-codegenprepare-expand-div64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-IR %s
 
 define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_sdiv:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i32 s12, s3, 31
 ; GCN-NEXT:    s_add_u32 s2, s2, s12
 ; GCN-NEXT:    s_mov_b32 s13, s12
 ; GCN-NEXT:    s_addc_u32 s3, s3, s12
-; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[12:13]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
-; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[12:13]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_sub_u32 s2, 0, s14
+; GCN-NEXT:    s_subb_u32 s3, 0, s15
+; GCN-NEXT:    s_ashr_i32 s16, s11, 31
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s17, s16
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s4, v2
-; GCN-NEXT:    v_mul_lo_u32 v6, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v2, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v7, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v2, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s10, s14
-; GCN-NEXT:    s_addc_u32 s1, s11, s14
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, s10, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, s11, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, s11, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s16
+; GCN-NEXT:    s_addc_u32 s1, s11, s16
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[16:17]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s15, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s15
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s14, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[12:13]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[16:17], s[12:13]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
 ; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i32 s2, s9, 31
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s10, s6, s0
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i32 s8, s1, 31
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s10, s6, s2
-; GCN-IR-NEXT:    s_mov_b32 s9, s8
-; GCN-IR-NEXT:    s_subb_u32 s11, s7, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[8:9], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s8
-; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s6
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s8
-; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
+; GCN-IR-NEXT:    s_subb_u32 s11, s7, s0
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[12:13], s[14:15]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s12, s14
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s10
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
 ; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s11, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[14:15], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[12:13], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[10:11], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s11, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s14, s15
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_xor_b64 s[22:23], s[18:19], -1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[22:23], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[10:11], s14
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[10:11], v4
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[10:11], s18
 ; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[18:19]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %Flow7
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[8:9], s[2:3]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[18:19]
+; GCN-IR-NEXT:  BB0_6: ; %Flow7
+; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
 ; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0

 ; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_ashr_i64 s[4:5], s[6:7], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 40
   %2 = ashr i64 %y, 40
   %result = sdiv i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out

 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_xor_b32 s4, s7, s6
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_or_b32 s4, s4, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv32_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s7
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_xor_b32 s4, s7, s6
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 32
   %2 = ashr i64 %y, 32
   %result = sdiv i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out
   ret void

 ; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 33
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_ashr_i64 s[4:5], s[6:7], 33
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv31_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 33
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 33
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 33
   %2 = ashr i64 %y, 33
   %result = sdiv i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out

 ; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 41
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_ashr_i64 s[4:5], s[6:7], 41
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv23_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 41
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 41
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 41
   %2 = ashr i64 %y, 41
   %result = sdiv i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out

 ; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 39
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    s_ashr_i64 s[4:5], s[6:7], 39
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv25_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 39
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s8
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 39
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_xor_b32 s4, s4, s8
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s4, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 39
   %2 = ashr i64 %y, 39
   %result = sdiv i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out

 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[8:9], s[8:9], 40
 ; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s8
 ; GCN-NEXT:    s_xor_b32 s0, s8, s0
-; GCN-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s0
 ; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 40
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
+; GCN-NEXT:    s_or_b32 s3, s0, 1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s3, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s10
 ; GCN-NEXT:    s_xor_b32 s0, s10, s2
 ; GCN-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
 ; GCN-NEXT:    v_bfe_i32 v2, v2, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_v2i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x11
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[8:9], 40
 ; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s8
 ; GCN-IR-NEXT:    s_xor_b32 s0, s8, s0
-; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
-; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s0
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[2:3], 40
+; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
+; GCN-IR-NEXT:    s_or_b32 s3, s0, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[10:11], 40
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s3, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v2, s2
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v3, s10
 ; GCN-IR-NEXT:    s_xor_b32 s0, s10, s2
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-IR-NEXT:    s_or_b32 s2, s0, 1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-IR-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s2, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
 ; GCN-IR-NEXT:    v_bfe_i32 v2, v2, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GCN-IR-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr <2 x i64> %x, <i64 40, i64 40>
   %2 = ashr <2 x i64> %y, <i64 40, i64 40>
   %result = sdiv <2 x i64> %1, %2
   store <2 x i64> %result, <2 x i64> addrspace(1)* %out

 ; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_mul_f32_e32 v2, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v3, -v2, v1, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-IR-NEXT:    s_load_dword s3, s[0:1], 0xc
 ; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xd
 ; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 24
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s0
-; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[2:3], 24
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[12:13], s[6:7], 24
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s10, s6, s0
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
-; GCN-IR-NEXT:    s_ashr_i32 s6, s7, 31
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s10, s0, s2
-; GCN-IR-NEXT:    s_mov_b32 s7, s6
-; GCN-IR-NEXT:    s_subb_u32 s11, s1, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[6:7], s[8:9]
-; GCN-IR-NEXT:    s_sub_u32 s8, s0, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s1, s6
-; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
+; GCN-IR-NEXT:    s_subb_u32 s11, s7, s0
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[12:13], s[14:15]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s12, s14
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s14, s10
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
 ; GCN-IR-NEXT:    s_add_i32 s14, s14, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s15, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s15
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s14
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s11, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[14:15], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[12:13], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
-; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[10:11], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s11, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s14, s15
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[20:21], s[14:15], 63
+; GCN-IR-NEXT:    s_xor_b64 s[22:23], s[18:19], -1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[22:23], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[10:11], s14
+; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[10:11], v4
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[10:11], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB9_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_3
+; GCN-IR-NEXT:  BB9_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
 ; GCN-IR-NEXT:    s_branch BB9_6
-; GCN-IR-NEXT:  BB9_4:
+; GCN-IR-NEXT:  BB9_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[18:19]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB9_7
-; GCN-IR-NEXT:  BB9_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB9_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB9_7: ; %Flow4
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[6:7], s[2:3]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[18:19]
+; GCN-IR-NEXT:  BB9_6: ; %Flow4
+; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
 ; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-IR-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i48 %x, 24
   %2 = ashr i48 %y, 24
   %result = sdiv i48 %1, %2
   store i48 %result, i48 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-LABEL: s_test_sdiv_k_num_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-NEXT:    s_add_u32 s0, s6, s2
-; GCN-NEXT:    s_addc_u32 s1, s7, s2
-; GCN-NEXT:    s_mov_b32 s3, s2
-; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[2:3]
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
-; GCN-NEXT:    s_sub_u32 s3, 0, s8
-; GCN-NEXT:    s_subb_u32 s10, 0, s9
+; GCN-NEXT:    s_ashr_i32 s8, s7, 31
+; GCN-NEXT:    s_add_u32 s0, s6, s8
+; GCN-NEXT:    s_addc_u32 s1, s7, s8
+; GCN-NEXT:    s_mov_b32 s9, s8
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[8:9]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; GCN-NEXT:    s_sub_u32 s2, 0, s10
+; GCN-NEXT:    s_subb_u32 s3, 0, s11
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s10, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s3, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
 ; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
 ; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
 ; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s3, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s10, v0
+; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
 ; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
 ; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10

 ; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v0, 0, v0
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s9, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s11, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s11
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s10, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s8, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], 24, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s10, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[2:3], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s2
-; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
-; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s8
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[8:9], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s5, s4
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s4
+; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s4
+; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s6, s7
+; GCN-IR-NEXT:    s_add_u32 s8, s10, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[16:17], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s15, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], 24, s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s8, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], 24, s14
+; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s10, 58, s10
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0
 ; GCN-IR-NEXT:  BB10_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s6, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s6, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s6, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s6, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s6, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_3
+; GCN-IR-NEXT:  BB10_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[6:7], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB10_6
-; GCN-IR-NEXT:  BB10_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB10_7
 ; GCN-IR-NEXT:  BB10_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB10_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB10_7: ; %udiv-end
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[12:13]
+; GCN-IR-NEXT:  BB10_6: ; %udiv-end
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s4, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s5, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s4, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = sdiv i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define i64 @v_test_sdiv_k_num_i64(i64 %x) {
 ; GCN-LABEL: v_test_sdiv_k_num_i64:

 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_mov_b32 s7, 0x41c00000
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_ashr_i32 s4, s6, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
-; GCN-NEXT:    s_or_b32 s4, s4, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v2, -v1, v0, s7
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_k_num_i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0x41c00000
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s6, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, s7, v1
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v2, -v1, v0, s7
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %x.shr = ashr i64 %x, 40
   %result = sdiv i64 24, %x.shr
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @s_test_sdiv24_k_den_i64(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-LABEL: s_test_sdiv24_k_den_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s8, 0x46b6fe00
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_ashr_i32 s4, s6, 30
-; GCN-NEXT:    s_or_b32 s4, s4, 1
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v0, -v1, s8, v0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    s_or_b32 s6, s4, 1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, s8
+; GCN-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_k_den_i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_mov_b32 s8, 0x46b6fe00
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s4, s6, 30
-; GCN-IR-NEXT:    s_or_b32 s4, s4, 1
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v0, -v1, s8, v0
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s4
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    s_or_b32 s6, s4, 1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, s8
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GCN-IR-NEXT:    s_cselect_b32 s4, s6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s4, v1
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %x.shr = ashr i64 %x, 40
   %result = sdiv i64 %x.shr, 23423
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/select-i1.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; FIXME: This should go in existing select.ll test, except the current testcase there is broken on GCN
 
 ; GCN-LABEL: {{^}}select_i1:
 ; GCN: v_cndmask_b32
 ; GCN-NOT: v_cndmask_b32
 define amdgpu_kernel void @select_i1(i1 addrspace(1)* %out, i32 %cond, i1 %a, i1 %b) nounwind {

llvm/test/CodeGen/AMDGPU/select-opt.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; Make sure to test with f32 and i32 compares. If we have to use float
 ; compares, we always have multiple condition registers. If we can do
 ; scalar compares, we don't want to use multiple condition registers.
 
 ; GCN-LABEL: {{^}}opt_select_i32_and_cmp_i32:
 ; GCN-DAG: v_cmp_ne_u32_e32 vcc,
 ; GCN-DAG: v_cmp_ne_u32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_and_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_and_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_and_cmp_i32(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %x, i32 %y) #0 {
   %icmp0 = icmp ne i32 %a, %b
   %icmp1 = icmp ne i32 %a, %c
   %and = and i1 %icmp0, %icmp1
   %select = select i1 %and, i32 %x, i32 %y
   store i32 %select, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}opt_select_i32_and_cmp_f32:
 ; GCN-DAG: v_cmp_lg_f32_e32 vcc
 ; GCN-DAG: v_cmp_lg_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_and_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_and_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_and_cmp_f32(i32 addrspace(1)* %out, float %a, float %b, float %c, i32 %x, i32 %y) #0 {
   %fcmp0 = fcmp one float %a, %b
   %fcmp1 = fcmp one float %a, %c
   %and = and i1 %fcmp0, %fcmp1
   %select = select i1 %and, i32 %x, i32 %y
   store i32 %select, i32 addrspace(1)* %out
define amdgpu_kernel void @opt_select_i64_and_cmp_f32(i64 addrspace(1)* %out, float %a, float %b, float %c, i64 %x, i64 %y) #0 {
   %select = select i1 %and, i64 %x, i64 %y
   store i64 %select, i64 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}opt_select_i32_or_cmp_i32:
 ; GCN-DAG: v_cmp_ne_u32_e32 vcc,
 ; GCN-DAG: v_cmp_ne_u32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_or_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_or_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @opt_select_i32_or_cmp_i32(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %x, i32 %y) #0 {
   %icmp0 = icmp ne i32 %a, %b
   %icmp1 = icmp ne i32 %a, %c
   %or = or i1 %icmp0, %icmp1
   %select = select i1 %or, i32 %x, i32 %y
   store i32 %select, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}opt_select_i32_or_cmp_f32:
 ; GCN-DAG: v_cmp_lg_f32_e32 vcc
 ; GCN-DAG: v_cmp_lg_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]]
-; GCN: s_or_b64 vcc, vcc, [[CMP1]]
-; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}, vcc
+; GCN: s_or_b64 [[CMP1]], vcc, [[CMP1]]
+; GCN: s_cselect_b32 [[SRESULT:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+; GCN-NOT: [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN-NOT: [[RESULT]]
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @opt_select_i32_or_cmp_f32(i32 addrspace(1)* %out, float %a, float %b, float %c, i32 %x, i32 %y) #0 {
   %fcmp0 = fcmp one float %a, %b
   %fcmp1 = fcmp one float %a, %c
   %or = or i1 %fcmp0, %fcmp1
   %select = select i1 %or, i32 %x, i32 %y
   store i32 %select, i32 addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/select-vectors.ll

define amdgpu_kernel void @v_select_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> addrspace(1)* %a.ptr, <16 x i8> addrspace(1)* %b.ptr, i32 %c) #0 {
   %b = load <16 x i8>, <16 x i8> addrspace(1)* %b.ptr
   %cmp = icmp eq i32 %c, 0
   %select = select i1 %cmp, <16 x i8> %a, <16 x i8> %b
   store <16 x i8> %select, <16 x i8> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_v4i8:
-; GCN: v_cndmask_b32
-; GCN-NOT: cndmask
+; GCN: s_cselect_b32
+; GCN-NOT: s_cselect_b32
 define amdgpu_kernel void @select_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a, <4 x i8> %b, i8 %c) #0 {
   %cmp = icmp eq i8 %c, 0
   %select = select i1 %cmp, <4 x i8> %a, <4 x i8> %b
   store <4 x i8> %select, <4 x i8> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}select_v2i16:
 ; GFX89: s_load_dword
 ; GFX89: s_load_dword
 ; GFX89: s_load_dword
-; GFX89: v_cndmask_b32
-; GFX89-NOT: v_cndmask_b32
+; GFX89: s_cselect_b32
+; GFX89-NOT: s_cselect_b32
 
-; SI: v_cndmask_b32_e32
-; SI-NOT: v_cndmask_b32e
+; SI: s_cselect_b32
+; SI-NOT: s_cselect_b32
 define amdgpu_kernel void @select_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %a, <2 x i16> %b, i32 %c) #0 {
   %cmp = icmp eq i32 %c, 0
   %select = select i1 %cmp, <2 x i16> %a, <2 x i16> %b
   store <2 x i16> %select, <2 x i16> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_select_v2i16:

llvm/test/CodeGen/AMDGPU/select64.ll

 entry:
   %0 = icmp ugt i32 %cond, 5
   %1 = select i1 %0, i64 0, i64 %in
   store i64 %1, i64 addrspace(1)* %out
   ret void
 }
 
 ; CHECK-LABEL: {{^}}select_trunc_i64:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @select_trunc_i64(i32 addrspace(1)* %out, i32 %cond, i64 %in) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %sel = select i1 %cmp, i64 0, i64 %in
   %trunc = trunc i64 %sel to i32
   store i32 %trunc, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; CHECK-LABEL: {{^}}select_trunc_i64_2:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond, i64 %a, i64 %b) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %sel = select i1 %cmp, i64 %a, i64 %b
   %trunc = trunc i64 %sel to i32
   store i32 %trunc, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; CHECK-LABEL: {{^}}v_select_trunc_i64_2:
-; CHECK: v_cndmask_b32
-; CHECK-NOT: v_cndmask_b32
+; CHECK: s_cselect_b32
+; CHECK-NOT: s_cselect_b32
 define amdgpu_kernel void @v_select_trunc_i64_2(i32 addrspace(1)* %out, i32 %cond, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %cmp = icmp ugt i32 %cond, 5
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %b = load i64, i64 addrspace(1)* %bptr, align 8
   %sel = select i1 %cmp, i64 %a, i64 %b
   %trunc = trunc i64 %sel to i32
   store i32 %trunc, i32 addrspace(1)* %out, align 4
   ret void

llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll

define amdgpu_kernel void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32 %in) {
   store double %result, double addrspace(1)* %out
   ret void
 }
 
 ; We can't fold the SGPRs into v_cndmask_b32_e64, because it already
 ; uses an SGPR (implicit vcc).
 
 ; GCN-LABEL: {{^}}sint_to_fp_i1_f64:
-; GCN-DAG: v_cmp_eq_u32_e64 vcc,
-; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: s_cmp_eq_u32
+; GCN-DAG: s_cselect_b32 s[[SSEL:[0-9]+]], 0xbff00000, 0
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[SEL:[0-9]+]], s[[SSEL]]
 ; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @sint_to_fp_i1_f64(double addrspace(1)* %out, i32 %in) {
   %cmp = icmp eq i32 %in, 0
   %fp = sitofp i1 %cmp to double
   store double %fp, double addrspace(1)* %out, align 4
   ret void
 }

llvm/test/CodeGen/AMDGPU/srem64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -amdgpu-codegenprepare-expand-div64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-IR %s
 
 define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_srem:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
 ; GCN-NEXT:    s_sub_u32 s2, 0, s12
 ; GCN-NEXT:    s_subb_u32 s3, 0, s13
 ; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_add_i32 s14, s12, 32
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s14, s8
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s14, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s14
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s16, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s17, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[6:7], s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[6:7], s16
+; GCN-IR-NEXT:    s_add_u32 s8, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
+; GCN-IR-NEXT:    s_mov_b32 s15, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s2, s14
+; GCN-IR-NEXT:    s_addc_u32 s11, s3, s11
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s2, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s2, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[12:13]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[12:13]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s0, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s1, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0

 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 41
 ; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 41
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem23_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 41
 ; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 41
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-IR-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 41
   %2 = ashr i64 %y, 41
   %result = srem i64 %1, %2

 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-IR-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 40
   %2 = ashr i64 %y, 40
   %result = srem i64 %1, %2

 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 39
 ; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 39
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem25_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 39
 ; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 39
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-IR-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 25
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 39
   %2 = ashr i64 %y, 39
   %result = srem i64 %1, %2

 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 33
 ; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 33
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-NEXT:    s_or_b32 s1, s1, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem31_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s1, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 33
 ; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 33
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s6
 ; GCN-IR-NEXT:    s_xor_b32 s1, s6, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[8:9], 0
+; GCN-IR-NEXT:    s_cselect_b32 s1, s1, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s1, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 31
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 33
   %2 = ashr i64 %y, 33
   %result = srem i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 ; 32 known sign bits
 define amdgpu_kernel void @s_test_srem32_64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_srem32_64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_load_dword s0, s[0:1], 0xe
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s6, s[0:1], 0xe
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GCN-NEXT:    s_xor_b32 s1, s7, s0
-; GCN-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
+; GCN-NEXT:    s_xor_b32 s0, s7, s6
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s8, s0, 1
 ; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s8, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s6
 ; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s7, v0
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem32_64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
+; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
-; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GCN-IR-NEXT:    s_xor_b32 s1, s7, s0
-; GCN-IR-NEXT:    s_ashr_i32 s1, s1, 30
-; GCN-IR-NEXT:    s_or_b32 s1, s1, 1
+; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
+; GCN-IR-NEXT:    s_xor_b32 s0, s7, s6
+; GCN-IR-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-IR-NEXT:    s_or_b32 s8, s0, 1
 ; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
 ; GCN-IR-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s8, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s6
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
+; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s7, v0
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 32
   %2 = ashr i64 %y, 32
   %result = srem i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 ; 33 known sign bits
 define amdgpu_kernel void @s_test_srem33_64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_srem33_64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v7, 0
-; GCN-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_ashr_i64 s[2:3], s[10:11], 31
-; GCN-NEXT:    s_ashr_i64 s[4:5], s[0:1], 31
+; GCN-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
+; GCN-NEXT:    s_ashr_i64 s[8:9], s[0:1], 31
 ; GCN-NEXT:    s_ashr_i32 s0, s1, 31
-; GCN-NEXT:    s_add_u32 s4, s4, s0
+; GCN-NEXT:    s_add_u32 s8, s8, s0
 ; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_addc_u32 s5, s5, s0
-; GCN-NEXT:    s_xor_b64 s[12:13], s[4:5], s[0:1]
+; GCN-NEXT:    s_addc_u32 s9, s9, s0
+; GCN-NEXT:    s_xor_b64 s[12:13], s[8:9], s[0:1]
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
-; GCN-NEXT:    s_sub_u32 s4, 0, s12
-; GCN-NEXT:    s_subb_u32 s5, 0, s13
-; GCN-NEXT:    s_ashr_i32 s10, s11, 31
+; GCN-NEXT:    s_sub_u32 s6, 0, s12
+; GCN-NEXT:    s_subb_u32 s8, 0, s13
+; GCN-NEXT:    s_mov_b32 s9, s5
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-NEXT:    s_mov_b32 s11, s10
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s4, v2
-; GCN-NEXT:    v_mul_lo_u32 v6, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v9, v2, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v2, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v2, v4, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s8, v0
+; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
-; GCN-NEXT:    v_mul_hi_u32 v11, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v6
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v7, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, v3, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v2, v5, s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s2, s10
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    s_addc_u32 s1, s3, s10
-; GCN-NEXT:    s_xor_b64 s[14:15], s[0:1], s[10:11]
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_mul_lo_u32 v3, s14, v2
-; GCN-NEXT:    v_mul_hi_u32 v4, s14, v0
-; GCN-NEXT:    v_mul_hi_u32 v5, s14, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, s15, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, s15, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    s_ashr_i32 s6, s7, 31
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s2, s6
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s7, s6
+; GCN-NEXT:    s_addc_u32 s1, s3, s6
+; GCN-NEXT:    s_xor_b64 s[14:15], s[0:1], s[6:7]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s15, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s15, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, s15, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s15, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s14, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s15, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s14, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s15
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-NEXT:    v_xor_b32_e32 v0, s10, v0
-; GCN-NEXT:    v_xor_b32_e32 v1, s10, v1
-; GCN-NEXT:    v_mov_b32_e32 v2, s10
-; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT:    v_xor_b32_e32 v0, s6, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s6, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem33_64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[0:1], 31
-; GCN-IR-NEXT:    s_ashr_i32 s0, s1, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
+; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 31
+; GCN-IR-NEXT:    s_ashr_i32 s10, s1, 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[2:3]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[10:11], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
-; GCN-IR-NEXT:    s_sub_u32 s8, s8, s0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s9, s0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[12:13], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
+; GCN-IR-NEXT:    s_mov_b32 s11, s10
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s10
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s10
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[10:11], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[20:21], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[20:21], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s19, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[2:3], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[2:3], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB8_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_3
+; GCN-IR-NEXT:  BB8_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN-IR-NEXT:    s_branch BB8_6
-; GCN-IR-NEXT:  BB8_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB8_7
 ; GCN-IR-NEXT:  BB8_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB8_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB8_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s8, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s8, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s9, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s8, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB8_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i64 %x, 31
   %2 = ashr i64 %y, 31
   %result = srem i64 %1, %2
   store i64 %result, i64 addrspace(1)* %out
   ret void

 ; GCN-NEXT:    v_or_b32_e32 v5, 1, v5
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v4, v4
 ; GCN-NEXT:    v_mad_f32 v3, -v4, v1, v3
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
 ; GCN-NEXT:    v_mul_lo_u32 v0, v1, v0
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v2, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-IR-NEXT:    s_load_dword s3, s[0:1], 0xc
 ; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xd
 ; GCN-IR-NEXT:    s_load_dword s0, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 24
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s0
-; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[2:3], 24
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
-; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s3, s2
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
-; GCN-IR-NEXT:    s_mov_b32 s11, s10
-; GCN-IR-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GCN-IR-NEXT:    s_sub_u32 s6, s0, s2
-; GCN-IR-NEXT:    s_subb_u32 s7, s1, s2
-; GCN-IR-NEXT:    s_sub_u32 s8, s8, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s8
-; GCN-IR-NEXT:    s_subb_u32 s9, s9, s10
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s9
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s9, 0
-; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s13, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s13
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s12
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[12:13], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[8:9], 0
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s12, s7, 31
+; GCN-IR-NEXT:    s_mov_b32 s1, s0
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[6:7], 24
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[8:9], s[0:1]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
+; GCN-IR-NEXT:    s_sub_u32 s6, s6, s12
+; GCN-IR-NEXT:    s_subb_u32 s7, s7, s12
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[10:11], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[10:11], s[12:13]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
+; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s16, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s12, s16
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[18:19], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[20:21], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[20:21], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s18, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s19, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[18:19], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[2:3], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s10, s8, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s11, s9, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[18:19], s[2:3], s18
+; GCN-IR-NEXT:    s_add_u32 s10, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, -1
+; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_mov_b32 s17, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_addc_u32 s13, s9, s13
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:  BB9_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s11
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s10, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s8, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s9, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s8, s15, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[18:19], s[18:19], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[8:9]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[16:17], s[14:15]
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s18
+; GCN-IR-NEXT:    s_subb_u32 s8, s11, s19
+; GCN-IR-NEXT:    s_ashr_i32 s16, s8, 31
+; GCN-IR-NEXT:    s_mov_b32 s17, s16
+; GCN-IR-NEXT:    s_and_b32 s8, s16, 1
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[16:17], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s18, s18, s20
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-IR-NEXT:    s_subb_u32 s19, s19, s21
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB9_3
+; GCN-IR-NEXT:  BB9_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[14:15], 1
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN-IR-NEXT:    s_branch BB9_6
-; GCN-IR-NEXT:  BB9_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB9_7
 ; GCN-IR-NEXT:  BB9_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB9_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB9_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s8, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s8, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s9, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s8, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB9_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, s2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-IR-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-IR-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = ashr i48 %x, 24
   %2 = ashr i48 %y, 24
   %result = srem i48 %1, %2
   store i48 %result, i48 addrspace(1)* %out

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s8, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s9, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s8, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], 24, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s9
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s8, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s9, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s8, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s8, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s9, v5
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s8, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s8, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s8, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s9, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
-; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[6:7], s[0:1]
-; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
-; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
-; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s6
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[6:7], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-IR-NEXT:    s_mov_b32 s5, s4
+; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GCN-IR-NEXT:    s_sub_u32 s2, s2, s4
+; GCN-IR-NEXT:    s_subb_u32 s3, s3, s4
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
 ; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB10_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB10_3
+; GCN-IR-NEXT:  BB10_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB10_6
-; GCN-IR-NEXT:  BB10_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB10_7
 ; GCN-IR-NEXT:  BB10_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB10_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB10_7: ; %udiv-end
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB10_6: ; %udiv-end
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
 ; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = srem i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define i64 @v_test_srem_k_num_i64(i64 %x) {
 ; GCN-LABEL: v_test_srem_k_num_i64:

 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s6, 0x41c00000
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[4:5], s[2:3], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s4
 ; GCN-NEXT:    s_ashr_i32 s5, s4, 30
 ; GCN-NEXT:    s_or_b32 s5, s5, 1
-; GCN-NEXT:    v_mov_b32_e32 v3, s5
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mad_f32 v2, -v1, v0, s6
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
+; GCN-NEXT:    s_cmp_lg_u64 s[6:7], 0
+; GCN-NEXT:    s_cselect_b32 s5, s5, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s5, v1
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s4
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_k_num_i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_mov_b32 s6, 0x41c00000
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[2:3], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s4
 ; GCN-IR-NEXT:    s_ashr_i32 s5, s4, 30
 ; GCN-IR-NEXT:    s_or_b32 s5, s5, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, s5
-; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    v_rcp_iflag_f32_e32 v1, v0
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    v_mul_f32_e32 v1, s6, v1
 ; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-IR-NEXT:    v_mad_f32 v2, -v1, v0, s6
 ; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v0|
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[6:7], 0
+; GCN-IR-NEXT:    s_cselect_b32 s5, s5, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s5, v1
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s4
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %x.shr = ashr i64 %x, 40
   %result = srem i64 24, %x.shr
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @s_test_srem24_k_den_i64(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-LABEL: s_test_srem24_k_den_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s8, 0x46b6fe00
+; GCN-NEXT:    s_mov_b32 s1, 0x46b6fe00
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-NEXT:    s_ashr_i32 s0, s6, 30
-; GCN-NEXT:    s_or_b32 s0, s0, 1
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x38331158, v0
-; GCN-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-NEXT:    v_mad_f32 v0, -v2, s8, v0
-; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_or_b32 s7, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mad_f32 v0, -v1, s1, v0
+; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v0|, s1
+; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-NEXT:    s_cselect_b32 s0, s7, 0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v1
 ; GCN-NEXT:    s_movk_i32 s0, 0x5b7f
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_k_den_i64:
 ; GCN-IR:       ; %bb.0:
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_mov_b32 s8, 0x46b6fe00
+; GCN-IR-NEXT:    s_mov_b32 s1, 0x46b6fe00
 ; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s2, -1
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[6:7], 40
 ; GCN-IR-NEXT:    v_cvt_f32_i32_e32 v0, s6
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s6, 30
-; GCN-IR-NEXT:    s_or_b32 s0, s0, 1
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-IR-NEXT:    v_mul_f32_e32 v2, 0x38331158, v0
-; GCN-IR-NEXT:    v_trunc_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_mad_f32 v0, -v2, s8, v0
-; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GCN-IR-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s8
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-IR-NEXT:    s_or_b32 s7, s0, 1
+; GCN-IR-NEXT:    v_mul_f32_e32 v1, 0x38331158, v0
+; GCN-IR-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-IR-NEXT:    v_mad_f32 v0, -v1, s1, v0
+; GCN-IR-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; GCN-IR-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v0|, s1
+; GCN-IR-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; GCN-IR-NEXT:    s_cselect_b32 s0, s7, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s0, v1
 ; GCN-IR-NEXT:    s_movk_i32 s0, 0x5b7f
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GCN-IR-NEXT:    s_mov_b32 s0, s4
 ; GCN-IR-NEXT:    s_mov_b32 s1, s5
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; GCN-IR-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm

llvm/test/CodeGen/AMDGPU/trunc.ll

define amdgpu_kernel void @sgpr_trunc_i32_to_i1(i32 addrspace(1)* %out, i32 %a) {
   ret void
 }
 
 ; GCN-LABEL: {{^}}s_trunc_i64_to_i1:
 ; SI: s_load_dwordx2 s{{\[}}[[SLO:[0-9]+]]:{{[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0x13
 ; VI: s_load_dwordx2 s{{\[}}[[SLO:[0-9]+]]:{{[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0x4c
 ; GCN: s_and_b32 [[MASKED:s[0-9]+]], 1, s[[SLO]]
 ; GCN: v_cmp_eq_u32_e64 s{{\[}}[[VLO:[0-9]+]]:[[VHI:[0-9]+]]], [[MASKED]], 1{{$}}
-; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, -12, 63, s{{\[}}[[VLO]]:[[VHI]]]
+; GCN: s_cmp_lg_u64 s{{\[}}[[VLO]]:[[VHI]]], 0
+; GCN: s_cselect_b32 {{s[0-9]+}}, 63, -12
 define amdgpu_kernel void @s_trunc_i64_to_i1(i32 addrspace(1)* %out, [8 x i32], i64 %x) {
   %trunc = trunc i64 %x to i1
   %sel = select i1 %trunc, i32 63, i32 -12
   store i32 %sel, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_trunc_i64_to_i1:

llvm/test/CodeGen/AMDGPU/udiv64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -amdgpu-codegenprepare-expand-div64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-IR %s
 
 define amdgpu_kernel void @s_test_udiv_i64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_udiv_i64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
-; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GCN-NEXT:    s_sub_u32 s4, 0, s2
-; GCN-NEXT:    s_subb_u32 s5, 0, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GCN-NEXT:    s_sub_u32 s2, 0, s12
+; GCN-NEXT:    s_subb_u32 s3, 0, s13
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s4, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s5, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s4, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, s4, v0
-; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
-; GCN-NEXT:    s_mov_b32 s5, s9
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, s4, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    s_mov_b32 s4, s8
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s12, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s12, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s13, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s13
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v6, s11
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v6, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_add_i32 s12, s12, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s12, s8
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s8, s12
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s15, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[6:7], s14
+; GCN-IR-NEXT:    s_add_u32 s6, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[8:9]
+; GCN-IR-NEXT:    s_mov_b32 s13, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s2, s12
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, s9
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s2, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s2, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s2, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[0:1], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = udiv i64 %x, %y
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }

 ; GCN-NEXT:    v_mul_lo_u32 v6, v0, v1
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_sub_i32_e32 v3, vcc, v3, v6
 ; GCN-NEXT:    v_subb_u32_e32 v4, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_sub_i32_e32 v5, vcc, v3, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v6, vcc, 0, v4, vcc
-; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v0
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, v0, v5
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 2, v1
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 1, v1
-; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], v3, v0
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v5, v8, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v9, v7, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, v0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v8, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv24_i48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; GCN-IR-NEXT:    s_load_dword s3, s[0:1], 0xc
 ; GCN-IR-NEXT:    s_load_dword s6, s[0:1], 0xd
 ; GCN-IR-NEXT:    s_load_dword s7, s[0:1], 0xe
 ; GCN-IR-NEXT:    s_mov_b32 s8, 0xffff
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0xff000000
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_and_b32 s1, s3, s8
 ; GCN-IR-NEXT:    s_and_b32 s0, s2, s9
 ; GCN-IR-NEXT:    s_and_b32 s3, s7, s8
 ; GCN-IR-NEXT:    s_and_b32 s2, s6, s9
-; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[2:3], 24
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
 ; GCN-IR-NEXT:    s_lshr_b64 s[6:7], s[0:1], 24
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
+; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[2:3], 24
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    s_mov_b64 s[0:1], 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s2
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s8, s10
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s12, s10, s11
+; GCN-IR-NEXT:    s_sub_u32 s10, s8, s12
+; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[14:15], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s14, s10, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_addc_u32 s15, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[14:15], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[6:7], s14
 ; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_not_b64 s[0:1], s[8:9]
+; GCN-IR-NEXT:    s_mov_b32 s13, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s0, s12
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, s9
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s1, 0
 ; GCN-IR-NEXT:  BB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s6, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s0, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s0, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s0, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s0, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s0, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[0:1]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_3
+; GCN-IR-NEXT:  BB7_4: ; %Flow3
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-IR-NEXT:    s_branch BB7_6
-; GCN-IR-NEXT:  BB7_4:
+; GCN-IR-NEXT:  BB7_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[14:15]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB7_7
-; GCN-IR-NEXT:  BB7_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB7_6: ; %Flow3
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB7_7: ; %udiv-end
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[14:15]
+; GCN-IR-NEXT:  BB7_6: ; %udiv-end
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
 ; GCN-IR-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-IR-NEXT:    s_endpgm
   %1 = lshr i48 %x, 24
   %2 = lshr i48 %y, 24
   %result = udiv i48 %1, %2

 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
-; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
-; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s6, v3
-; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s7, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s6, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s7, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
+; GCN-NEXT:    v_sub_i32_e64 v3, s[0:1], 24, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, vcc, v4, v5, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v3
+; GCN-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v4
+; GCN-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v4
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[2:3], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s2, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s3, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
+; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB8_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB8_3
+; GCN-IR-NEXT:  BB8_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB8_6
-; GCN-IR-NEXT:  BB8_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB8_7
 ; GCN-IR-NEXT:  BB8_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB8_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB8_7: ; %udiv-end
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB8_6: ; %udiv-end
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = udiv i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 ; define i64 @v_test_udiv_k_num_i64(i64 %x) {
 ;   %result = udiv i64 24, %x

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, 24
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, 24
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, 24
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
 ; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v4
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], 23, v4
 ; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v4, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v8, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 59, v2
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[2:3], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB11_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_sub_u32 s6, 59, s8
+; GCN-IR-NEXT:    s_subb_u32 s7, 0, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB11_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s10, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB11_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v3
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffc4, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], 0, -1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s2, s8, 0xffffffc4
+; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB11_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, 23, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, 0, v7, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, 24, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v7, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s7, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[6:7], 1
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[6:7], s[8:9], s[6:7]
+; GCN-IR-NEXT:    s_sub_u32 s4, 23, s10
+; GCN-IR-NEXT:    s_subb_u32 s4, 0, s11
+; GCN-IR-NEXT:    s_ashr_i32 s8, s4, 31
+; GCN-IR-NEXT:    s_and_b32 s4, s8, 1
+; GCN-IR-NEXT:    s_and_b32 s8, s8, 24
+; GCN-IR-NEXT:    s_sub_u32 s10, s10, s8
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    s_subb_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-IR-NEXT:    s_add_u32 s2, s2, 1
+; GCN-IR-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[8:9], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB11_3
+; GCN-IR-NEXT:  BB11_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[6:7], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB11_6
-; GCN-IR-NEXT:  BB11_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB11_7
 ; GCN-IR-NEXT:  BB11_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB11_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB11_7: ; %udiv-end
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[10:11]
+; GCN-IR-NEXT:  BB11_6: ; %udiv-end
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = udiv i64 %x, 24
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define i64 @v_test_udiv_k_den_i64(i64 %x) {
 ; GCN-LABEL: v_test_udiv_k_den_i64:

llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll

define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i32> %in) {
   store <4 x double> %cast, <4 x double> addrspace(1)* %out, align 16
   ret void
 }
 
 ; We can't fold the SGPRs into v_cndmask_b32_e32, because it already
 ; uses an SGPR (implicit vcc).
 
 ; GCN-LABEL: {{^}}uint_to_fp_i1_to_f64:
-; GCN-DAG: v_cmp_eq_u32_e64 vcc
-; GCN-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}}
+; GCN-DAG: s_cmp_eq_u32
+; GCN-DAG: s_cselect_b32 s[[SSEL:[0-9]+]], 0x3ff00000, 0
+; GCN-DAG: v_mov_b32_e32 v[[SEL:[0-9]+]], s[[SSEL]]
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
 ; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[ZERO]]:[[SEL]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @uint_to_fp_i1_to_f64(double addrspace(1)* %out, i32 %in) {
   %cmp = icmp eq i32 %in, 0
   %fp = uitofp i1 %cmp to double
   store double %fp, double addrspace(1)* %out, align 4
   ret void

llvm/test/CodeGen/AMDGPU/urem64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=gfx600 -amdgpu-bypass-slow-div=0 -amdgpu-codegenprepare-expand-div64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-IR %s
 
 define amdgpu_kernel void @s_test_urem_i64(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ; GCN-LABEL: s_test_urem_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0xd
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
 ; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
 ; GCN-NEXT:    s_sub_u32 s2, 0, s12
 ; GCN-NEXT:    s_subb_u32 s3, 0, s13
 ; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v3
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
-; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
-; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
-; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_hi_u32 v8, v1, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v6, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
-; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
-; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
-; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
-; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, s10, v0
-; GCN-NEXT:    v_mul_hi_u32 v6, s10, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, s11, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, s11, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, s11, v0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v5, v0, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], s10, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s13
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s13, v5
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s12, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s12, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v5, s11
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v5, v1, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, v5, v1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_urem_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[6:7], 0
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s6
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 0
-; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s11
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s10
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v2, v3
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[2:3], 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[10:11], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[8:9]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[4:5], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[0:1], 0
+; GCN-IR-NEXT:    s_flbit_i32_b32 s12, s0
+; GCN-IR-NEXT:    s_add_i32 s14, s12, 32
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s1
+; GCN-IR-NEXT:    s_cmp_eq_u32 s1, 0
+; GCN-IR-NEXT:    s_cselect_b32 s10, s14, s8
+; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s6
+; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s7
+; GCN-IR-NEXT:    s_cmp_eq_u32 s7, 0
+; GCN-IR-NEXT:    s_cselect_b32 s14, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s8, s10, s14
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[16:17], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[18:19], s[12:13], -1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[18:19], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s16, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s17, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[6:7], s8
+; GCN-IR-NEXT:    s_cbranch_vccz BB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_not_b32_e32 v2, v2
-; GCN-IR-NEXT:    s_add_u32 s8, s2, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v4
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v2, v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s9, s3, -1
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], -1, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[6:7], s16
+; GCN-IR-NEXT:    s_add_u32 s8, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s9, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
+; GCN-IR-NEXT:    s_mov_b32 s15, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s2, s14
+; GCN-IR-NEXT:    s_addc_u32 s11, s3, s11
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  BB0_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s8, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s2, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s3, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s2, s13, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[16:17], s[16:17], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_sub_u32 s2, s8, s16
+; GCN-IR-NEXT:    s_subb_u32 s2, s9, s17
+; GCN-IR-NEXT:    s_ashr_i32 s14, s2, 31
+; GCN-IR-NEXT:    s_mov_b32 s15, s14
+; GCN-IR-NEXT:    s_and_b32 s2, s14, 1
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[14:15], s[0:1]
+; GCN-IR-NEXT:    s_sub_u32 s16, s16, s18
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    s_subb_u32 s17, s17, s19
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s11, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB0_3
+; GCN-IR-NEXT:  BB0_4: ; %Flow6
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[12:13], 1
+; GCN-IR-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-IR-NEXT:    s_branch BB0_6
-; GCN-IR-NEXT:  BB0_4:
+; GCN-IR-NEXT:  BB0_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[12:13]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB0_7
-; GCN-IR-NEXT:  BB0_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB0_6: ; %Flow6
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB0_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[12:13]
+; GCN-IR-NEXT:  BB0_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s0, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s1, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-IR-NEXT:    s_mov_b32 s6, -1
 ; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0

 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v3, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s6, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s6, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s7, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s6, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e64 v0, s[0:1], 24, v0
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s7
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
-; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s6, v0
-; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s7, v5
-; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s6, v4
-; GCN-NEXT:    v_subrev_i32_e64 v3, s[0:1], s6, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], s7, v5
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
-; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[2:3], s6, v0
+; GCN-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v2, s[2:3]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
+; GCN-NEXT:    v_subb_u32_e64 v2, vcc, v2, v3, s[2:3]
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_subb_u32_e64 v1, vcc, 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[2:3], 0, v6
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v1
 ; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v3, s[2:3]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_urem_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc5, v2
-; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[2:3], 0, -1, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB6_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], 24, v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s8, s4, s5
+; GCN-IR-NEXT:    s_add_u32 s6, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[6:7], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB6_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], 24, s6
+; GCN-IR-NEXT:    s_cbranch_vccz BB6_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_add_u32 s2, s6, -1
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], 24, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 58, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    s_addc_u32 s3, s7, -1
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[0:1], 0, 0, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], 24, s12
+; GCN-IR-NEXT:    s_add_u32 s6, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s7, s3, -1
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB6_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, s2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, v2, v7, vcc
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, s6, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v11, s7, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[0:1], v7, v11, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s11, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[14:15], s[14:15], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
+; GCN-IR-NEXT:    s_sub_u32 s4, s6, s14
+; GCN-IR-NEXT:    s_subb_u32 s4, s7, s15
+; GCN-IR-NEXT:    s_ashr_i32 s12, s4, 31
+; GCN-IR-NEXT:    s_mov_b32 s13, s12
+; GCN-IR-NEXT:    s_and_b32 s4, s12, 1
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[12:13], s[2:3]
+; GCN-IR-NEXT:    s_sub_u32 s14, s14, s16
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_subb_u32 s15, s15, s17
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB6_3
+; GCN-IR-NEXT:  BB6_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB6_6
-; GCN-IR-NEXT:  BB6_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB6_7
 ; GCN-IR-NEXT:  BB6_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB6_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB6_7: ; %udiv-end
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, s6, v1
-; GCN-IR-NEXT:    v_mul_hi_u32 v2, s6, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, s7, v0
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, s6, v0
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, 24, 0, s[10:11]
+; GCN-IR-NEXT:  BB6_6: ; %udiv-end
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, s2, v1
+; GCN-IR-NEXT:    v_mul_hi_u32 v2, s2, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = urem i64 24, %x
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @s_test_urem_k_den_i64(i64 addrspace(1)* %out, i64 %x) {
 ; GCN-LABEL: s_test_urem_k_den_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
 ; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x41c00000
 ; GCN-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-NEXT:    s_movk_i32 s2, 0xffe8
 ; GCN-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GCN-NEXT:    v_trunc_f32_e32 v1, v1
 ; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v1, s2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v0, v2
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v3, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v3, vcc
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    s_mov_b32 s4, s8
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, s2
 ; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
 ; GCN-NEXT:    v_mul_lo_u32 v5, v2, s2
 ; GCN-NEXT:    v_mul_lo_u32 v6, v0, s2
 ; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v0, v4
-; GCN-NEXT:    s_mov_b32 s9, s5
+; GCN-NEXT:    s_mov_b32 s5, s9
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v10, v2, v6
 ; GCN-NEXT:    v_mul_hi_u32 v6, v2, v6
 ; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
 ; GCN-NEXT:    v_mul_hi_u32 v2, v0, 24
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, 24
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, 24
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GCN-NEXT:    v_mov_b32_e32 v2, s7
+; GCN-NEXT:    v_mov_b32_e32 v2, s11
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
 ; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, 24, v2
 ; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v3, vcc
 ; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, -1, v6, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], 23, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GCN-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[0:1]
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, 23, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_urem_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
-; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-IR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s6
-; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s7
-; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, s2
-; GCN-IR-NEXT:    v_cmp_eq_u32_e64 vcc, s7, 0
-; GCN-IR-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 59, v2
-; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[2:3], 0, 0, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], 0
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], vcc
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[0:1], -1
-; GCN-IR-NEXT:    s_and_b64 s[2:3], s[2:3], vcc
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
-; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[0:1], v[3:4], v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 63, v0
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], s[6:7], v0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s2
+; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
+; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s3
+; GCN-IR-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-IR-NEXT:    s_cselect_b32 s6, s4, s5
+; GCN-IR-NEXT:    s_sub_u32 s8, 59, s6
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[8:9], 63
+; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[12:13], s[8:9], 63
+; GCN-IR-NEXT:    s_xor_b64 s[14:15], s[10:11], -1
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[14:15], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_5
+; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
+; GCN-IR-NEXT:    s_sub_i32 s7, 63, s8
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, vcc
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s7
+; GCN-IR-NEXT:    s_cbranch_vccz BB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[6:7], s[6:7], v3
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffc4, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[0:1], 0, -1, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s6, s6, 0xffffffc4
+; GCN-IR-NEXT:    s_addc_u32 s7, 0, -1
+; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
+; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  BB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, 23, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, 0, v7, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
-; GCN-IR-NEXT:    v_and_b32_e32 v10, 24, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, v8
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[0:1], v6, v10
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v7, s[0:1]
+; GCN-IR-NEXT:    s_lshr_b32 s4, s9, 31
+; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[8:9], 1
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[4:5]
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; GCN-IR-NEXT:    s_sub_u32 s4, 23, s12
+; GCN-IR-NEXT:    s_subb_u32 s4, 0, s13
+; GCN-IR-NEXT:    s_ashr_i32 s10, s4, 31
+; GCN-IR-NEXT:    s_and_b32 s4, s10, 1
+; GCN-IR-NEXT:    s_and_b32 s10, s10, 24
+; GCN-IR-NEXT:    s_sub_u32 s12, s12, s10
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT:    s_subb_u32 s13, s13, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-IR-NEXT:    s_add_u32 s6, s6, 1
+; GCN-IR-NEXT:    s_addc_u32 s7, s7, 0
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, vcc
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_cbranch_vccz BB7_3
+; GCN-IR-NEXT:  BB7_4: ; %Flow5
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], 1
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-IR-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-IR-NEXT:    s_branch BB7_6
-; GCN-IR-NEXT:  BB7_4:
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s7
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s6
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; GCN-IR-NEXT:    s_branch BB7_7
 ; GCN-IR-NEXT:  BB7_5:
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
-; GCN-IR-NEXT:  BB7_6: ; %Flow5
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v1, v3, v1
-; GCN-IR-NEXT:  BB7_7: ; %udiv-end
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[10:11]
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[10:11]
+; GCN-IR-NEXT:  BB7_6: ; %udiv-end
 ; GCN-IR-NEXT:    v_mul_hi_u32 v2, v0, 24
 ; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, 24
 ; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, 24
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, s7
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
-; GCN-IR-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IR-NEXT:    s_mov_b32 s6, -1
-; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IR-NEXT:    s_mov_b32 s2, -1
+; GCN-IR-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GCN-IR-NEXT:    s_endpgm
   %result = urem i64 %x, 24
   store i64 %result, i64 addrspace(1)* %out
   ret void
 }
 
 ; FIXME: Constant bus violation
 ; define i64 @v_test_urem_k_num_i64(i64 %x) {

llvm/test/CodeGen/AMDGPU/vector-alloca-bitcast.ll

 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-ALLOCA %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-PROMOTE %s
 ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-promote-alloca -sroa -instcombine < %s | FileCheck -check-prefix=OPT %s
 
 target datalayout = "A5"
 
 ; OPT-LABEL: @vector_read_alloca_bitcast(
 ; OPT-NOT:   alloca
 ; OPT:       %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
 ; OPT-NEXT:  store i32 %0, i32 addrspace(1)* %out, align 4
 
 ; GCN-LABEL: {{^}}vector_read_alloca_bitcast:
 ; GCN-ALLOCA-COUNT-4: buffer_store_dword
 ; GCN-ALLOCA:         buffer_load_dword
 
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cmp_eq_u32_e64 [[CC1:[^,]+]], s{{[0-9]+}}, 1
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC2:[^,]+]], s{{[0-9]+}}, 2
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], [[CC2]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC3:[^,]+]], s{{[0-9]+}}, 3
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], [[CC3]]
-
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
 ; GCN-PROMOTE: ScratchSize: 0
 
 define amdgpu_kernel void @vector_read_alloca_bitcast(i32 addrspace(1)* %out, i32 %index) {
 entry:
   %tmp = alloca [4 x i32], addrspace(5)
   %x = bitcast [4 x i32] addrspace(5)* %tmp to i32 addrspace(5)*
   %y = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
   %z = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2

 ; OPT:       %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
 ; OPT-NEXT:  %add2 = add nuw nsw i32 %0, 1
 ; OPT-NEXT:  store i32 %add2, i32 addrspace(1)* %out, align 4
 
 ; GCN-LABEL: {{^}}vector_read_alloca_multiuse:
 ; GCN-ALLOCA-COUNT-4: buffer_store_dword
 ; GCN-ALLOCA:         buffer_load_dword
 
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2
 ; GCN-PROMOTE: v_cmp_eq_u32_e64 [[CC1:[^,]+]], s{{[0-9]+}}, 1
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
 ; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND1:v[0-9]+]], 0, 1, [[CC1]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC2:[^,]+]], s{{[0-9]+}}, 2
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], [[CC2]]
-; GCN-PROMOTE: v_cmp_ne_u32_e64 [[CC3:[^,]+]], s{{[0-9]+}}, 3
-; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], [[CC3]]
+; GCN_PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND2:v[0-9]+]], 2, [[IND1]], vcc
+; GCN-PROMOTE: s_cselect_b64 vcc, -1, 0
+; GCN-PROMOTE: v_cndmask_b32_e{{32|64}} [[IND3:v[0-9]+]], 3, [[IND2]], vcc
 
 ; GCN-PROMOTE: ScratchSize: 0
 
 define amdgpu_kernel void @vector_read_alloca_multiuse(i32 addrspace(1)* %out, i32 %index) {
 entry:
   %tmp = alloca [4 x i32], addrspace(5)
   %b = bitcast [4 x i32] addrspace(5)* %tmp to float addrspace(5)*
   %x = bitcast float addrspace(5)* %b to i32 addrspace(5)*

llvm/test/CodeGen/AMDGPU/vselect.ll

 ;RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck --check-prefix=SI --check-prefix=VI --check-prefix=FUNC %s
 ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG --check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}test_select_v2i32:
 
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Z
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Y
 
-; SI: v_cmp_gt_i32_e32 vcc
-; SI: v_cndmask_b32_e32
-; SI: v_cmp_gt_i32_e32 vcc
-; SI: v_cndmask_b32_e32
+; SI: s_cmp_gt_i32
+; SI: s_cselect_b32
+; SI: s_cmp_gt_i32
+; SI: s_cselect_b32
 
 define amdgpu_kernel void @test_select_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in0, <2 x i32> addrspace(1)* %in1, <2 x i32> %val) {
 entry:
   %load0 = load <2 x i32>, <2 x i32> addrspace(1)* %in0
   %load1 = load <2 x i32>, <2 x i32> addrspace(1)* %in1
   %cmp = icmp sgt <2 x i32> %load0, %load1
   %result = select <2 x i1> %cmp, <2 x i32> %val, <2 x i32> %load0
   store <2 x i32> %result, <2 x i32> addrspace(1)* %out

 
 ;FUNC-LABEL: {{^}}test_select_v4i32:
 
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[4].X
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].W
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Z
 ; EG-DAG: CNDE_INT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW], T[0-9]+\.[XYZW]}}, KC0[3].Y
 
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
-; SI: v_cndmask_b32_e32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
 
 define amdgpu_kernel void @test_select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in0, <4 x i32> addrspace(1)* %in1, <4 x i32> %val) {
 entry:
   %load0 = load <4 x i32>, <4 x i32> addrspace(1)* %in0
   %load1 = load <4 x i32>, <4 x i32> addrspace(1)* %in1
   %cmp = icmp sgt <4 x i32> %load0, %load1
   %result = select <4 x i1> %cmp, <4 x i32> %val, <4 x i32> %load0
   store <4 x i32> %result, <4 x i32> addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/wave32.ll

 ; GFX1032: v_sub_co_ci_u32_e32 v{{[0-9]+}}, vcc_lo, {{[vs][0-9]+}}, v{{[0-9]+}}, vcc_lo
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, [[SDST:s\[[0-9:]+\]]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_ci_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc{{$}}
 ; GFX1064: v_add_co_ci_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}, [[SDST]]
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_u32_e64 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1064: v_add_co_ci_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc{{$}}
-; GFX1064: v_sub_co_u32_e64 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX1064: v_subrev_co_ci_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], {{[vs][0-9]+}}, v{{[0-9]+}}, vcc{{$}}
-; GFX1064: v_sub_co_ci_u32_e32 v{{[0-9]+}}, vcc, {{[vs][0-9]+}}, v{{[0-9]+}}, vcc{{$}}
+; GFX1064: v_sub_co_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], s{{[0-9]+}}, v{{[0-9]+}}
+; GFX1064: v_subrev_co_ci_u32_e64 v{{[0-9]+}}, vcc, {{[vs][0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}]
+; GFX1064: v_sub_co_ci_u32_e64 v{{[0-9]+}}, s[{{[0-9:]+}}], {{[vs][0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}]
 define amdgpu_kernel void @test_udiv64(i64 addrspace(1)* %arg) #0 {
 bb:
   %tmp = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 1
   %tmp1 = load i64, i64 addrspace(1)* %tmp, align 8
   %tmp2 = load i64, i64 addrspace(1)* %arg, align 8
   %tmp3 = udiv i64 %tmp1, %tmp2
   %tmp4 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 2
   store i64 %tmp3, i64 addrspace(1)* %tmp4, align 8