[AMDGPU] Select S_CSELECT_B32 for uniform extensions from i1

[foad]

Work in progress. Lots of regressions. Lots of manual tests need updating.

llvm/lib/Target/AMDGPU/SIInstructions.td

 /********** ================== **********/
 
 def : GCNPat <
   (f32 (fpow (VOP3Mods f32:$src0, i32:$src0_mods), (VOP3Mods f32:$src1, i32:$src1_mods))),
   (V_EXP_F32_e64 SRCMODS.NONE, (V_MUL_LEGACY_F32_e64 $src1_mods, $src1, SRCMODS.NONE, (V_LOG_F32_e64 $src0_mods, $src0), 0, 0))
 >;
 
 def : GCNPat <
+  (i32 (UniformUnaryFrag<sext> SCC)),
+  (S_CSELECT_B32 (i32 0), (i32 -1))
+>;
+
+def : GCNPat <
   (i32 (sext i1:$src0)),
   (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                      /*src1mod*/(i32 0), /*src1*/(i32 -1), $src0)
 >;
 
-class Ext32Pat <SDNode ext> : GCNPat <
+multiclass Ext32Pat <SDNode ext> {
+  def : GCNPat <
+    (i32 (UniformUnaryFrag<ext> SCC)),
+    (S_CSELECT_B32 (i32 0), (i32 1))
+  >;
+  def : GCNPat <
     (i32 (ext i1:$src0)),
     (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                        /*src1mod*/(i32 0), /*src1*/(i32 1), $src0)
   >;
+}
 
-def : Ext32Pat <zext>;
-def : Ext32Pat <anyext>;
+defm : Ext32Pat <zext>;
+defm : Ext32Pat <anyext>;
 
 // The multiplication scales from [0,1) to the unsigned integer range,
 // rounding down a bit to avoid unwanted overflow.
 def : GCNPat <
   (AMDGPUurecip i32:$src0),
   (V_CVT_U32_F32_e32
     (V_MUL_F32_e32 (i32 CONST.FP_4294966784),
                    (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0))))

llvm/lib/Target/AMDGPU/VOP2Instructions.td

 def : GCNPat<
   (i64 (zext (op i16:$src0, i16:$src1))),
    (REG_SEQUENCE VReg_64,
      (inst $src0, $src1), sub0,
      (V_MOV_B32_e32 (i32 0)), sub1)
 >;
 }
 
-class ZExt_i16_i1_Pat <SDNode ext> : GCNPat <
+multiclass ZExt_i16_i1_Pat <SDNode ext> {
+  def : GCNPat <
+    (i16 (UniformUnaryFrag<ext> SCC)),
+    (S_CSELECT_B32 (i32 0), (i32 1))
+  >;
+  def : GCNPat <
     (i16 (ext i1:$src)),
     (V_CNDMASK_B32_e64 (i32 0/*src0mod*/), (i32 0/*src0*/),
                        (i32 0/*src1mod*/), (i32 1/*src1*/),
                        $src)
   >;
+}
 
 foreach vt = [i16, v2i16] in {
 def : GCNPat <
   (and vt:$src0, vt:$src1),
   (V_AND_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
 >;
 
 def : GCNPat <

 defm : Arithmetic_i16_0Hi_Pats<clshl_rev_16, V_LSHLREV_B16_e64>;
 defm : Arithmetic_i16_0Hi_Pats<clshr_rev_16, V_LSHRREV_B16_e64>;
 defm : Arithmetic_i16_0Hi_Pats<cashr_rev_16, V_ASHRREV_I16_e64>;
 
 }  // End Predicates = [Has16BitInsts, isGFX8GFX9]
 
 let Predicates = [Has16BitInsts] in {
 
-def : ZExt_i16_i1_Pat<zext>;
-def : ZExt_i16_i1_Pat<anyext>;
+defm : ZExt_i16_i1_Pat<zext>;
+defm : ZExt_i16_i1_Pat<anyext>;
+
+def : GCNPat <
+  (i16 (UniformUnaryFrag<sext> SCC)),
+  (S_CSELECT_B32 (i32 0), (i32 -1))
+>;
 
 def : GCNPat <
   (i16 (sext i1:$src)),
   (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                      /*src1mod*/(i32 0), /*src1*/(i32 -1), $src)
 >;
 
 } // End Predicates = [Has16BitInsts]

llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll

; GCN-NEXT:    ; return to shader part epilog
   ret i32 %r
 }
 
 define amdgpu_kernel void @sgpr_trunc_brcond(i32 %cond) {
 ; GCN-LABEL: sgpr_trunc_brcond:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s1, s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s0, -1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s2, 0, 1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_xor_b32 s1, s1, -1
+; GCN-NEXT:    s_xor_b32 s1, s1, s2
 ; GCN-NEXT:    s_and_b32 s1, s1, 1
 ; GCN-NEXT:    s_cmp_lg_u32 s1, 0
 ; GCN-NEXT:    s_cbranch_scc0 .LBB3_2
 ; GCN-NEXT:  ; %bb.1: ; %bb1
 ; GCN-NEXT:    v_mov_b32_e32 v0, 1
 ; GCN-NEXT:    s_mov_b32 s0, 0
 ; GCN-NEXT:    flat_store_dword v[0:1], v0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)

 }
 
 define amdgpu_kernel void @brcond_sgpr_trunc_and(i32 %cond0, i32 %cond1) {
 ; GCN-LABEL: brcond_sgpr_trunc_and:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_and_b32 s1, s0, s1
-; GCN-NEXT:    s_xor_b32 s1, s1, -1
-; GCN-NEXT:    s_and_b32 s1, s1, 1
 ; GCN-NEXT:    s_mov_b32 s0, -1
+; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s2, 0, 1
+; GCN-NEXT:    s_xor_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s1, s1, 1
 ; GCN-NEXT:    s_cmp_lg_u32 s1, 0
 ; GCN-NEXT:    s_cbranch_scc0 .LBB4_2
 ; GCN-NEXT:  ; %bb.1: ; %bb1
 ; GCN-NEXT:    v_mov_b32_e32 v0, 1
 ; GCN-NEXT:    s_mov_b32 s0, 0
 ; GCN-NEXT:    flat_store_dword v[0:1], v0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB4_2: ; %Flow

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir

 body: |
   bb.0:
     liveins: $sgpr0
 
     ; GCN-LABEL: name: anyext_regclass_sgpr_s1_to_sgpr_s32
     ; GCN: liveins: $sgpr0
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN-NEXT: $sgpr0 = COPY [[COPY]]
+    ; GCN-NEXT: $scc = COPY [[COPY]]
+    ; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GCN-NEXT: $sgpr0 = COPY [[S_CSELECT_B32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sreg_32(s1) = G_TRUNC %0
     %2:sgpr(s32) = G_ANYEXT %1
     $sgpr0 = COPY %2
 ...

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sext.mir

 body: |
   bb.0:
     liveins: $sgpr0
 
     ; GCN-LABEL: name: sext_sgpr_reg_class_s1_to_sgpr_s32
     ; GCN: liveins: $sgpr0
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN-NEXT: [[S_BFE_I32_:%[0-9]+]]:sreg_32 = S_BFE_I32 [[COPY]], 65536, implicit-def $scc
-    ; GCN-NEXT: $sgpr0 = COPY [[S_BFE_I32_]]
+    ; GCN-NEXT: $scc = COPY [[COPY]]
+    ; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, -1, implicit $scc
+    ; GCN-NEXT: $sgpr0 = COPY [[S_CSELECT_B32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sreg_32(s1) = G_TRUNC %0
     %2:sgpr(s32) = G_SEXT %1
     $sgpr0 = COPY %2
 ...

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-zext.mir

 body: |
   bb.0:
     liveins: $sgpr0
 
     ; GCN-LABEL: name: zext_sgpr_reg_class_s1_to_sgpr_s32
     ; GCN: liveins: $sgpr0
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], 1, implicit-def $scc
-    ; GCN-NEXT: $sgpr0 = COPY [[S_AND_B32_]]
+    ; GCN-NEXT: $scc = COPY [[COPY]]
+    ; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GCN-NEXT: $sgpr0 = COPY [[S_CSELECT_B32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sreg_32(s1) = G_TRUNC %0
     %2:sgpr(s32) = G_ZEXT %1
     $sgpr0 = COPY %2
 ...

llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
 
 ; Test the localizer did something and we don't materialize all
 ; constants in SGPRs in the entry block.
 
 define amdgpu_kernel void @localize_constants(i1 %cond) {
 ; GFX9-LABEL: localize_constants:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x0
 ; GFX9-NEXT:    s_mov_b32 s0, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_xor_b32 s1, s1, -1
+; GFX9-NEXT:    s_xor_b32 s1, s1, s2
 ; GFX9-NEXT:    s_and_b32 s1, s1, 1
 ; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %bb1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x5be6
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x1c7

 @gv2 = addrspace(1) global i32 undef, align 4
 @gv3 = addrspace(1) global i32 undef, align 4
 
 define amdgpu_kernel void @localize_globals(i1 %cond) {
 ; GFX9-LABEL: localize_globals:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x0
 ; GFX9-NEXT:    s_mov_b32 s0, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_xor_b32 s1, s1, -1
+; GFX9-NEXT:    s_xor_b32 s1, s1, s2
 ; GFX9-NEXT:    s_and_b32 s1, s1, 1
 ; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX9-NEXT:  ; %bb.1: ; %bb1
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, gv2@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, gv2@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0

llvm/test/CodeGen/AMDGPU/anyext.ll

 ; GCN-LABEL: anyext_i1_i32:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_lg_u32 s4, 0
-; GCN-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT:    s_cselect_b32 s4, 0, 1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: anyext_i1_i32:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX8-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s2, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
-; GFX8-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX8-NEXT:    v_not_b32_e32 v0, v0
-; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX8-NEXT:    s_andn2_b32 s4, 1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: anyext_i1_i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
-; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX9-NEXT:    v_not_b32_e32 v0, v0
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9-NEXT:    s_andn2_b32 s0, 1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 entry:
   %tmp = icmp eq i32 %cond, 0
   %tmp1 = zext i1 %tmp to i8
   %tmp2 = xor i8 %tmp1, -1
   %tmp3 = and i8 %tmp2, 1
   %tmp4 = zext i8 %tmp3 to i32

 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 0x8000, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GCN-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GCN-NEXT:    s_cselect_b32 s0, 0, 1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: anyext_v2i16_to_v2i32:
 ; GFX8:       ; %bb.0: ; %bb
 ; GFX8-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s2, -1
 ; GFX8-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0x8000
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX8-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: anyext_v2i16_to_v2i32:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    global_load_short_d16_hi v0, v[0:1], off
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0x80008000, v0
 ; GFX9-NEXT:    v_bfi_b32 v0, v1, 0, v0
 ; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 bb:
   %tmp = load i16, i16 addrspace(1)* undef, align 2
   %tmp2 = insertelement <2 x i16> undef, i16 %tmp, i32 1
   %tmp4 = and <2 x i16> %tmp2, <i16 -32768, i16 -32768>
   %tmp5 = zext <2 x i16> %tmp4 to <2 x i32>
   %tmp6 = shl nuw <2 x i32> %tmp5, <i32 16, i32 16>

llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll

 ; GFX7-NEXT:  .LBB0_3:
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX7-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX7-NEXT:  .LBB0_4: ; %Flow
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX7-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX7-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX7-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX7-NEXT:  ; %bb.5: ; %if
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX89-LABEL: add_i32_constant:
 ; GFX89:       ; %bb.0: ; %entry
 ; GFX89-NEXT:    s_mov_b64 s[10:11], exec

 ; GFX89-NEXT:  .LBB0_3:
 ; GFX89-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX89-NEXT:    s_waitcnt vmcnt(0)
 ; GFX89-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX89-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX89-NEXT:  .LBB0_4: ; %Flow
 ; GFX89-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX89-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX89-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX89-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX89-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX89-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX89-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX89-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX89-NEXT:  ; %bb.5: ; %if
 ; GFX89-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX89-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX89-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: add_i32_constant:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], exec

 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1064-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX1064-NEXT:  .LBB0_4: ; %Flow
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1064-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX1064-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX1064-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX1064-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX1064-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1064-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1064-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX1064-NEXT:  ; %bb.5: ; %if
 ; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX1064-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i32_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo

 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s9
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1032-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX1032-NEXT:  .LBB0_4: ; %Flow
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s8
 ; GFX1032-NEXT:    s_wqm_b32 s4, -1
-; GFX1032-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s4
-; GFX1032-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX1032-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX1032-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1032-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1032-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX1032-NEXT:  ; %bb.5: ; %if
 ; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX1032-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: add_i32_constant:
 ; GFX1164:       ; %bb.0: ; %entry
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], exec

 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX1164-NEXT:  .LBB0_4: ; %Flow
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1164-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT:    s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1164-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX1164-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1164-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1164-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX1164-NEXT:  ; %bb.5: ; %if
 ; GFX1164-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX1164-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX1164-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: add_i32_constant:
 ; GFX1132:       ; %bb.0: ; %entry

 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s9
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
 ; GFX1132-NEXT:  .LBB0_4: ; %Flow
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s8
 ; GFX1132-NEXT:    s_wqm_b32 s4, -1
-; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1132-NEXT:    s_cbranch_vccnz .LBB0_6
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX1132-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1132-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1132-NEXT:    s_cbranch_scc1 .LBB0_6
 ; GFX1132-NEXT:  ; %bb.5: ; %if
 ; GFX1132-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX1132-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GFX1132-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %cond1 = call i1 @llvm.amdgcn.wqm.vote(i1 true)
   %old = call i32 @llvm.amdgcn.raw.buffer.atomic.add(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0)
   %cond2 = call i1 @llvm.amdgcn.wqm.vote(i1 true)
   %cond = and i1 %cond1, %cond2
   br i1 %cond, label %if, label %else
 if:
   %bitcast = bitcast i32 %old to float
   call void @llvm.amdgcn.raw.buffer.store.f32(float %bitcast, <4 x i32> %out, i32 0, i32 0, i32 0)
   ret void
 else:
   ret void
 }
 
 define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %inout, i32 %val) {
 ; GFX7-LABEL: add_i32_varying:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_wqm_b64 s[8:9], -1
 ; GFX7-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc
-; GFX7-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GFX7-NEXT:    s_cbranch_vccnz .LBB1_2
+; GFX7-NEXT:    s_and_b64 s[4:5], s[8:9], exec
+; GFX7-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX7-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX7-NEXT:    s_cbranch_scc1 .LBB1_2
 ; GFX7-NEXT:  ; %bb.1: ; %if
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:  .LBB1_2: ; %else
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: add_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry

 ; GFX8-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v0
 ; GFX8-NEXT:  .LBB1_4: ; %Flow
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX8-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX8-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX8-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX8-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX8-NEXT:  ; %bb.5: ; %if
 ; GFX8-NEXT:    buffer_store_dword v3, off, s[0:3], 0
 ; GFX8-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: add_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_mov_b64 s[8:9], exec

 ; GFX9-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX9-NEXT:    v_add_u32_e32 v3, s4, v0
 ; GFX9-NEXT:  .LBB1_4: ; %Flow
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX9-NEXT:  ; %bb.5: ; %if
 ; GFX9-NEXT:    buffer_store_dword v3, off, s[0:3], 0
 ; GFX9-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: add_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_mov_b64 s[8:9], exec

 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1064-NEXT:    v_add_nc_u32_e32 v4, s4, v0
 ; GFX1064-NEXT:  .LBB1_4: ; %Flow
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1064-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX1064-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX1064-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX1064-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX1064-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1064-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1064-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX1064-NEXT:  ; %bb.5: ; %if
 ; GFX1064-NEXT:    buffer_store_dword v4, off, s[0:3], 0
 ; GFX1064-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_mov_b32 s8, exec_lo

 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s9
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1032-NEXT:    v_add_nc_u32_e32 v4, s4, v0
 ; GFX1032-NEXT:  .LBB1_4: ; %Flow
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s8
 ; GFX1032-NEXT:    s_wqm_b32 s4, -1
-; GFX1032-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s4
-; GFX1032-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX1032-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX1032-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1032-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1032-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX1032-NEXT:  ; %bb.5: ; %if
 ; GFX1032-NEXT:    buffer_store_dword v4, off, s[0:3], 0
 ; GFX1032-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: add_i32_varying:
 ; GFX1164:       ; %bb.0: ; %entry
 ; GFX1164-NEXT:    s_mov_b64 s[8:9], exec

 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_add_nc_u32_e32 v4, s4, v0
 ; GFX1164-NEXT:  .LBB1_4: ; %Flow
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX1164-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT:    s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1164-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX1164-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1164-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1164-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX1164-NEXT:  ; %bb.5: ; %if
 ; GFX1164-NEXT:    buffer_store_b32 v4, off, s[0:3], 0
 ; GFX1164-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX1164-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: add_i32_varying:
 ; GFX1132:       ; %bb.0: ; %entry

 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_add_nc_u32_e32 v4, s4, v0
 ; GFX1132-NEXT:  .LBB1_4: ; %Flow
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s8
 ; GFX1132-NEXT:    s_wqm_b32 s4, -1
-; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1132-NEXT:    s_cbranch_vccnz .LBB1_6
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX1132-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX1132-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX1132-NEXT:    s_cbranch_scc1 .LBB1_6
 ; GFX1132-NEXT:  ; %bb.5: ; %if
 ; GFX1132-NEXT:    buffer_store_b32 v4, off, s[0:3], 0
 ; GFX1132-NEXT:  .LBB1_6: ; %UnifiedReturnBlock
 ; GFX1132-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %cond1 = call i1 @llvm.amdgcn.wqm.vote(i1 true)
   %old = call i32 @llvm.amdgcn.raw.buffer.atomic.add(i32 %val, <4 x i32> %inout, i32 0, i32 0, i32 0)

llvm/test/CodeGen/AMDGPU/ctpop16.ll

 define amdgpu_kernel void @ctpop_i16_in_br(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %ctpop_arg, i16 %cond) {
 ; SI-LABEL: ctpop_i16_in_br:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dword s4, s[0:1], 0xd
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_lshr_b32 s5, s4, 16
 ; SI-NEXT:    s_cmp_lg_u32 s5, 0
-; SI-NEXT:    s_cbranch_scc0 .LBB14_4
+; SI-NEXT:    s_cbranch_scc0 .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    s_mov_b32 s11, 0xf000
 ; SI-NEXT:    s_mov_b32 s10, -1
 ; SI-NEXT:    s_mov_b32 s8, s2
 ; SI-NEXT:    s_mov_b32 s9, s3
 ; SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0 offset:2
 ; SI-NEXT:    s_mov_b64 s[2:3], 0
-; SI-NEXT:    s_cbranch_execnz .LBB14_3
-; SI-NEXT:  .LBB14_2: ; %if
+; SI-NEXT:    s_branch .LBB14_3
+; SI-NEXT:  .LBB14_2:
+; SI-NEXT:    s_mov_b64 s[2:3], -1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:  .LBB14_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_cselect_b32 s2, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s2, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB14_5
+; SI-NEXT:  ; %bb.4: ; %if
 ; SI-NEXT:    s_and_b32 s2, s4, 0xffff
 ; SI-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:  .LBB14_3: ; %endif
+; SI-NEXT:  .LBB14_5: ; %endif
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
-; SI-NEXT:  .LBB14_4:
-; SI-NEXT:    s_mov_b64 s[2:3], -1
-; SI-NEXT:    v_mov_b32_e32 v0, 0
-; SI-NEXT:    s_branch .LBB14_2
 ;
 ; VI-LABEL: ctpop_i16_in_br:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dword s4, s[0:1], 0x34
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s5, s4, 16
 ; VI-NEXT:    v_cmp_ne_u16_e64 s[6:7], s5, 0
 ; VI-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; VI-NEXT:    s_cbranch_vccz .LBB14_4
+; VI-NEXT:    s_cbranch_vccz .LBB14_2
 ; VI-NEXT:  ; %bb.1: ; %else
 ; VI-NEXT:    s_mov_b32 s11, 0xf000
 ; VI-NEXT:    s_mov_b32 s10, -1
 ; VI-NEXT:    s_mov_b32 s8, s2
 ; VI-NEXT:    s_mov_b32 s9, s3
 ; VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0 offset:2
 ; VI-NEXT:    s_mov_b64 s[2:3], 0
-; VI-NEXT:    s_cbranch_execnz .LBB14_3
-; VI-NEXT:  .LBB14_2: ; %if
+; VI-NEXT:    s_branch .LBB14_3
+; VI-NEXT:  .LBB14_2:
+; VI-NEXT:    s_mov_b64 s[2:3], -1
+; VI-NEXT:    ; implicit-def: $vgpr0
+; VI-NEXT:  .LBB14_3: ; %Flow
+; VI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; VI-NEXT:    s_cselect_b32 s2, 0, 1
+; VI-NEXT:    s_cmp_lg_u32 s2, 1
+; VI-NEXT:    s_cbranch_scc1 .LBB14_5
+; VI-NEXT:  ; %bb.4: ; %if
 ; VI-NEXT:    s_and_b32 s2, s4, 0xffff
 ; VI-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
-; VI-NEXT:  .LBB14_3: ; %endif
+; VI-NEXT:  .LBB14_5: ; %endif
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
-; VI-NEXT:  .LBB14_4:
-; VI-NEXT:    s_mov_b64 s[2:3], -1
-; VI-NEXT:    ; implicit-def: $vgpr0
-; VI-NEXT:    s_branch .LBB14_2
 ;
 ; EG-LABEL: ctpop_i16_in_br:
 ; EG:       ; %bb.0: ; %entry
 ; EG-NEXT:    ALU 0, @20, KC0[], KC1[]
 ; EG-NEXT:    TEX 0 @14
 ; EG-NEXT:    ALU_PUSH_BEFORE 6, @21, KC0[], KC1[]
 ; EG-NEXT:    JUMP @7 POP:1
 ; EG-NEXT:    ALU 0, @28, KC0[CB0:0-32], KC1[]

llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll

define void @main(float %arg) {
   ; CHECK-NEXT:   $vcc_lo = COPY [[S_AND_B32_]]
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.2, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.bb2:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
-  ; CHECK-NEXT:   %20:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   %1:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY]], 0, %20, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   %2:vgpr_32 = contract reassoc nofpexcept V_ADD_F32_e64 0, %1, 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_FMAC_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_FMAC_F32_e64_1:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY]], 0, [[V_FMAC_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_ADD_F32_e64 0, [[V_FMAC_F32_e64_1]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.bb11:
   ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %1, %bb.1
-  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %2, %bb.1
-  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, [[S_MOV_B32_2]], %bb.1
-  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[PHI2]], implicit $exec
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_FMAC_F32_e64_1]], %bb.1
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_ADD_F32_e64_]], %bb.1
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:sreg_32 = PHI [[S_MOV_B32_1]], %bb.0, [[S_MOV_B32_2]], %bb.1
+  ; CHECK-NEXT:   $scc = COPY [[PHI2]]
+  ; CHECK-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
   ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 1
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY [[V_CNDMASK_B32_e64_]]
-  ; CHECK-NEXT:   S_CMP_LG_U32 killed [[COPY1]], killed [[S_MOV_B32_3]], implicit-def $scc
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
-  ; CHECK-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 $exec_lo, killed [[COPY2]], implicit-def dead $scc
+  ; CHECK-NEXT:   S_CMP_LG_U32 killed [[S_CSELECT_B32_]], killed [[S_MOV_B32_3]], implicit-def $scc
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+  ; CHECK-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 $exec_lo, killed [[COPY1]], implicit-def dead $scc
   ; CHECK-NEXT:   $vcc_lo = COPY [[S_AND_B32_1]]
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.4, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.bb15:
   ; CHECK-NEXT:   successors: %bb.4(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: {{  $}}

llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll

define amdgpu_kernel void @uniform_opt_lshr_and_cmp(i1 addrspace(1)* %out, i32 %x) {
   ; GCN-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset.cast, align 4, addrspace 4)
   ; GCN-NEXT:   [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 11, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset.cast, addrspace 4)
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]]
   ; GCN-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 2
   ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORD_IMM]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; GCN-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 2, [[S_LOAD_DWORD_IMM]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_LG_U32 killed [[S_AND_B32_1]], 0, implicit-def $scc
   ; GCN-NEXT:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64_xexec = COPY [[COPY2]]
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY [[COPY2]]
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], killed [[S_MOV_B32_1]], implicit-def $scc
   ; GCN-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit $scc
   ; GCN-NEXT:   S_BRANCH %bb.1
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.1.out.true:
   ; GCN-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; GCN-NEXT:   [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[COPY3]], killed [[S_MOV_B64_]], implicit-def dead $scc
+  ; GCN-NEXT:   [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[COPY3]], killed [[S_MOV_B64_]], implicit-def dead $scc
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub1
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub0
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
   ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1, killed [[S_MOV_B32_3]], %subreg.sub2, killed [[S_MOV_B32_2]], %subreg.sub3
-  ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_XOR_B64_]], implicit $exec
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   $scc = COPY [[S_XOR_B64_]]
+  ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[COPY6]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.2.out.else:
-  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub1
-  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub0
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub1
+  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub0
   ; GCN-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
   ; GCN-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[S_MOV_B32_5]], %subreg.sub2, killed [[S_MOV_B32_4]], %subreg.sub3
-  ; GCN-NEXT:   [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[COPY3]], implicit $exec
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_1]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[S_MOV_B32_5]], %subreg.sub2, killed [[S_MOV_B32_4]], %subreg.sub3
+  ; GCN-NEXT:   $scc = COPY [[COPY3]]
+  ; GCN-NEXT:   [[S_CSELECT_B32_1:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+  ; GCN-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_1]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[COPY9]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
 entry:
   %0 = and i32 %x, 2
   %1 = icmp ne i32 %0, 0
   ; Don't optimize the truncate in the SDag away.
   br i1 %1, label %out.true, label %out.else
 
 out.true:
   %2 = xor i1 %1, -1
   store i1 %2, i1 addrspace(1)* %out
   ret void
 
 out.else:
   store i1 %1, i1 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll

define amdgpu_kernel void @uniform_trunc_i16_to_i1(i1 addrspace(1)* %out, i16 %x, i1 %z) {
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
   ; GCN-NEXT:   [[S_SEXT_I32_I16_:%[0-9]+]]:sreg_32 = S_SEXT_I32_I16 [[S_LOAD_DWORD_IMM]]
   ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 65536, [[S_LOAD_DWORD_IMM]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_LG_U32 killed [[S_AND_B32_]], 0, implicit-def $scc
   ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   S_CMP_LT_I32 killed [[S_SEXT_I32_I16_]], killed [[S_MOV_B32_2]], implicit-def $scc
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY4]], killed [[COPY3]], implicit-def dead $scc
-  ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 killed [[COPY4]], killed [[COPY3]], implicit-def dead $scc
+  ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
+  ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[COPY5]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i16 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
   store i1 %select, i1 addrspace(1)* %out
   ret void
 }
 
 define i1 @divergent_trunc_i16_to_i1(i1 addrspace(1)* %out, i16 %x, i1 %z) {
define amdgpu_kernel void @uniform_trunc_i32_to_i1(i1 addrspace(1)* %out, i32 %x, i1 %z) {
   ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
   ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY4]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   S_CMP_LT_I32 killed [[COPY3]], killed [[S_MOV_B32_2]], implicit-def $scc
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY6]], killed [[COPY5]], implicit-def dead $scc
-  ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 killed [[COPY6]], killed [[COPY5]], implicit-def dead $scc
+  ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
+  ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[COPY7]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i32 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
   store i1 %select, i1 addrspace(1)* %out
   ret void
 }
 
 define i1 @divergent_trunc_i32_to_i1(i1 addrspace(1)* %out, i32 %x, i1 %z) {
define amdgpu_kernel void @uniform_trunc_i64_to_i1(i1 addrspace(1)* %out, i64 %x, i1 %z) {
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
   ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
   ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[S_LOAD_DWORD_IMM]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
   ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY killed [[S_MOV_B64_]]
   ; GCN-NEXT:   [[V_CMP_LT_I64_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_I64_e64 killed [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[V_CMP_LT_I64_e64_]], killed [[COPY7]], implicit-def dead $scc
-  ; GCN-NEXT:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.3, addrspace 1)
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 killed [[V_CMP_LT_I64_e64_]], killed [[COPY7]], implicit-def dead $scc
+  ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
+  ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+  ; GCN-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET killed [[COPY9]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.3, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i64 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
   store i1 %select, i1 addrspace(1)* %out
   ret void
 }
 
 define i1 @divergent_trunc_i64_to_i1(i1 addrspace(1)* %out, i64 %x, i1 %z) {

llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck %s
 
 define amdgpu_kernel void @cannot_create_empty_or_backwards_segment(i1 %arg, i1 %arg1, i1 %arg2, i1 %arg3, i1 %arg4, i1 %arg5) {
 ; CHECK-LABEL: cannot_create_empty_or_backwards_segment:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_mov_b64 s[26:27], s[2:3]
 ; CHECK-NEXT:    s_mov_b64 s[24:25], s[0:1]
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; CHECK-NEXT:    s_add_u32 s24, s24, s7
 ; CHECK-NEXT:    s_addc_u32 s25, s25, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 0
-; CHECK-NEXT:    s_cselect_b64 s[14:15], -1, 0
+; CHECK-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 8
 ; CHECK-NEXT:    s_cselect_b64 s[8:9], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 16
-; CHECK-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT:    s_cselect_b64 s[10:11], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 24
 ; CHECK-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
 ; CHECK-NEXT:    s_xor_b64 s[2:3], s[6:7], -1
 ; CHECK-NEXT:    s_bitcmp1_b32 s1, 0
-; CHECK-NEXT:    s_cselect_b64 s[10:11], -1, 0
-; CHECK-NEXT:    s_bitcmp1_b32 s1, 8
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[14:15]
 ; CHECK-NEXT:    s_cselect_b64 s[12:13], -1, 0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
-; CHECK-NEXT:    s_and_b64 s[2:3], exec, s[2:3]
-; CHECK-NEXT:    s_and_b64 s[4:5], exec, s[8:9]
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_bitcmp1_b32 s1, 8
+; CHECK-NEXT:    s_cselect_b64 s[14:15], -1, 0
+; CHECK-NEXT:    s_and_b64 s[0:1], exec, s[2:3]
+; CHECK-NEXT:    s_and_b64 s[2:3], exec, s[8:9]
 ; CHECK-NEXT:    s_branch .LBB0_3
 ; CHECK-NEXT:  .LBB0_1: ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[18:19], 0
 ; CHECK-NEXT:    s_mov_b64 s[20:21], -1
 ; CHECK-NEXT:    s_mov_b64 s[16:17], -1
 ; CHECK-NEXT:    s_mov_b64 s[22:23], -1
 ; CHECK-NEXT:  .LBB0_2: ; %Flow7
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[22:23]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_12
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_13
 ; CHECK-NEXT:  .LBB0_3: ; %bb7
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_1
+; CHECK-NEXT:    s_and_b64 s[16:17], s[10:11], exec
+; CHECK-NEXT:    s_cselect_b32 s16, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s16, 1
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_1
 ; CHECK-NEXT:  ; %bb.4: ; %bb8
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_mov_b64 vcc, s[2:3]
+; CHECK-NEXT:    s_mov_b64 vcc, s[0:1]
 ; CHECK-NEXT:    s_cbranch_vccz .LBB0_6
 ; CHECK-NEXT:  ; %bb.5: ; %bb9
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[16:17], 0
 ; CHECK-NEXT:    s_mov_b64 s[18:19], -1
 ; CHECK-NEXT:    s_mov_b64 s[22:23], s[8:9]
-; CHECK-NEXT:    s_cbranch_execz .LBB0_7
-; CHECK-NEXT:    s_branch .LBB0_8
+; CHECK-NEXT:    s_branch .LBB0_7
 ; CHECK-NEXT:  .LBB0_6: ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[16:17], -1
 ; CHECK-NEXT:    s_mov_b64 s[18:19], 0
 ; CHECK-NEXT:    s_mov_b64 s[22:23], 0
-; CHECK-NEXT:  .LBB0_7: ; %bb10
+; CHECK-NEXT:  .LBB0_7: ; %Flow8
+; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    s_and_b64 s[20:21], s[16:17], exec
+; CHECK-NEXT:    s_cselect_b32 s20, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s20, 1
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_9
+; CHECK-NEXT:  ; %bb.8: ; %bb10
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[18:19], -1
 ; CHECK-NEXT:    s_mov_b64 s[16:17], 0
-; CHECK-NEXT:    s_mov_b64 s[22:23], s[12:13]
-; CHECK-NEXT:  .LBB0_8: ; %Flow9
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[14:15]
+; CHECK-NEXT:  .LBB0_9: ; %Flow9
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    s_and_b64 s[22:23], s[22:23], exec
+; CHECK-NEXT:    s_cselect_b32 s22, 0, 1
 ; CHECK-NEXT:    s_mov_b64 s[20:21], -1
-; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[22:23]
+; CHECK-NEXT:    s_cmp_lg_u32 s22, 1
 ; CHECK-NEXT:    s_mov_b64 s[22:23], -1
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_2
-; CHECK-NEXT:  ; %bb.9: ; %bb13
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_2
+; CHECK-NEXT:  ; %bb.10: ; %bb13
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_mov_b64 vcc, s[4:5]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_11
-; CHECK-NEXT:  ; %bb.10: ; %bb16
+; CHECK-NEXT:    s_mov_b64 vcc, s[2:3]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_12
+; CHECK-NEXT:  ; %bb.11: ; %bb16
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[16:17], 0
 ; CHECK-NEXT:    s_mov_b64 s[20:21], -1
-; CHECK-NEXT:    s_mov_b64 s[22:23], s[10:11]
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[12:13]
 ; CHECK-NEXT:    s_mov_b64 s[18:19], s[16:17]
 ; CHECK-NEXT:    s_branch .LBB0_2
-; CHECK-NEXT:  .LBB0_11: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:  .LBB0_12: ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[22:23], -1
 ; CHECK-NEXT:    s_mov_b64 s[20:21], 0
 ; CHECK-NEXT:    ; implicit-def: $sgpr16_sgpr17
 ; CHECK-NEXT:    s_mov_b64 s[18:19], s[16:17]
 ; CHECK-NEXT:    s_branch .LBB0_2
-; CHECK-NEXT:  .LBB0_12: ; %loop.exit.guard6
+; CHECK-NEXT:  .LBB0_13: ; %loop.exit.guard6
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_xor_b64 s[14:15], s[20:21], -1
+; CHECK-NEXT:    s_xor_b64 s[22:23], s[20:21], -1
 ; CHECK-NEXT:    s_mov_b64 s[20:21], -1
-; CHECK-NEXT:    s_and_b64 vcc, exec, s[14:15]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_16
-; CHECK-NEXT:  ; %bb.13: ; %bb14
-; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_15
-; CHECK-NEXT:  ; %bb.14: ; %bb15
-; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    buffer_store_dword v1, off, s[24:27], 0 offset:4
-; CHECK-NEXT:    buffer_store_dword v1, off, s[24:27], 0
-; CHECK-NEXT:  .LBB0_15: ; %Flow
+; CHECK-NEXT:    s_and_b64 vcc, exec, s[22:23]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_17
+; CHECK-NEXT:  ; %bb.14: ; %bb14
+; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    s_and_b64 s[20:21], s[4:5], exec
+; CHECK-NEXT:    s_cselect_b32 s20, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s20, 1
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_16
+; CHECK-NEXT:  ; %bb.15: ; %bb15
+; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    buffer_store_dword v0, off, s[24:27], 0 offset:4
+; CHECK-NEXT:    buffer_store_dword v0, off, s[24:27], 0
+; CHECK-NEXT:  .LBB0_16: ; %Flow
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    s_mov_b64 s[20:21], 0
-; CHECK-NEXT:  .LBB0_16: ; %Flow13
+; CHECK-NEXT:  .LBB0_17: ; %Flow13
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_3
-; CHECK-NEXT:  ; %bb.17: ; %loop.exit.guard
+; CHECK-NEXT:    s_and_b64 s[20:21], s[20:21], exec
+; CHECK-NEXT:    s_cselect_b32 s20, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s20, 1
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_3
+; CHECK-NEXT:  ; %bb.18: ; %loop.exit.guard
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[16:17]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_22
-; CHECK-NEXT:  ; %bb.18: ; %loop.exit.guard5
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_23
+; CHECK-NEXT:  ; %bb.19: ; %loop.exit.guard5
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[18:19]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_22
-; CHECK-NEXT:  ; %bb.19: ; %bb17
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_23
+; CHECK-NEXT:  ; %bb.20: ; %bb17
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_21
-; CHECK-NEXT:  ; %bb.20: ; %bb19
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 1, v0
 ; CHECK-NEXT:    s_cbranch_vccz .LBB0_22
-; CHECK-NEXT:  .LBB0_21: ; %bb21
+; CHECK-NEXT:  ; %bb.21: ; %bb19
+; CHECK-NEXT:    s_and_b64 s[0:1], s[4:5], exec
+; CHECK-NEXT:    s_cselect_b32 s0, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 1
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_23
+; CHECK-NEXT:  .LBB0_22: ; %bb21
 ; CHECK-NEXT:    s_endpgm
-; CHECK-NEXT:  .LBB0_22: ; %UnifiedUnreachableBlock
+; CHECK-NEXT:  .LBB0_23: ; %UnifiedUnreachableBlock
 bb:
   br label %bb6
 
 bb6:                                              ; preds = %bb15, %bb14, %bb
   br label %bb7
 
 bb7:                                              ; preds = %bb16, %bb6
   br i1 %arg2, label %bb8, label %bb20

llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll

bb:
   %i8 = icmp ugt i32 %i, 31
   %i9 = select i1 %i8, i32 %i1, i32 %i7
   ret i32 %i9
 }
 
 define amdgpu_kernel void @s_add_co_br_user(i32 %i) {
 ; GFX7-LABEL: s_add_co_br_user:
 ; GFX7:       ; %bb.0: ; %bb
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_add_i32 s0, s2, s2
-; GFX7-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX7-NEXT:    s_add_i32 s0, s4, s4
+; GFX7-NEXT:    s_cmp_lt_u32 s0, s4
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX7-NEXT:    s_and_b64 s[2:3], s[0:1], exec
+; GFX7-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX7-NEXT:    s_or_b32 s0, s0, s1
 ; GFX7-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX7-NEXT:    s_addc_u32 s0, s2, 0
-; GFX7-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v0
-; GFX7-NEXT:    s_cbranch_vccnz .LBB1_2
+; GFX7-NEXT:    s_addc_u32 s0, s4, 0
+; GFX7-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX7-NEXT:    s_cbranch_scc1 .LBB1_2
 ; GFX7-NEXT:  ; %bb.1: ; %bb0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 9
 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:  .LBB1_2: ; %bb1
 ; GFX7-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 10
 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_add_co_br_user:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_add_i32 s0, s2, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX9-NEXT:    s_add_i32 s0, s4, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s4
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_and_b64 s[2:3], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX9-NEXT:    s_addc_u32 s0, s2, 0
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v0
-; GFX9-NEXT:    s_cbranch_vccnz .LBB1_2
+; GFX9-NEXT:    s_addc_u32 s0, s4, 0
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX9-NEXT:    s_cbranch_scc1 .LBB1_2
 ; GFX9-NEXT:  ; %bb.1: ; %bb0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 9
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:  .LBB1_2: ; %bb1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 10
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_add_co_br_user:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_i32 s1, s0, s0
 ; GFX10-NEXT:    s_cmp_lt_u32 s1, s0
 ; GFX10-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX10-NEXT:    s_and_b32 s2, s1, exec_lo
+; GFX10-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX10-NEXT:    s_cmpk_lg_u32 s1, 0x0
 ; GFX10-NEXT:    s_addc_u32 s0, s0, 0
-; GFX10-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s0, v0
-; GFX10-NEXT:    s_cbranch_vccnz .LBB1_2
+; GFX10-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX10-NEXT:    s_cbranch_scc1 .LBB1_2
 ; GFX10-NEXT:  ; %bb.1: ; %bb0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 9
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:  .LBB1_2: ; %bb1
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 10
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_add_co_br_user:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_add_i32 s1, s0, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_cmp_lt_u32 s1, s0
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
+; GFX11-NEXT:    s_and_b32 s2, s1, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s2, 0, 1
 ; GFX11-NEXT:    s_cmpk_lg_u32 s1, 0x0
 ; GFX11-NEXT:    s_addc_u32 s0, s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s0, v0
-; GFX11-NEXT:    s_cbranch_vccnz .LBB1_2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX11-NEXT:    s_cbranch_scc1 .LBB1_2
 ; GFX11-NEXT:  ; %bb.1: ; %bb0
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 9
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:  .LBB1_2: ; %bb1
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 10

llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll

 ; SI-NEXT:    buffer_load_ushort v8, v[2:3], s[4:7], 0 addr64 offset:12 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[2:3], s[4:7], 0 addr64 offset:14 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v6, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB0_3
-; SI-NEXT:    s_branch .LBB0_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB0_3
 ; SI-NEXT:  .LBB0_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB0_3: ; %T
+; SI-NEXT:  .LBB0_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB0_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v4, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:4 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v5, v[0:1], s[4:7], 0 addr64 offset:6 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:8 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:10 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:12 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 offset:14 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:  .LBB0_4: ; %exit
+; SI-NEXT:  .LBB0_5: ; %exit
 ; SI-NEXT:    v_bfe_i32 v0, v3, 0, 16
 ; SI-NEXT:    v_bfe_i32 v1, v4, 0, 16
 ; SI-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; SI-NEXT:    v_mov_b32_e32 v3, 0xffff
 ; SI-NEXT:    v_mov_b32_e32 v4, 0x8000
 ; SI-NEXT:    v_mov_b32_e32 v5, 0xffff0000
 ; SI-NEXT:    v_bfrev_b32_e32 v6, 1
 ; SI-NEXT:    v_mov_b32_e32 v7, 0xffff8000

 ;
 ; GFX9-LABEL: vec_8xi16_extract_4xi16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_cbranch_execz .LBB0_3
-; GFX9-NEXT:    s_branch .LBB0_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB0_3
 ; GFX9-NEXT:  .LBB0_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX9-NEXT:  .LBB0_3: ; %T
+; GFX9-NEXT:  .LBB0_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB0_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:  .LBB0_4: ; %exit
+; GFX9-NEXT:  .LBB0_5: ; %exit
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v3 op_sel_hi:[0,0]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v2 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_or_b32_e32 v2, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

 ; SI-NEXT:    buffer_load_ushort v6, v[2:3], s[4:7], 0 addr64 offset:12 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[2:3], s[4:7], 0 addr64 offset:14 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v3, v6, v3
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v7
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB1_3
-; SI-NEXT:    s_branch .LBB1_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB1_3
 ; SI-NEXT:  .LBB1_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr5
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB1_3: ; %T
+; SI-NEXT:  .LBB1_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB1_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:4 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:6 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v4, v[0:1], s[4:7], 0 addr64 offset:10 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 offset:12 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:14 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v0
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v1
-; SI-NEXT:  .LBB1_4: ; %exit
+; SI-NEXT:  .LBB1_5: ; %exit
 ; SI-NEXT:    v_bfe_i32 v0, v5, 0, 16
 ; SI-NEXT:    v_bfe_i32 v1, v4, 0, 16
 ; SI-NEXT:    v_bfe_i32 v3, v3, 0, 16
 ; SI-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; SI-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; SI-NEXT:    v_mov_b32_e32 v5, 0x8000
 ; SI-NEXT:    v_mov_b32_e32 v6, 0xffff0000
 ; SI-NEXT:    v_bfrev_b32_e32 v7, 1

 ;
 ; GFX9-LABEL: vec_8xi16_extract_4xi16_2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_cbranch_execz .LBB1_3
-; GFX9-NEXT:    s_branch .LBB1_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB1_3
 ; GFX9-NEXT:  .LBB1_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX9-NEXT:  .LBB1_3: ; %T
+; GFX9-NEXT:  .LBB1_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:  .LBB1_4: ; %exit
+; GFX9-NEXT:  .LBB1_5: ; %exit
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v5 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v4 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_or_b32_e32 v3, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v5
 ; SI-NEXT:    v_or_b32_e32 v2, v6, v2
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB2_3
-; SI-NEXT:    s_branch .LBB2_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB2_3
 ; SI-NEXT:  .LBB2_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB2_3: ; %T
+; SI-NEXT:  .LBB2_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB2_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)

 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v0, v4, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT:  .LBB2_4: ; %exit
+; SI-NEXT:  .LBB2_5: ; %exit
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_mov_b32_e32 v3, 0x3fa00000
 ; SI-NEXT:    v_mov_b32_e32 v4, 0x3f200000
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    v_cmp_nge_f32_e32 vcc, 0.5, v0
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
 ; SI-NEXT:    v_cmp_nge_f32_e32 vcc, 0.5, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; SI-NEXT:    v_cmp_nge_f32_e32 vcc, 0.5, v2
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; SI-NEXT:    v_mov_b32_e32 v3, v2
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: vec_8xf16_extract_4xf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_cbranch_execz .LBB2_3
-; GFX9-NEXT:    s_branch .LBB2_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB2_3
 ; GFX9-NEXT:  .LBB2_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX9-NEXT:  .LBB2_3: ; %T
+; GFX9-NEXT:  .LBB2_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB2_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:  .LBB2_4: ; %exit
+; GFX9-NEXT:  .LBB2_5: ; %exit
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_perm_b32 v0, v3, v3, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0x3d00
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc

 ; SI-NEXT:    buffer_load_ushort v8, v[2:3], s[4:7], 0 addr64 offset:28 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[2:3], s[4:7], 0 addr64 offset:30 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v6, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB3_3
-; SI-NEXT:    s_branch .LBB3_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB3_3
 ; SI-NEXT:  .LBB3_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB3_3: ; %T
+; SI-NEXT:  .LBB3_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB3_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v4, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)

 ; SI-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:28 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 offset:30 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:  .LBB3_4: ; %exit
+; SI-NEXT:  .LBB3_5: ; %exit
 ; SI-NEXT:    v_bfe_i32 v0, v3, 0, 16
 ; SI-NEXT:    v_bfe_i32 v1, v4, 0, 16
 ; SI-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; SI-NEXT:    v_mov_b32_e32 v3, 0xffff
 ; SI-NEXT:    v_mov_b32_e32 v4, 0x8000
 ; SI-NEXT:    v_mov_b32_e32 v5, 0xffff0000
 ; SI-NEXT:    v_bfrev_b32_e32 v6, 1
 ; SI-NEXT:    v_mov_b32_e32 v7, 0xffff8000

 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB3_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr2 killed $vgpr3
-; GFX9-NEXT:    s_cbranch_execz .LBB3_3
-; GFX9-NEXT:    s_branch .LBB3_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB3_3
 ; GFX9-NEXT:  .LBB3_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
-; GFX9-NEXT:  .LBB3_3: ; %T
+; GFX9-NEXT:  .LBB3_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT:  .LBB3_4: ; %exit
+; GFX9-NEXT:  .LBB3_5: ; %exit
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v5 op_sel_hi:[0,0]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v4 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_or_b32_e32 v3, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

 ; SI-NEXT:    buffer_load_ushort v8, v[2:3], s[4:7], 0 addr64 offset:28 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[2:3], s[4:7], 0 addr64 offset:30 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v7, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v6, v3
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB4_3
-; SI-NEXT:    s_branch .LBB4_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB4_3
 ; SI-NEXT:  .LBB4_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB4_3: ; %T
+; SI-NEXT:  .LBB4_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB4_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)

 ; SI-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:28 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 offset:30 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:  .LBB4_4: ; %exit
+; SI-NEXT:  .LBB4_5: ; %exit
 ; SI-NEXT:    v_bfe_i32 v0, v3, 0, 16
 ; SI-NEXT:    v_bfe_i32 v1, v4, 0, 16
 ; SI-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; SI-NEXT:    v_bfe_i32 v3, v5, 0, 16
 ; SI-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; SI-NEXT:    v_mov_b32_e32 v5, 0x8000
 ; SI-NEXT:    v_mov_b32_e32 v6, 0xffff0000
 ; SI-NEXT:    v_bfrev_b32_e32 v7, 1

 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB4_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr2 killed $vgpr3
-; GFX9-NEXT:    s_cbranch_execz .LBB4_3
-; GFX9-NEXT:    s_branch .LBB4_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB4_3
 ; GFX9-NEXT:  .LBB4_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
-; GFX9-NEXT:  .LBB4_3: ; %T
+; GFX9-NEXT:  .LBB4_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB4_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT:  .LBB4_4: ; %exit
+; GFX9-NEXT:  .LBB4_5: ; %exit
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v7 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v6 op_sel_hi:[0,1]
 ; GFX9-NEXT:    v_or_b32_e32 v3, 0xffff8000, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v5
 ; SI-NEXT:    v_or_b32_e32 v2, v6, v2
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    s_mov_b64 vcc, exec
-; SI-NEXT:    s_cbranch_execz .LBB5_3
-; SI-NEXT:    s_branch .LBB5_4
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_branch .LBB5_3
 ; SI-NEXT:  .LBB5_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_mov_b64 vcc, 0
-; SI-NEXT:  .LBB5_3: ; %T
+; SI-NEXT:  .LBB5_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB5_5
+; SI-NEXT:  ; %bb.4: ; %T
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 offset:2 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)

 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v0, v4, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT:  .LBB5_4: ; %exit
+; SI-NEXT:  .LBB5_5: ; %exit
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_mov_b32_e32 v3, 0x3fa00000
 ; SI-NEXT:    v_mov_b32_e32 v4, 0x3f200000
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2

 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB5_2
 ; GFX9-NEXT:  ; %bb.1: ; %F
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr2 killed $vgpr3
-; GFX9-NEXT:    s_cbranch_execz .LBB5_3
-; GFX9-NEXT:    s_branch .LBB5_4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    s_branch .LBB5_3
 ; GFX9-NEXT:  .LBB5_2:
+; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
-; GFX9-NEXT:  .LBB5_3: ; %T
+; GFX9-NEXT:  .LBB5_3: ; %Flow
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 1
+; GFX9-NEXT:    s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT:  ; %bb.4: ; %T
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
-; GFX9-NEXT:  .LBB5_4: ; %exit
+; GFX9-NEXT:  .LBB5_5: ; %exit
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_perm_b32 v0, v5, v5, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3d00
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v5, v2, v3, vcc

llvm/test/CodeGen/AMDGPU/fp_to_sint.ll

; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %conv = fptosi <4 x float> %x to <4 x i64>
   store <4 x i64> %conv, <4 x i64> addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @fp_to_uint_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 {
 ; SI-LABEL: fp_to_uint_f32_to_i1:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, s4
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, s2
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_f32_to_i1:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, s4
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: fp_to_uint_f32_to_i1:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 12, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
 ; EG-NEXT:    CF_END
; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %conv = fptosi float %in to i1
   store i1 %conv, i1 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 {
 ; SI-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, |s4|
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, |s2|
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cmp_eq_f32_e64 s[4:5], -1.0, |s4|
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 12, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
 ; EG-NEXT:    CF_END

llvm/test/CodeGen/AMDGPU/fp_to_uint.ll

; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %conv = fptoui <4 x float> %x to <4 x i64>
   store <4 x i64> %conv, <4 x i64> addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @fp_to_uint_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 {
 ; SI-LABEL: fp_to_uint_f32_to_i1:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, s4
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, s2
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_f32_to_i1:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, s4
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: fp_to_uint_f32_to_i1:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 12, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
 ; EG-NEXT:    CF_END
; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %conv = fptoui float %in to i1
   store i1 %conv, i1 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 {
 ; SI-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, |s4|
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, |s2|
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cmp_eq_f32_e64 s[4:5], 1.0, |s4|
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: fp_to_uint_fabs_f32_to_i1:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    ALU 12, @4, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
 ; EG-NEXT:    CF_END

llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll

 
 define amdgpu_kernel void @llvm_amdgcn_is_shared(i8* %ptr) {
 ; GFX8V3-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V3:       ; %bb.0:
 ; GFX8V3-NEXT:    s_load_dword s0, s[4:5], 0x40
 ; GFX8V3-NEXT:    s_load_dword s1, s[6:7], 0x4
 ; GFX8V3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V3-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V3-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V3-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V3-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V3-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V3-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V3-NEXT:    s_endpgm
 ;
 ; GFX8V4-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dword s0, s[4:5], 0x40
 ; GFX8V4-NEXT:    s_load_dword s1, s[6:7], 0x4
 ; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V4-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V4-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V4-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V4-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V4-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    s_endpgm
 ;
 ; GFX8V5-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V5:       ; %bb.0:
 ; GFX8V5-NEXT:    s_load_dword s0, s[4:5], 0xcc
 ; GFX8V5-NEXT:    s_load_dword s1, s[4:5], 0x4
 ; GFX8V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V5-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V5-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V5-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V5-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V5-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V5-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V5-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V5-NEXT:    s_endpgm
 ;
 ; GFX9V3-LABEL: llvm_amdgcn_is_shared:
 ; GFX9V3:       ; %bb.0:
 ; GFX9V3-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V3-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
 ; GFX9V3-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V3-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V3-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V3-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V3-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V3-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V3-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V3-NEXT:    s_endpgm
 ;
 ; GFX9V4-LABEL: llvm_amdgcn_is_shared:
 ; GFX9V4:       ; %bb.0:
 ; GFX9V4-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V4-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
 ; GFX9V4-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V4-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V4-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V4-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V4-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V4-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V4-NEXT:    s_endpgm
 ;
 ; GFX9V5-LABEL: llvm_amdgcn_is_shared:
 ; GFX9V5:       ; %bb.0:
 ; GFX9V5-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V5-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
 ; GFX9V5-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V5-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V5-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V5-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V5-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V5-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V5-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V5-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V5-NEXT:    s_endpgm
   %is.shared = call i1 @llvm.amdgcn.is.shared(i8* %ptr)
   %zext = zext i1 %is.shared to i32
   store volatile i32 %zext, i32 addrspace(1)* undef
   ret void
 }
 
 define amdgpu_kernel void @llvm_amdgcn_is_private(i8* %ptr) {
 ; GFX8V3-LABEL: llvm_amdgcn_is_private:
 ; GFX8V3:       ; %bb.0:
 ; GFX8V3-NEXT:    s_load_dword s0, s[4:5], 0x44
 ; GFX8V3-NEXT:    s_load_dword s1, s[6:7], 0x4
 ; GFX8V3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V3-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V3-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V3-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V3-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V3-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V3-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V3-NEXT:    s_endpgm
 ;
 ; GFX8V4-LABEL: llvm_amdgcn_is_private:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dword s0, s[4:5], 0x44
 ; GFX8V4-NEXT:    s_load_dword s1, s[6:7], 0x4
 ; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V4-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V4-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V4-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V4-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V4-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    s_endpgm
 ;
 ; GFX8V5-LABEL: llvm_amdgcn_is_private:
 ; GFX8V5:       ; %bb.0:
 ; GFX8V5-NEXT:    s_load_dword s0, s[4:5], 0xc8
 ; GFX8V5-NEXT:    s_load_dword s1, s[4:5], 0x4
 ; GFX8V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V5-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V5-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX8V5-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V5-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX8V5-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8V5-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V5-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V5-NEXT:    s_endpgm
 ;
 ; GFX9V3-LABEL: llvm_amdgcn_is_private:
 ; GFX9V3:       ; %bb.0:
 ; GFX9V3-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V3-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
 ; GFX9V3-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V3-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V3-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V3-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V3-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V3-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V3-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V3-NEXT:    s_endpgm
 ;
 ; GFX9V4-LABEL: llvm_amdgcn_is_private:
 ; GFX9V4:       ; %bb.0:
 ; GFX9V4-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V4-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
 ; GFX9V4-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V4-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V4-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V4-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V4-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V4-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V4-NEXT:    s_endpgm
 ;
 ; GFX9V5-LABEL: llvm_amdgcn_is_private:
 ; GFX9V5:       ; %bb.0:
 ; GFX9V5-NEXT:    s_load_dword s0, s[4:5], 0x4
 ; GFX9V5-NEXT:    s_getreg_b32 s1, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
 ; GFX9V5-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX9V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V5-NEXT:    s_cmp_eq_u32 s0, s1
-; GFX9V5-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9V5-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V5-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9V5-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9V5-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9V5-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9V5-NEXT:    s_endpgm
   %is.private = call i1 @llvm.amdgcn.is.private(i8* %ptr)
   %zext = zext i1 %is.private to i32
   store volatile i32 %zext, i32 addrspace(1)* undef
   ret void
 }

llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll

 ; the compiler doesn't crash.
 define amdgpu_kernel void @insert_split_bb(<2 x i32> addrspace(1)* %out, i32 addrspace(1)* %in, i32 %a, i32 %b) {
 ; SI-LABEL: insert_split_bb:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dword s6, s[4:5], 0x4
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_lg_u32 s6, 0
-; SI-NEXT:    s_cbranch_scc0 .LBB30_4
+; SI-NEXT:    s_cbranch_scc0 .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    s_load_dword s7, s[2:3], 0x1
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT:    s_branch .LBB30_3
+; SI-NEXT:  .LBB30_2:
+; SI-NEXT:    s_mov_b64 s[4:5], -1
+; SI-NEXT:  .LBB30_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s4, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB30_5
+; SI-NEXT:  ; %bb.4: ; %if
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b64 vcc, vcc
-; SI-NEXT:    s_cbranch_vccnz .LBB30_3
-; SI-NEXT:  .LBB30_2: ; %if
 ; SI-NEXT:    s_load_dword s7, s[2:3], 0x0
-; SI-NEXT:  .LBB30_3: ; %endif
+; SI-NEXT:  .LBB30_5: ; %endif
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s6
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    v_mov_b32_e32 v1, s7
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
-; SI-NEXT:  .LBB30_4:
-; SI-NEXT:    s_branch .LBB30_2
 ;
 ; VI-LABEL: insert_split_bb:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dword s6, s[4:5], 0x10
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_lg_u32 s6, 0
-; VI-NEXT:    s_cbranch_scc0 .LBB30_4
+; VI-NEXT:    s_cbranch_scc0 .LBB30_2
 ; VI-NEXT:  ; %bb.1: ; %else
 ; VI-NEXT:    s_load_dword s7, s[2:3], 0x4
-; VI-NEXT:    s_cbranch_execnz .LBB30_3
-; VI-NEXT:  .LBB30_2: ; %if
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:    s_branch .LBB30_3
+; VI-NEXT:  .LBB30_2:
+; VI-NEXT:    s_mov_b64 s[4:5], -1
+; VI-NEXT:  .LBB30_3: ; %Flow
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    s_cmp_lg_u32 s4, 1
+; VI-NEXT:    s_cbranch_scc1 .LBB30_5
+; VI-NEXT:  ; %bb.4: ; %if
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s7, s[2:3], 0x0
-; VI-NEXT:  .LBB30_3: ; %endif
+; VI-NEXT:  .LBB30_5: ; %endif
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
-; VI-NEXT:  .LBB30_4:
-; VI-NEXT:    s_branch .LBB30_2
 entry:
   %0 = insertelement <2 x i32> undef, i32 %a, i32 0
   %1 = icmp eq i32 %a, 0
   br i1 %1, label %if, label %else
 
 if:
   %2 = load i32, i32 addrspace(1)* %in
   %3 = insertelement <2 x i32> %0, i32 %2, i32 1

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll

 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    s_not_b64 exec, exec
 ; GCN-NEXT:    v_mov_b32_e32 v0, 42
 ; GCN-NEXT:    s_not_b64 exec, exec
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_lg_u32 s3, 56
 ; GCN-NEXT:    s_mov_b64 s[2:3], -1
-; GCN-NEXT:    s_cbranch_scc1 .LBB2_3
-; GCN-NEXT:  ; %bb.1: ; %Flow
-; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
-; GCN-NEXT:    s_cbranch_vccz .LBB2_4
-; GCN-NEXT:  .LBB2_2: ; %.exit
-; GCN-NEXT:    s_endpgm
-; GCN-NEXT:  .LBB2_3: ; %.one
+; GCN-NEXT:    s_cbranch_scc0 .LBB2_2
+; GCN-NEXT:  ; %bb.1: ; %.one
 ; GCN-NEXT:    v_add_u32_e32 v1, vcc, 1, v0
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], 0
 ; GCN-NEXT:    s_mov_b64 s[2:3], 0
-; GCN-NEXT:    s_cbranch_execnz .LBB2_2
-; GCN-NEXT:  .LBB2_4: ; %.zero
+; GCN-NEXT:  .LBB2_2: ; %Flow
+; GCN-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; GCN-NEXT:    s_cselect_b32 s2, 0, 1
+; GCN-NEXT:    s_cmp_lg_u32 s2, 1
+; GCN-NEXT:    s_cbranch_scc1 .LBB2_4
+; GCN-NEXT:  ; %bb.3: ; %.zero
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:  .LBB2_4: ; %.exit
 ; GCN-NEXT:    s_endpgm
   %val = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 0, i32 0)
   %cmp = icmp eq i32 %val, 56
   %tmp = call i32 @llvm.amdgcn.set.inactive.i32(i32 %in, i32 42) #0
   br i1 %cmp, label %.zero, label %.one
 
 .zero:
   store i32 %tmp, i32 addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll

 ; GCN-LABEL: alloca_promote_atomicrmw_private_lds_promote:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s4, 1
-; GCN-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT:    s_cselect_b32 s4, 0, 1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 entry:
   %tmp = alloca [2 x i32], addrspace(5)
   %gep1 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 0
   %gep2 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 1
   store i32 0, i32 addrspace(5)* %gep1
   store i32 1, i32 addrspace(5)* %gep2

 ; GCN-LABEL: alloca_promote_cmpxchg_private:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NEXT:    s_mov_b32 s2, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s4, 1
-; GCN-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT:    s_cselect_b32 s4, 0, 1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 entry:
   %tmp = alloca [2 x i32], addrspace(5)
   %gep1 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 0
   %gep2 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 1
   store i32 0, i32 addrspace(5)* %gep1
   store i32 1, i32 addrspace(5)* %gep2
   %gep3 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 %in
   %xchg = cmpxchg i32 addrspace(5)* %gep3, i32 0, i32 1 acq_rel monotonic
   %val = extractvalue { i32, i1 } %xchg, 0
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/saddo.ll

 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_add_i32 s12, s8, s9
 ; SI-NEXT:    s_cmp_lt_i32 s9, 0
 ; SI-NEXT:    s_cselect_b64 s[10:11], -1, 0
 ; SI-NEXT:    s_cmp_lt_i32 s12, s8
-; SI-NEXT:    s_mov_b32 s1, s5
 ; SI-NEXT:    s_cselect_b64 s[8:9], -1, 0
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_xor_b64 s[8:9], s[10:11], s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT:    s_xor_b64 s[0:1], s[10:11], s[8:9]
+; SI-NEXT:    s_and_b64 s[0:1], s[8:9], exec
+; SI-NEXT:    s_cselect_b32 s0, 0, 1
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s7
 ; SI-NEXT:    s_mov_b32 s6, s2
 ; SI-NEXT:    s_mov_b32 s7, s3
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_saddo_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    s_add_i32 s4, s0, s1
 ; VI-NEXT:    s_cmp_lt_i32 s1, 0
 ; VI-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; VI-NEXT:    s_cmp_lt_i32 s4, s0
 ; VI-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
+; VI-NEXT:    s_cselect_b32 s0, 0, 1
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    flat_store_dword v[0:1], v4
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    flat_store_byte v[2:3], v0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_saddo_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    s_add_i32 s0, s2, s3
 ; GFX9-NEXT:    v_add_i32 v1, s2, v1 clamp
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, s0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc
 ; GFX9-NEXT:    global_store_dword v0, v2, s[4:5]
 ; GFX9-NEXT:    global_store_byte v0, v1, s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_saddo_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_nc_i32 v0, s2, s3 clamp
 ; GFX10-NEXT:    s_add_i32 s0, s2, s3
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 1, 0, vcc_lo
 ; GFX10-NEXT:    global_store_dword v1, v2, s[4:5]
 ; GFX10-NEXT:    global_store_byte v1, v0, s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_saddo_i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x34
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_add_nc_i32 v0, s4, s5 clamp
 ; GFX11-NEXT:    s_add_i32 s4, s4, s5
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s4, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 1, 0, vcc_lo
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
 ; GFX11-NEXT:    global_store_b8 v1, v0, s[2:3]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
   %val = extractvalue { i32, i1 } %sadd, 0
   %carry = extractvalue { i32, i1 } %sadd, 1

 ; SI-NEXT:    s_mov_b32 s9, s1
 ; SI-NEXT:    s_mov_b32 s4, s2
 ; SI-NEXT:    s_mov_b32 s5, s3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, v0, v1
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v0
 ; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT:    s_cselect_b32 s0, 0, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dword v2, off, s[8:11], 0
 ; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: v_saddo_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    flat_load_dword v5, v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, v4, v5
 ; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v5
 ; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v6, v4
 ; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT:    s_cselect_b32 s0, 0, 1
 ; VI-NEXT:    flat_store_dword v[0:1], v6
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    flat_store_byte v[2:3], v0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_saddo_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v1, v0, s[4:5]
 ; GFX9-NEXT:    global_load_dword v2, v0, s[6:7]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_i32 v3, v1, v2 clamp
 ; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v1, v3
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc
 ; GFX9-NEXT:    global_store_byte v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: v_saddo_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[4:5]
 ; GFX10-NEXT:    global_load_dword v2, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_nc_i32 v3, v1, v2 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v2
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 1, 0, vcc_lo
 ; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX10-NEXT:    global_store_byte v0, v2, s[2:3]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: v_saddo_i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_load_b32 v1, v0, s[4:5]
 ; GFX11-NEXT:    global_load_b32 v2, v0, s[6:7]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_nc_i32 v3, v1, v2 clamp
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, v1, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 1, 0, vcc_lo
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    global_store_b8 v0, v2, s[2:3]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %a = load i32, i32 addrspace(1)* %aptr, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
   %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind

 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_add_u32 s12, s4, s6
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    s_addc_u32 s13, s5, s7
 ; SI-NEXT:    v_mov_b32_e32 v1, s5
 ; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
 ; SI-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[6:7], 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s12
+; SI-NEXT:    s_xor_b64 s[4:5], s[4:5], vcc
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; SI-NEXT:    s_mov_b32 s8, s0
 ; SI-NEXT:    s_mov_b32 s9, s1
 ; SI-NEXT:    v_mov_b32_e32 v1, s13
-; SI-NEXT:    s_xor_b64 s[4:5], s[4:5], vcc
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
 ; SI-NEXT:    s_mov_b32 s0, s2
 ; SI-NEXT:    s_mov_b32 s1, s3
 ; SI-NEXT:    s_mov_b32 s2, s10
 ; SI-NEXT:    s_mov_b32 s3, s11
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_saddo_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    s_add_u32 s0, s4, s6
-; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    v_mov_b32_e32 v3, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s5
 ; VI-NEXT:    s_addc_u32 s1, s5, s7
-; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[3:4]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[6:7], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
-; VI-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; VI-NEXT:    s_xor_b64 s[2:3], s[4:5], vcc
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
-; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; VI-NEXT:    s_and_b64 s[0:1], s[2:3], exec
+; VI-NEXT:    s_cselect_b32 s0, 0, 1
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    flat_store_byte v[2:3], v0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_saddo_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_add_u32 s8, s4, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    s_addc_u32 s9, s5, s7
 ; GFX9-NEXT:    v_cmp_lt_i64_e64 s[10:11], s[6:7], 0
 ; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
+; GFX9-NEXT:    s_xor_b64 s[4:5], s[10:11], vcc
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT:    s_xor_b64 s[0:1], s[10:11], vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b64 s[0:1], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    global_store_byte v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_saddo_i64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s8, s4, s6
 ; GFX10-NEXT:    s_addc_u32 s9, s5, s7
-; GFX10-NEXT:    v_cmp_lt_i64_e64 s6, s[6:7], 0
-; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, s[8:9], s[4:5]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s8
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, s[8:9], s[4:5]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s5, s[6:7], 0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s9
-; GFX10-NEXT:    s_xor_b32 s4, s6, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX10-NEXT:    s_xor_b32 s4, s5, s4
+; GFX10-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX10-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    global_store_byte v2, v3, s[2:3]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_saddo_i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_add_u32 s8, s4, s6
 ; GFX11-NEXT:    s_addc_u32 s9, s5, s7
-; GFX11-NEXT:    v_cmp_lt_i64_e64 s6, s[6:7], 0
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    v_cmp_lt_i64_e64 s4, s[8:9], s[4:5]
-; GFX11-NEXT:    v_mov_b32_e32 v0, s8
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_xor_b32 s4, s6, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX11-NEXT:    v_cmp_lt_i64_e64 s5, s[6:7], 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s4, s5, s4
+; GFX11-NEXT:    s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s4, 0, 1
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s4
+; GFX11-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    global_store_b8 v2, v3, s[2:3]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %sadd, 0
   %carry = extractvalue { i64, i1 } %sadd, 1

 ; SI-NEXT:    s_mov_b32 s5, s3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
 ; SI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
 ; SI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
 ; SI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[8:11], 0
 ; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT:    s_cselect_b32 s0, 0, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: v_saddo_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v6, s2
 ; VI-NEXT:    v_mov_b32_e32 v7, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, v0, v2
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, v1, v3, vcc
 ; VI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
 ; VI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[0:1]
 ; VI-NEXT:    flat_store_dwordx2 v[4:5], v[8:9]
 ; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT:    s_cselect_b32 s0, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    flat_store_byte v[6:7], v0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_saddo_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v6, s[8:9]
 ; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[10:11]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v0, v2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
 ; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
 ; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]
 ; GFX9-NEXT:    global_store_dwordx2 v6, v[4:5], s[4:5]
 ; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    global_store_byte v6, v0, s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: v_saddo_i64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v6, s[8:9]
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v6, s[10:11]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
-; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, v[4:5], v[0:1]
-; GFX10-NEXT:    s_xor_b32 s0, vcc_lo, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX10-NEXT:    s_xor_b32 s0, s0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-NEXT:    global_store_dwordx2 v6, v[4:5], s[4:5]
 ; GFX10-NEXT:    global_store_byte v6, v0, s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: v_saddo_i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b256 s[4:11], s[0:1], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[8:9]
 ; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[10:11]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v2
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cmp_lt_i64_e64 s0, v[4:5], v[0:1]
-; GFX11-NEXT:    s_xor_b32 s0, vcc_lo, s0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-NEXT:    v_cmp_gt_i64_e64 s0, 0, v[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX11-NEXT:    s_xor_b32 s0, s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b64 v6, v[4:5], s[4:5]
 ; GFX11-NEXT:    global_store_b8 v6, v0, s[6:7]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind

 ; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
 ; SI-NEXT:    s_mov_b32 s8, s0
 ; SI-NEXT:    s_mov_b32 s9, s1
 ; SI-NEXT:    s_mov_b32 s12, s2
 ; SI-NEXT:    s_mov_b32 s13, s3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, v1, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
-; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v5, v1
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
 ; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v4, v0
+; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v5, v1
+; SI-NEXT:    s_xor_b64 s[2:3], vcc, s[2:3]
 ; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
-; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, 1
+; SI-NEXT:    s_and_b64 s[0:1], s[2:3], exec
+; SI-NEXT:    s_cselect_b32 s0, 0, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s4
 ; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[8:11], 0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[12:15], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: v_saddo_v2i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v6, s2
 ; VI-NEXT:    v_mov_b32_e32 v7, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v9, vcc, v1, v3
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, v0, v2
-; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
-; VI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v1
 ; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
 ; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v0
+; VI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v1
+; VI-NEXT:    s_xor_b64 s[2:3], vcc, s[2:3]
 ; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
-; VI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, 1
+; VI-NEXT:    s_and_b64 s[0:1], s[2:3], exec
+; VI-NEXT:    s_cselect_b32 s0, 0, 1
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    flat_store_dwordx2 v[4:5], v[8:9]
 ; VI-NEXT:    flat_store_dwordx2 v[6:7], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_saddo_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v6, s[4:5]
 ; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[6:7]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u32_e32 v5, v1, v3
 ; GFX9-NEXT:    v_add_i32 v1, v1, v3 clamp
 ; GFX9-NEXT:    v_add_u32_e32 v4, v0, v2
 ; GFX9-NEXT:    v_add_i32 v0, v0, v2 clamp
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v1
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v4, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 1, 0, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v6, v[4:5], s[0:1]
 ; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: v_saddo_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v5, s[4:5]
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v5, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_nc_u32_e32 v4, v1, v3
 ; GFX10-NEXT:    v_add_nc_i32 v1, v1, v3 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e32 v3, v0, v2
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc_lo
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 1, 0, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx2 v5, v[3:4], s[0:1]
 ; GFX10-NEXT:    global_store_dwordx2 v5, v[0:1], s[2:3]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: v_saddo_v2i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_load_b64 v[0:1], v5, s[4:5]
 ; GFX11-NEXT:    global_load_b64 v[2:3], v5, s[6:7]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, v1, v3
 ; GFX11-NEXT:    v_add_nc_i32 v1, v1, v3 clamp
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, v0, v2
 ; GFX11-NEXT:    v_add_nc_i32 v0, v0, v2 clamp
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc_lo
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 1, 0, vcc_lo
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b64 v5, v[3:4], s[0:1]
 ; GFX11-NEXT:    global_store_b64 v5, v[0:1], s[2:3]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
   %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
   %sadd = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
   %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
   %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
   store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
   %carry.ext = zext <2 x i1> %carry to <2 x i32>
   store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
   ret void
 }

llvm/test/CodeGen/AMDGPU/sdiv64.ll

 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
-; GCN-IR-NEXT:    s_mov_b32 s15, 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
 ; GCN-IR-NEXT:    s_ashr_i32 s2, s9, 31
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[6:7]
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
 ; GCN-IR-NEXT:    s_sub_u32 s12, s6, s0
 ; GCN-IR-NEXT:    s_subb_u32 s13, s7, s0

 ; GCN-IR-NEXT:    s_subb_u32 s17, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[16:17], 63
 ; GCN-IR-NEXT:    s_or_b64 s[20:21], s[10:11], s[20:21]
 ; GCN-IR-NEXT:    s_and_b64 s[10:11], s[20:21], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s13
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s12
 ; GCN-IR-NEXT:    s_or_b64 s[20:21], s[20:21], s[22:23]
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[20:21], exec
+; GCN-IR-NEXT:    s_cselect_b32 s15, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
+; GCN-IR-NEXT:    s_cmp_lg_u32 s15, 1
+; GCN-IR-NEXT:    s_mov_b32 s15, 0
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s20, s16, 1
 ; GCN-IR-NEXT:    s_addc_u32 s21, s17, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[20:21], 0
-; GCN-IR-NEXT:    s_sub_i32 s16, 63, s16
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s16
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], exec
+; GCN-IR-NEXT:    s_cselect_b32 s17, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s16
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s10
+; GCN-IR-NEXT:    s_cmp_lg_u32 s17, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s20
 ; GCN-IR-NEXT:    s_add_u32 s19, s6, -1
 ; GCN-IR-NEXT:    s_addc_u32 s20, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[14:15]
 ; GCN-IR-NEXT:    s_add_u32 s12, s8, s18
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0

 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[0:3], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xb
-; GCN-IR-NEXT:    s_mov_b32 s15, 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s5, s5
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[4:5], 24
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s7
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 16
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 24
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[2:3], 16
 ; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31

 ; GCN-IR-NEXT:    s_subb_u32 s17, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[16:17], 63
 ; GCN-IR-NEXT:    s_or_b64 s[20:21], s[10:11], s[20:21]
 ; GCN-IR-NEXT:    s_and_b64 s[10:11], s[20:21], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s13
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s12
 ; GCN-IR-NEXT:    s_or_b64 s[20:21], s[20:21], s[22:23]
+; GCN-IR-NEXT:    s_and_b64 s[20:21], s[20:21], exec
+; GCN-IR-NEXT:    s_cselect_b32 s15, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_5
+; GCN-IR-NEXT:    s_cmp_lg_u32 s15, 1
+; GCN-IR-NEXT:    s_mov_b32 s15, 0
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB9_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s20, s16, 1
 ; GCN-IR-NEXT:    s_addc_u32 s21, s17, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[20:21], 0
-; GCN-IR-NEXT:    s_sub_i32 s16, 63, s16
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s16
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_4
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], exec
+; GCN-IR-NEXT:    s_cselect_b32 s17, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s16
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s10
+; GCN-IR-NEXT:    s_cmp_lg_u32 s17, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s20
 ; GCN-IR-NEXT:    s_add_u32 s19, s6, -1
 ; GCN-IR-NEXT:    s_addc_u32 s20, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[14:15]
 ; GCN-IR-NEXT:    s_add_u32 s12, s8, s18
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0

 ; GCN-IR-NEXT:    s_addc_u32 s13, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[12:13], 63
 ; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[14:15]
 ; GCN-IR-NEXT:    s_and_b64 s[8:9], s[14:15], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, 24
 ; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
+; GCN-IR-NEXT:    s_and_b64 s[14:15], s[14:15], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB10_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[14:15], 0
-; GCN-IR-NEXT:    s_sub_i32 s11, 63, s12
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_lshl_b64 s[8:9], 24, s11
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_4
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GCN-IR-NEXT:    s_cselect_b32 s11, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s12
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], 24, s8
+; GCN-IR-NEXT:    s_cmp_lg_u32 s11, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], 24, s14
 ; GCN-IR-NEXT:    s_add_u32 s16, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s3, -1
 ; GCN-IR-NEXT:    s_sub_u32 s10, 58, s10
 ; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0

llvm/test/CodeGen/AMDGPU/setcc-multiple-use.ll

 ; CHECK-LABEL: f:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    ds_read_b32 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; CHECK-NEXT:    v_cndmask_b32_e64 v1, 1, 0, vcc_lo
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc_lo
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = load i32, i32 addrspace(3)* null, align 16
   %i6 = icmp ult i32 0, %i
   %i7 = sext i1 %i6 to i32
   %i8 = add i32 %i7, 1
   %i9 = and i32 %i8, %i7
   ret i32 %i9
 }

llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll

 
 define amdgpu_kernel void @sgpr_if_else_salu_br(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
 ; SI-LABEL: sgpr_if_else_salu_br:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xb
 ; SI-NEXT:    s_load_dword s2, s[0:1], 0xf
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_lg_u32 s4, 0
-; SI-NEXT:    s_cbranch_scc0 .LBB0_4
+; SI-NEXT:    s_cbranch_scc0 .LBB0_2
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    s_add_i32 s7, s7, s2
-; SI-NEXT:    s_cbranch_execnz .LBB0_3
-; SI-NEXT:  .LBB0_2: ; %if
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_branch .LBB0_3
+; SI-NEXT:  .LBB0_2:
+; SI-NEXT:    s_mov_b64 s[2:3], -1
+; SI-NEXT:    ; implicit-def: $sgpr7
+; SI-NEXT:  .LBB0_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_cselect_b32 s2, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s2, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB0_5
+; SI-NEXT:  ; %bb.4: ; %if
 ; SI-NEXT:    s_sub_i32 s7, s5, s6
-; SI-NEXT:  .LBB0_3: ; %endif
+; SI-NEXT:  .LBB0_5: ; %endif
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_add_i32 s4, s7, s4
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
-; SI-NEXT:  .LBB0_4:
-; SI-NEXT:    ; implicit-def: $sgpr7
-; SI-NEXT:    s_branch .LBB0_2
 
 entry:
   %0 = icmp eq i32 %a, 0
   br i1 %0, label %if, label %else
 
 if:
   %1 = sub i32 %b, %c
   br label %endif

 }
 
 define amdgpu_kernel void @sgpr_if_else_salu_br_opt(i32 addrspace(1)* %out, [8 x i32], i32 %a, [8 x i32], i32 %b, [8 x i32], i32 %c, [8 x i32], i32 %d, [8 x i32], i32 %e) {
 ; SI-LABEL: sgpr_if_else_salu_br_opt:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_load_dword s4, s[0:1], 0x13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_lg_u32 s4, 0
-; SI-NEXT:    s_cbranch_scc0 .LBB1_4
+; SI-NEXT:    s_cbranch_scc0 .LBB1_2
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    s_load_dword s2, s[0:1], 0x2e
 ; SI-NEXT:    s_load_dword s3, s[0:1], 0x37
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_add_i32 s5, s2, s3
-; SI-NEXT:    s_cbranch_execnz .LBB1_3
-; SI-NEXT:  .LBB1_2: ; %if
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_branch .LBB1_3
+; SI-NEXT:  .LBB1_2:
+; SI-NEXT:    s_mov_b64 s[2:3], -1
+; SI-NEXT:    ; implicit-def: $sgpr5
+; SI-NEXT:  .LBB1_3: ; %Flow
+; SI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_cselect_b32 s2, 0, 1
+; SI-NEXT:    s_cmp_lg_u32 s2, 1
+; SI-NEXT:    s_cbranch_scc1 .LBB1_5
+; SI-NEXT:  ; %bb.4: ; %if
 ; SI-NEXT:    s_load_dword s2, s[0:1], 0x1c
 ; SI-NEXT:    s_load_dword s3, s[0:1], 0x25
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_add_i32 s5, s2, s3
-; SI-NEXT:  .LBB1_3: ; %endif
+; SI-NEXT:  .LBB1_5: ; %endif
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; SI-NEXT:    s_add_i32 s4, s5, s4
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
-; SI-NEXT:  .LBB1_4:
-; SI-NEXT:    ; implicit-def: $sgpr5
-; SI-NEXT:    s_branch .LBB1_2
 
 entry:
   %cmp0 = icmp eq i32 %a, 0
   br i1 %cmp0, label %if, label %else
 
 if:
   %add0 = add i32 %b, %c
   br label %endif

llvm/test/CodeGen/AMDGPU/sign_extend.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-- -amdgpu-scalarize-global-loads=false -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s -allow-deprecated-dag-overlap -enable-var-scope --check-prefix=SI
 ; RUN: llc -mtriple=amdgcn-- -amdgpu-scalarize-global-loads=false -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck %s -allow-deprecated-dag-overlap -enable-var-scope --check-prefix=VI
 
 define amdgpu_kernel void @s_sext_i1_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
 ; SI-LABEL: s_sext_i1_to_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, s3
 ; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_cselect_b32 s0, 0, -1
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_sext_i1_to_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, s3
 ; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_cselect_b32 s0, 0, -1
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
   %cmp = icmp eq i32 %a, %b
   %sext = sext i1 %cmp to i32
   store i32 %sext, i32 addrspace(1)* %out, align 4
   ret void
 }
 

 ; SI-LABEL: s_sext_i1_to_i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, s3
 ; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_cselect_b32 s0, 0, -1
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_sext_i1_to_i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, s3
 ; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_cselect_b32 s0, 0, -1
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
   %cmp = icmp eq i32 %a, %b
   %sext = sext i1 %cmp to i16
   store i16 %sext, i16 addrspace(1)* %out
   ret void
 }
 

 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s4, s5
 ; SI-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; SI-NEXT:    s_cmp_eq_u32 s6, s7
 ; SI-NEXT:    s_cselect_b64 s[6:7], -1, 0
 ; SI-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; SI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_cselect_b32 s4, 0, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_sext_i1_to_i16_with_and:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x2c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s4, s5
 ; VI-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; VI-NEXT:    s_cmp_eq_u32 s6, s7
 ; VI-NEXT:    s_cselect_b64 s[6:7], -1, 0
 ; VI-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
-; VI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; VI-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT:    s_cselect_b32 s4, 0, -1
+; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %cmp0 = icmp eq i32 %a, %b
   %cmp1 = icmp eq i32 %c, %d
   %cmp = and i1 %cmp0, %cmp1
   %sext = sext i1 %cmp to i16
   store i16 %sext, i16 addrspace(1)* %out
   ret void

llvm/test/CodeGen/AMDGPU/srem64.ll

 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
 ; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[16:17], exec
+; GCN-IR-NEXT:    s_cselect_b32 s11, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_cmp_lg_u32 s11, 1
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s12
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
 ; GCN-IR-NEXT:    s_add_u32 s16, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[10:11]
 ; GCN-IR-NEXT:    s_add_u32 s10, s6, s14
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0

 ; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem33_64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GCN-IR-NEXT:    s_mov_b32 s13, 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[0:1], 31
 ; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
 ; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
 ; GCN-IR-NEXT:    s_mov_b32 s11, s10
 ; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[0:1]

 ; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[14:15], 63
 ; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
 ; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[18:19], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_5
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB8_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
-; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[2:3], s14
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_4
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], exec
+; GCN-IR-NEXT:    s_cselect_b32 s15, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s14
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[2:3], s10
+; GCN-IR-NEXT:    s_cmp_lg_u32 s15, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[2:3], s18
 ; GCN-IR-NEXT:    s_add_u32 s18, s8, -1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s9, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[12:13]
 ; GCN-IR-NEXT:    s_add_u32 s12, s6, s16
 ; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0

 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[0:3], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xb
-; GCN-IR-NEXT:    s_mov_b32 s13, 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s5, s5
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s7
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[4:5], 24
 ; GCN-IR-NEXT:    s_ashr_i64 s[4:5], s[6:7], 24
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 16
 ; GCN-IR-NEXT:    s_lshl_b64 s[4:5], s[4:5], 16
 ; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[2:3], 16

 ; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[14:15], 63
 ; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
 ; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s5
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s4
 ; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    s_and_b64 s[18:19], s[18:19], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_5
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_mov_b32 s13, 0
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB9_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
-; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
-; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[4:5], s14
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_4
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], exec
+; GCN-IR-NEXT:    s_cselect_b32 s15, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s14
+; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[4:5], s10
+; GCN-IR-NEXT:    s_cmp_lg_u32 s15, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[4:5], s18
 ; GCN-IR-NEXT:    s_add_u32 s18, s6, -1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
 ; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0

 ; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[4:5], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[2:3], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[2:3], 63
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[10:11], s[12:13]
 ; GCN-IR-NEXT:    s_and_b64 s[10:11], s[12:13], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, 24
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB10_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s10, s2, 1
 ; GCN-IR-NEXT:    s_addc_u32 s11, s3, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
 ; GCN-IR-NEXT:    s_sub_i32 s2, 63, s2
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], 24, s2
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_4
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s10
 ; GCN-IR-NEXT:    s_add_u32 s14, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s5, -1
 ; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0

llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll

 bb:
   tail call void @wobble()
   unreachable
 }
 
 define amdgpu_kernel void @kernel(i32 addrspace(1)* %arg1.global, i1 %tmp3.i.i, i32 %tmp5.i.i, i32 %tmp427.i, i1 %tmp438.i, double %tmp27.i, i1 %tmp48.i) {
 ; GLOBALNESS1-LABEL: kernel:
 ; GLOBALNESS1:       ; %bb.0: ; %bb
-; GLOBALNESS1-NEXT:    s_mov_b64 s[54:55], s[6:7]
-; GLOBALNESS1-NEXT:    s_load_dwordx4 s[36:39], s[8:9], 0x0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[36:37], s[6:7]
+; GLOBALNESS1-NEXT:    s_load_dwordx4 s[76:79], s[8:9], 0x0
 ; GLOBALNESS1-NEXT:    s_load_dword s6, s[8:9], 0x14
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v42, v0
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v44, 0
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v41, v0
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v42, 0
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS1-NEXT:    global_store_dword v[0:1], v44, off
+; GLOBALNESS1-NEXT:    global_store_dword v[0:1], v42, off
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT:    global_load_dword v0, v44, s[36:37]
+; GLOBALNESS1-NEXT:    global_load_dword v0, v42, s[76:77]
 ; GLOBALNESS1-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
-; GLOBALNESS1-NEXT:    s_mov_b64 s[64:65], s[4:5]
-; GLOBALNESS1-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
-; GLOBALNESS1-NEXT:    s_load_dword s7, s[8:9], 0x20
 ; GLOBALNESS1-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GLOBALNESS1-NEXT:    s_add_u32 s0, s0, s17
 ; GLOBALNESS1-NEXT:    s_addc_u32 s1, s1, 0
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v45, 0x40994400
-; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s38, 0
-; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e64 s[36:37], s[4:5], v[44:45]
-; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e64 s[40:41], s[4:5], 0
-; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT:    s_xor_b64 s[94:95], s[4:5], -1
+; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s78, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[38:39], s[8:9]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GLOBALNESS1-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
+; GLOBALNESS1-NEXT:    s_nop 0
+; GLOBALNESS1-NEXT:    s_load_dword s8, s[8:9], 0x20
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[82:83], -1, 0
+; GLOBALNESS1-NEXT:    s_xor_b64 s[84:85], s[82:83], -1
 ; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s6, 0
-; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
-; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT:    s_xor_b64 s[88:89], s[4:5], -1
-; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s7, 0
-; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT:    s_getpc_b64 s[6:7]
-; GLOBALNESS1-NEXT:    s_add_u32 s6, s6, wobble@gotpcrel32@lo+4
-; GLOBALNESS1-NEXT:    s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12
-; GLOBALNESS1-NEXT:    s_xor_b64 s[86:87], s[4:5], -1
-; GLOBALNESS1-NEXT:    s_load_dwordx2 s[66:67], s[6:7], 0x0
-; GLOBALNESS1-NEXT:    s_mov_b32 s98, s16
-; GLOBALNESS1-NEXT:    s_mov_b64 s[62:63], s[8:9]
-; GLOBALNESS1-NEXT:    s_mov_b32 s99, s15
-; GLOBALNESS1-NEXT:    s_mov_b32 s100, s14
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS1-NEXT:    s_xor_b64 s[86:87], s[6:7], -1
+; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s8, 0
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS1-NEXT:    s_xor_b64 s[88:89], s[6:7], -1
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v43, 0x40994400
+; GLOBALNESS1-NEXT:    s_mov_b32 s52, s16
+; GLOBALNESS1-NEXT:    s_mov_b32 s53, s15
+; GLOBALNESS1-NEXT:    s_mov_b32 s54, s14
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[34:35], s[10:11]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[92:93], 0x80
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[42:43], 1, v1
-; GLOBALNESS1-NEXT:    s_mov_b32 s69, 0x3ff00000
+; GLOBALNESS1-NEXT:    s_mov_b64 s[80:81], 0x80
+; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e64 s[42:43], s[4:5], v[42:43]
+; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e64 s[100:101], s[4:5], 0
+; GLOBALNESS1-NEXT:    s_mov_b32 s57, 0x3ff00000
 ; GLOBALNESS1-NEXT:    s_mov_b32 s32, 0
 ; GLOBALNESS1-NEXT:    ; implicit-def: $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
 ; GLOBALNESS1-NEXT:    s_waitcnt vmcnt(0)
-; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[4:5], 0, v0
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s4, 0
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s5, 1
-; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s4, 2
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s5, 3
-; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s4, 4
-; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[90:91], 1, v0
-; GLOBALNESS1-NEXT:    v_writelane_b32 v41, s5, 5
+; GLOBALNESS1-NEXT:    v_readfirstlane_b32 s6, v0
+; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s6, 0
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[90:91], -1, 0
+; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s6, 1
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[92:93], -1, 0
+; GLOBALNESS1-NEXT:    s_cmp_eq_u32 s6, 1
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[94:95], -1, 0
+; GLOBALNESS1-NEXT:    s_cmp_eq_u32 s6, 0
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[96:97], -1, 0
+; GLOBALNESS1-NEXT:    s_getpc_b64 s[6:7]
+; GLOBALNESS1-NEXT:    s_add_u32 s6, s6, wobble@gotpcrel32@lo+4
+; GLOBALNESS1-NEXT:    s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12
+; GLOBALNESS1-NEXT:    s_load_dwordx2 s[98:99], s[6:7], 0x0
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_4
 ; GLOBALNESS1-NEXT:  .LBB1_1: ; %bb70.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_readlane_b32 s6, v41, 4
-; GLOBALNESS1-NEXT:    v_readlane_b32 s7, v41, 5
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_29
+; GLOBALNESS1-NEXT:    s_and_b64 s[6:7], s[96:97], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s6, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s6, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc0 .LBB1_30
 ; GLOBALNESS1-NEXT:  .LBB1_2: ; %Flow6
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GLOBALNESS1-NEXT:  .LBB1_3: ; %Flow19
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a63, v31

 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a39, v7
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a38, v6
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a37, v5
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a36, v4
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a35, v3
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a34, v2
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a33, v1
 ; GLOBALNESS1-NEXT:    v_accvgpr_write_b32 a32, v0
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_30
+; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_31
 ; GLOBALNESS1-NEXT:  .LBB1_4: ; %bb5
 ; GLOBALNESS1-NEXT:    ; =>This Loop Header: Depth=1
-; GLOBALNESS1-NEXT:    ; Child Loop BB1_15 Depth 2
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[92:93], s[92:93] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    ; Child Loop BB1_16 Depth 2
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[80:81], s[80:81] op_sel:[0,1]
 ; GLOBALNESS1-NEXT:    flat_load_dword v40, v[0:1]
-; GLOBALNESS1-NEXT:    s_add_u32 s8, s62, 40
-; GLOBALNESS1-NEXT:    buffer_store_dword v44, off, s[0:3], 0
-; GLOBALNESS1-NEXT:    flat_load_dword v43, v[0:1]
-; GLOBALNESS1-NEXT:    s_addc_u32 s9, s63, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[64:65]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS1-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS1-NEXT:    buffer_store_dword v42, off, s[0:3], 0
+; GLOBALNESS1-NEXT:    flat_load_dword v44, v[0:1]
+; GLOBALNESS1-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s54
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s53
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s52
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[66:67]
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[42:43]
+; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[98:99]
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[82:83], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_8
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_9
 ; GLOBALNESS1-NEXT:  ; %bb.5: ; %NodeBlock
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s39, 1
+; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s79, 1
 ; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_7
 ; GLOBALNESS1-NEXT:  ; %bb.6: ; %LeafBlock3
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s39, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s79, 1
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], 0
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], -1
 ; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; GLOBALNESS1-NEXT:    s_cbranch_execnz .LBB1_8
-; GLOBALNESS1-NEXT:    s_branch .LBB1_23
+; GLOBALNESS1-NEXT:    s_branch .LBB1_8
 ; GLOBALNESS1-NEXT:  .LBB1_7: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], -1
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS1-NEXT:    s_branch .LBB1_23
-; GLOBALNESS1-NEXT:  .LBB1_8: ; %Flow16
+; GLOBALNESS1-NEXT:  .LBB1_8: ; %Flow17
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s8, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s8, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc0 .LBB1_24
+; GLOBALNESS1-NEXT:  .LBB1_9: ; %Flow16
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_24
-; GLOBALNESS1-NEXT:  .LBB1_9: ; %baz.exit.i
+; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_25
+; GLOBALNESS1-NEXT:  .LBB1_10: ; %baz.exit.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
 ; GLOBALNESS1-NEXT:    flat_load_dword v0, v[0:1]
-; GLOBALNESS1-NEXT:    s_mov_b32 s68, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s70, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s71, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s72, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s73, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s74, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s75, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s76, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s77, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s78, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s79, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s80, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s81, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s82, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s83, s69
-; GLOBALNESS1-NEXT:    s_mov_b32 s84, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s85, s69
+; GLOBALNESS1-NEXT:    s_mov_b32 s56, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s58, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s59, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s60, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s61, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s62, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s63, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s64, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s65, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s66, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s67, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s68, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s69, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s70, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s71, s57
+; GLOBALNESS1-NEXT:    s_mov_b32 s72, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s73, s57
 ; GLOBALNESS1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[96:97], 0, v0
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[68:69], s[68:69] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[2:3], s[70:71], s[70:71] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[4:5], s[72:73], s[72:73] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[6:7], s[74:75], s[74:75] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[8:9], s[76:77], s[76:77] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[10:11], s[78:79], s[78:79] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[12:13], s[80:81], s[80:81] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[14:15], s[82:83], s[82:83] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[16:17], s[84:85], s[84:85] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[18:19], s[86:87], s[86:87] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[20:21], s[88:89], s[88:89] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[22:23], s[90:91], s[90:91] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[24:25], s[92:93], s[92:93] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[26:27], s[94:95], s[94:95] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[28:29], s[96:97], s[96:97] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[30:31], s[98:99], s[98:99] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[70:71], s[96:97]
-; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_26
-; GLOBALNESS1-NEXT:  ; %bb.10: ; %bb33.i
+; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[44:45], 0, v0
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[56:57], s[56:57] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[2:3], s[58:59], s[58:59] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[4:5], s[60:61], s[60:61] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[6:7], s[62:63], s[62:63] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[8:9], s[64:65], s[64:65] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[10:11], s[66:67], s[66:67] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[12:13], s[68:69], s[68:69] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[14:15], s[70:71], s[70:71] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[16:17], s[72:73], s[72:73] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[18:19], s[74:75], s[74:75] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[20:21], s[76:77], s[76:77] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[22:23], s[78:79], s[78:79] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[24:25], s[80:81], s[80:81] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[26:27], s[82:83], s[82:83] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[28:29], s[84:85], s[84:85] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[30:31], s[86:87], s[86:87] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[58:59], s[44:45]
+; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_27
+; GLOBALNESS1-NEXT:  ; %bb.11: ; %bb33.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[2:3], 0, 0
 ; GLOBALNESS1-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GLOBALNESS1-NEXT:    v_readlane_b32 s4, v41, 0
-; GLOBALNESS1-NEXT:    v_readlane_b32 s5, v41, 1
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_12
-; GLOBALNESS1-NEXT:  ; %bb.11: ; %bb39.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v45, v44
-; GLOBALNESS1-NEXT:    global_store_dwordx2 v[2:3], v[44:45], off
-; GLOBALNESS1-NEXT:  .LBB1_12: ; %bb44.lr.ph.i
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[90:91], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_13
+; GLOBALNESS1-NEXT:  ; %bb.12: ; %bb39.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v43, v42
+; GLOBALNESS1-NEXT:    global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS1-NEXT:  .LBB1_13: ; %bb44.lr.ph.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v43
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v44
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e32 v2, 0, v40, vcc
-; GLOBALNESS1-NEXT:    s_mov_b64 s[72:73], s[42:43]
-; GLOBALNESS1-NEXT:    s_mov_b32 s75, s39
 ; GLOBALNESS1-NEXT:    s_waitcnt vmcnt(0)
-; GLOBALNESS1-NEXT:    v_cmp_nlt_f64_e64 s[56:57], 0, v[0:1]
-; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[58:59], 0, v2
-; GLOBALNESS1-NEXT:    s_branch .LBB1_15
-; GLOBALNESS1-NEXT:  .LBB1_13: ; %Flow7
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
+; GLOBALNESS1-NEXT:    v_cmp_nlt_f64_e64 s[46:47], 0, v[0:1]
+; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[48:49], 0, v2
+; GLOBALNESS1-NEXT:    s_branch .LBB1_16
+; GLOBALNESS1-NEXT:  .LBB1_14: ; %Flow7
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
 ; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS1-NEXT:  .LBB1_14: ; %bb63.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[86:87]
-; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_25
-; GLOBALNESS1-NEXT:  .LBB1_15: ; %bb44.i
+; GLOBALNESS1-NEXT:  .LBB1_15: ; %bb63.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[88:89], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc0 .LBB1_26
+; GLOBALNESS1-NEXT:  .LBB1_16: ; %bb44.i
 ; GLOBALNESS1-NEXT:    ; Parent Loop BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    ; => This Inner Loop Header: Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[94:95]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS1-NEXT:  ; %bb.16: ; %bb46.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[88:89]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS1-NEXT:  ; %bb.17: ; %bb50.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_20
-; GLOBALNESS1-NEXT:  ; %bb.18: ; %bb3.i.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[40:41]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_20
-; GLOBALNESS1-NEXT:  ; %bb.19: ; %bb6.i.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[56:57]
-; GLOBALNESS1-NEXT:  .LBB1_20: ; %spam.exit.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[90:91]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS1-NEXT:  ; %bb.21: ; %bb55.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    s_add_u32 s60, s62, 40
-; GLOBALNESS1-NEXT:    s_addc_u32 s61, s63, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[64:65]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[54:55]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[60:61]
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[84:85], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS1-NEXT:  ; %bb.17: ; %bb46.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[86:87], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS1-NEXT:  ; %bb.18: ; %bb50.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[42:43], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_21
+; GLOBALNESS1-NEXT:  ; %bb.19: ; %bb3.i.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[100:101], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_21
+; GLOBALNESS1-NEXT:  ; %bb.20: ; %bb6.i.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[46:47], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:  .LBB1_21: ; %spam.exit.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[92:93], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS1-NEXT:  ; %bb.22: ; %bb55.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    s_add_u32 s50, s38, 40
+; GLOBALNESS1-NEXT:    s_addc_u32 s51, s39, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[50:51]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v42
-; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[66:67]
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s54
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s53
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s52
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
+; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[98:99]
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[64:65]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[54:55]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[60:61]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[50:51]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s54
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s53
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s52
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    global_store_dwordx2 v[0:1], a[32:33], off
-; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[66:67]
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[58:59]
-; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_13
-; GLOBALNESS1-NEXT:  ; %bb.22: ; %bb62.i
-; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[98:99]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[48:49]
+; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_14
+; GLOBALNESS1-NEXT:  ; %bb.23: ; %bb62.i
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS1-NEXT:    global_store_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS1-NEXT:    s_branch .LBB1_13
-; GLOBALNESS1-NEXT:  .LBB1_23: ; %LeafBlock
+; GLOBALNESS1-NEXT:    global_store_dwordx2 v[0:1], v[42:43], off
+; GLOBALNESS1-NEXT:    s_branch .LBB1_14
+; GLOBALNESS1-NEXT:  .LBB1_24: ; %LeafBlock
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s39, 0
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s79, 0
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
 ; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
 ; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_9
-; GLOBALNESS1-NEXT:  .LBB1_24: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_10
+; GLOBALNESS1-NEXT:  .LBB1_25: ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS1-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_3
-; GLOBALNESS1-NEXT:  .LBB1_25: ; %Flow14
+; GLOBALNESS1-NEXT:  .LBB1_26: ; %Flow14
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[36:37]
-; GLOBALNESS1-NEXT:    s_mov_b32 s36, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s37, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s38, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s39, s93
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[40:41]
-; GLOBALNESS1-NEXT:    s_mov_b32 s40, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s41, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s42, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s43, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s44, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s45, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s46, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s47, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s48, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s49, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s50, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s51, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s52, s93
-; GLOBALNESS1-NEXT:    s_mov_b32 s53, s93
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[40:41], s[6:7]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GLOBALNESS1-NEXT:    s_mov_b32 s39, s75
-; GLOBALNESS1-NEXT:    s_mov_b64 s[42:43], s[72:73]
-; GLOBALNESS1-NEXT:  .LBB1_26: ; %Flow15
+; GLOBALNESS1-NEXT:    s_mov_b32 s60, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s61, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s62, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s63, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s64, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s65, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s66, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s67, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s68, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s69, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s70, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s71, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s72, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s73, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s74, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s75, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s76, s81
+; GLOBALNESS1-NEXT:    s_mov_b32 s77, s81
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], s[60:61], s[60:61] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[2:3], s[62:63], s[62:63] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[4:5], s[64:65], s[64:65] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[6:7], s[66:67], s[66:67] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[8:9], s[68:69], s[68:69] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[10:11], s[70:71], s[70:71] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[12:13], s[72:73], s[72:73] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[14:15], s[74:75], s[74:75] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[16:17], s[76:77], s[76:77] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[18:19], s[78:79], s[78:79] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[20:21], s[80:81], s[80:81] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[22:23], s[82:83], s[82:83] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[24:25], s[84:85], s[84:85] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[26:27], s[86:87], s[86:87] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[28:29], s[88:89], s[88:89] op_sel:[0,1]
+; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[30:31], s[90:91], s[90:91] op_sel:[0,1]
+; GLOBALNESS1-NEXT:  .LBB1_27: ; %Flow15
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[70:71]
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[96:97]
+; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[58:59]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[44:45]
 ; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_2
-; GLOBALNESS1-NEXT:  ; %bb.27: ; %bb67.i
+; GLOBALNESS1-NEXT:  ; %bb.28: ; %bb67.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_readlane_b32 s6, v41, 2
-; GLOBALNESS1-NEXT:    v_readlane_b32 s7, v41, 3
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_1
-; GLOBALNESS1-NEXT:  ; %bb.28: ; %bb69.i
+; GLOBALNESS1-NEXT:    s_and_b64 s[6:7], s[94:95], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s6, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s6, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_1
+; GLOBALNESS1-NEXT:  ; %bb.29: ; %bb69.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[32:33], 0, 0
-; GLOBALNESS1-NEXT:    global_store_dwordx2 v[32:33], v[44:45], off
+; GLOBALNESS1-NEXT:    global_store_dwordx2 v[32:33], v[42:43], off
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_1
-; GLOBALNESS1-NEXT:  .LBB1_29: ; %bb73.i
+; GLOBALNESS1-NEXT:  .LBB1_30: ; %bb73.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[32:33], 0, 0
-; GLOBALNESS1-NEXT:    global_store_dwordx2 v[32:33], v[44:45], off
+; GLOBALNESS1-NEXT:    global_store_dwordx2 v[32:33], v[42:43], off
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_2
-; GLOBALNESS1-NEXT:  .LBB1_30: ; %loop.exit.guard
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GLOBALNESS1-NEXT:  .LBB1_31: ; %loop.exit.guard
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], -1
-; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_32
-; GLOBALNESS1-NEXT:  ; %bb.31: ; %bb7.i.i
-; GLOBALNESS1-NEXT:    s_add_u32 s8, s62, 40
-; GLOBALNESS1-NEXT:    s_addc_u32 s9, s63, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[64:65]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS1-NEXT:    s_cbranch_scc0 .LBB1_33
+; GLOBALNESS1-NEXT:  ; %bb.32: ; %bb7.i.i
+; GLOBALNESS1-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS1-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s54
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s53
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s52
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
-; GLOBALNESS1-NEXT:  .LBB1_32: ; %Flow
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_34
-; GLOBALNESS1-NEXT:  ; %bb.33: ; %bb11.i.i
-; GLOBALNESS1-NEXT:    s_add_u32 s8, s62, 40
-; GLOBALNESS1-NEXT:    s_addc_u32 s9, s63, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[64:65]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS1-NEXT:  .LBB1_33: ; %Flow
+; GLOBALNESS1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GLOBALNESS1-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_35
+; GLOBALNESS1-NEXT:  ; %bb.34: ; %bb11.i.i
+; GLOBALNESS1-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS1-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s54
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s53
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s52
+; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS1-NEXT:  .LBB1_34: ; %UnifiedUnreachableBlock
+; GLOBALNESS1-NEXT:  .LBB1_35: ; %UnifiedUnreachableBlock
 ;
 ; GLOBALNESS0-LABEL: kernel:
 ; GLOBALNESS0:       ; %bb.0: ; %bb
-; GLOBALNESS0-NEXT:    s_mov_b64 s[54:55], s[6:7]
-; GLOBALNESS0-NEXT:    s_load_dwordx4 s[36:39], s[8:9], 0x0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[36:37], s[6:7]
+; GLOBALNESS0-NEXT:    s_load_dwordx4 s[76:79], s[8:9], 0x0
 ; GLOBALNESS0-NEXT:    s_load_dword s6, s[8:9], 0x14
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v42, v0
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v44, 0
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v41, v0
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v42, 0
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS0-NEXT:    global_store_dword v[0:1], v44, off
+; GLOBALNESS0-NEXT:    global_store_dword v[0:1], v42, off
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT:    global_load_dword v0, v44, s[36:37]
+; GLOBALNESS0-NEXT:    global_load_dword v0, v42, s[76:77]
 ; GLOBALNESS0-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
-; GLOBALNESS0-NEXT:    s_mov_b64 s[62:63], s[4:5]
-; GLOBALNESS0-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
-; GLOBALNESS0-NEXT:    s_load_dword s7, s[8:9], 0x20
 ; GLOBALNESS0-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GLOBALNESS0-NEXT:    s_add_u32 s0, s0, s17
 ; GLOBALNESS0-NEXT:    s_addc_u32 s1, s1, 0
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v45, 0x40994400
-; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s38, 0
-; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e64 s[36:37], s[4:5], v[44:45]
-; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e64 s[40:41], s[4:5], 0
-; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT:    s_xor_b64 s[94:95], s[4:5], -1
+; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s78, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[38:39], s[8:9]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GLOBALNESS0-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
+; GLOBALNESS0-NEXT:    s_nop 0
+; GLOBALNESS0-NEXT:    s_load_dword s8, s[8:9], 0x20
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[82:83], -1, 0
+; GLOBALNESS0-NEXT:    s_xor_b64 s[84:85], s[82:83], -1
 ; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s6, 0
-; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
-; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT:    s_xor_b64 s[88:89], s[4:5], -1
-; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s7, 0
-; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT:    s_getpc_b64 s[6:7]
-; GLOBALNESS0-NEXT:    s_add_u32 s6, s6, wobble@gotpcrel32@lo+4
-; GLOBALNESS0-NEXT:    s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12
-; GLOBALNESS0-NEXT:    s_xor_b64 s[86:87], s[4:5], -1
-; GLOBALNESS0-NEXT:    s_load_dwordx2 s[66:67], s[6:7], 0x0
-; GLOBALNESS0-NEXT:    s_mov_b32 s98, s16
-; GLOBALNESS0-NEXT:    s_mov_b64 s[60:61], s[8:9]
-; GLOBALNESS0-NEXT:    s_mov_b32 s99, s15
-; GLOBALNESS0-NEXT:    s_mov_b32 s100, s14
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS0-NEXT:    s_xor_b64 s[86:87], s[6:7], -1
+; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s8, 0
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS0-NEXT:    s_xor_b64 s[88:89], s[6:7], -1
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v43, 0x40994400
+; GLOBALNESS0-NEXT:    s_mov_b32 s50, s16
+; GLOBALNESS0-NEXT:    s_mov_b32 s51, s15
+; GLOBALNESS0-NEXT:    s_mov_b32 s52, s14
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[34:35], s[10:11]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[92:93], 0x80
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[42:43], 1, v1
-; GLOBALNESS0-NEXT:    s_mov_b32 s69, 0x3ff00000
+; GLOBALNESS0-NEXT:    s_mov_b64 s[80:81], 0x80
+; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e64 s[42:43], s[4:5], v[42:43]
+; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e64 s[100:101], s[4:5], 0
+; GLOBALNESS0-NEXT:    s_mov_b32 s57, 0x3ff00000
 ; GLOBALNESS0-NEXT:    s_mov_b32 s32, 0
 ; GLOBALNESS0-NEXT:    ; implicit-def: $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
 ; GLOBALNESS0-NEXT:    s_waitcnt vmcnt(0)
-; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[4:5], 0, v0
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s4, 0
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s5, 1
-; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s4, 2
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s5, 3
-; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s4, 4
-; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[90:91], 1, v0
-; GLOBALNESS0-NEXT:    v_writelane_b32 v41, s5, 5
+; GLOBALNESS0-NEXT:    v_readfirstlane_b32 s6, v0
+; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s6, 0
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[90:91], -1, 0
+; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s6, 1
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[92:93], -1, 0
+; GLOBALNESS0-NEXT:    s_cmp_eq_u32 s6, 1
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[94:95], -1, 0
+; GLOBALNESS0-NEXT:    s_cmp_eq_u32 s6, 0
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[96:97], -1, 0
+; GLOBALNESS0-NEXT:    s_getpc_b64 s[6:7]
+; GLOBALNESS0-NEXT:    s_add_u32 s6, s6, wobble@gotpcrel32@lo+4
+; GLOBALNESS0-NEXT:    s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12
+; GLOBALNESS0-NEXT:    s_load_dwordx2 s[98:99], s[6:7], 0x0
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_4
 ; GLOBALNESS0-NEXT:  .LBB1_1: ; %bb70.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_readlane_b32 s6, v41, 4
-; GLOBALNESS0-NEXT:    v_readlane_b32 s7, v41, 5
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_29
+; GLOBALNESS0-NEXT:    s_and_b64 s[6:7], s[96:97], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s6, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s6, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc0 .LBB1_30
 ; GLOBALNESS0-NEXT:  .LBB1_2: ; %Flow6
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
 ; GLOBALNESS0-NEXT:  .LBB1_3: ; %Flow19
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a63, v31

 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a39, v7
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a38, v6
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a37, v5
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a36, v4
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a35, v3
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a34, v2
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a33, v1
 ; GLOBALNESS0-NEXT:    v_accvgpr_write_b32 a32, v0
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_30
+; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_31
 ; GLOBALNESS0-NEXT:  .LBB1_4: ; %bb5
 ; GLOBALNESS0-NEXT:    ; =>This Loop Header: Depth=1
-; GLOBALNESS0-NEXT:    ; Child Loop BB1_15 Depth 2
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[92:93], s[92:93] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    ; Child Loop BB1_16 Depth 2
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[80:81], s[80:81] op_sel:[0,1]
 ; GLOBALNESS0-NEXT:    flat_load_dword v40, v[0:1]
-; GLOBALNESS0-NEXT:    s_add_u32 s8, s60, 40
-; GLOBALNESS0-NEXT:    buffer_store_dword v44, off, s[0:3], 0
-; GLOBALNESS0-NEXT:    flat_load_dword v43, v[0:1]
-; GLOBALNESS0-NEXT:    s_addc_u32 s9, s61, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[62:63]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS0-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS0-NEXT:    buffer_store_dword v42, off, s[0:3], 0
+; GLOBALNESS0-NEXT:    flat_load_dword v44, v[0:1]
+; GLOBALNESS0-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s52
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s51
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s50
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[66:67]
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[42:43]
+; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[98:99]
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[82:83], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_8
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_9
 ; GLOBALNESS0-NEXT:  ; %bb.5: ; %NodeBlock
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s39, 1
+; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s79, 1
 ; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_7
 ; GLOBALNESS0-NEXT:  ; %bb.6: ; %LeafBlock3
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s39, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s79, 1
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], 0
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], -1
 ; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; GLOBALNESS0-NEXT:    s_cbranch_execnz .LBB1_8
-; GLOBALNESS0-NEXT:    s_branch .LBB1_23
+; GLOBALNESS0-NEXT:    s_branch .LBB1_8
 ; GLOBALNESS0-NEXT:  .LBB1_7: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], -1
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS0-NEXT:    s_branch .LBB1_23
-; GLOBALNESS0-NEXT:  .LBB1_8: ; %Flow16
+; GLOBALNESS0-NEXT:  .LBB1_8: ; %Flow17
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s8, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s8, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc0 .LBB1_24
+; GLOBALNESS0-NEXT:  .LBB1_9: ; %Flow16
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_24
-; GLOBALNESS0-NEXT:  .LBB1_9: ; %baz.exit.i
+; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_25
+; GLOBALNESS0-NEXT:  .LBB1_10: ; %baz.exit.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
 ; GLOBALNESS0-NEXT:    flat_load_dword v0, v[0:1]
-; GLOBALNESS0-NEXT:    s_mov_b32 s68, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s70, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s71, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s72, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s73, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s74, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s75, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s76, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s77, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s78, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s79, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s80, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s81, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s82, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s83, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s84, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s85, s69
+; GLOBALNESS0-NEXT:    s_mov_b32 s56, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s58, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s59, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s60, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s61, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s62, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s63, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s64, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s65, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s66, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s67, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s68, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s69, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s70, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s71, s57
+; GLOBALNESS0-NEXT:    s_mov_b32 s72, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s73, s57
 ; GLOBALNESS0-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[96:97], 0, v0
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[68:69], s[68:69] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[2:3], s[70:71], s[70:71] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[4:5], s[72:73], s[72:73] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[6:7], s[74:75], s[74:75] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[8:9], s[76:77], s[76:77] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[10:11], s[78:79], s[78:79] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[12:13], s[80:81], s[80:81] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[14:15], s[82:83], s[82:83] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[16:17], s[84:85], s[84:85] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[18:19], s[86:87], s[86:87] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[20:21], s[88:89], s[88:89] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[22:23], s[90:91], s[90:91] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[24:25], s[92:93], s[92:93] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[26:27], s[94:95], s[94:95] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[28:29], s[96:97], s[96:97] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[30:31], s[98:99], s[98:99] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[70:71], s[96:97]
-; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_26
-; GLOBALNESS0-NEXT:  ; %bb.10: ; %bb33.i
+; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[44:45], 0, v0
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[56:57], s[56:57] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[2:3], s[58:59], s[58:59] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[4:5], s[60:61], s[60:61] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[6:7], s[62:63], s[62:63] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[8:9], s[64:65], s[64:65] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[10:11], s[66:67], s[66:67] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[12:13], s[68:69], s[68:69] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[14:15], s[70:71], s[70:71] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[16:17], s[72:73], s[72:73] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[18:19], s[74:75], s[74:75] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[20:21], s[76:77], s[76:77] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[22:23], s[78:79], s[78:79] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[24:25], s[80:81], s[80:81] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[26:27], s[82:83], s[82:83] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[28:29], s[84:85], s[84:85] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[30:31], s[86:87], s[86:87] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[58:59], s[44:45]
+; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_27
+; GLOBALNESS0-NEXT:  ; %bb.11: ; %bb33.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[2:3], 0, 0
 ; GLOBALNESS0-NEXT:    global_load_dwordx2 v[0:1], v[2:3], off
-; GLOBALNESS0-NEXT:    v_readlane_b32 s4, v41, 0
-; GLOBALNESS0-NEXT:    v_readlane_b32 s5, v41, 1
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_12
-; GLOBALNESS0-NEXT:  ; %bb.11: ; %bb39.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v45, v44
-; GLOBALNESS0-NEXT:    global_store_dwordx2 v[2:3], v[44:45], off
-; GLOBALNESS0-NEXT:  .LBB1_12: ; %bb44.lr.ph.i
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[90:91], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_13
+; GLOBALNESS0-NEXT:  ; %bb.12: ; %bb39.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v43, v42
+; GLOBALNESS0-NEXT:    global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS0-NEXT:  .LBB1_13: ; %bb44.lr.ph.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v43
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v44
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e32 v2, 0, v40, vcc
-; GLOBALNESS0-NEXT:    s_mov_b64 s[72:73], s[42:43]
-; GLOBALNESS0-NEXT:    s_mov_b32 s75, s39
 ; GLOBALNESS0-NEXT:    s_waitcnt vmcnt(0)
-; GLOBALNESS0-NEXT:    v_cmp_nlt_f64_e64 s[56:57], 0, v[0:1]
-; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[58:59], 0, v2
-; GLOBALNESS0-NEXT:    s_branch .LBB1_15
-; GLOBALNESS0-NEXT:  .LBB1_13: ; %Flow7
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
+; GLOBALNESS0-NEXT:    v_cmp_nlt_f64_e64 s[46:47], 0, v[0:1]
+; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[48:49], 0, v2
+; GLOBALNESS0-NEXT:    s_branch .LBB1_16
+; GLOBALNESS0-NEXT:  .LBB1_14: ; %Flow7
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
 ; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS0-NEXT:  .LBB1_14: ; %bb63.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[86:87]
-; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_25
-; GLOBALNESS0-NEXT:  .LBB1_15: ; %bb44.i
+; GLOBALNESS0-NEXT:  .LBB1_15: ; %bb63.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[88:89], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc0 .LBB1_26
+; GLOBALNESS0-NEXT:  .LBB1_16: ; %bb44.i
 ; GLOBALNESS0-NEXT:    ; Parent Loop BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    ; => This Inner Loop Header: Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[94:95]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS0-NEXT:  ; %bb.16: ; %bb46.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[88:89]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS0-NEXT:  ; %bb.17: ; %bb50.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_20
-; GLOBALNESS0-NEXT:  ; %bb.18: ; %bb3.i.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[40:41]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_20
-; GLOBALNESS0-NEXT:  ; %bb.19: ; %bb6.i.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[56:57]
-; GLOBALNESS0-NEXT:  .LBB1_20: ; %spam.exit.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[90:91]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_14
-; GLOBALNESS0-NEXT:  ; %bb.21: ; %bb55.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    s_add_u32 s64, s60, 40
-; GLOBALNESS0-NEXT:    s_addc_u32 s65, s61, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[62:63]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[54:55]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[64:65]
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[84:85], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS0-NEXT:  ; %bb.17: ; %bb46.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[86:87], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS0-NEXT:  ; %bb.18: ; %bb50.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[42:43], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_21
+; GLOBALNESS0-NEXT:  ; %bb.19: ; %bb3.i.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[100:101], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_21
+; GLOBALNESS0-NEXT:  ; %bb.20: ; %bb6.i.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[46:47], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:  .LBB1_21: ; %spam.exit.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[92:93], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_15
+; GLOBALNESS0-NEXT:  ; %bb.22: ; %bb55.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    s_add_u32 s54, s38, 40
+; GLOBALNESS0-NEXT:    s_addc_u32 s55, s39, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[54:55]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v42
-; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[66:67]
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s52
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s51
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s50
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
+; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[98:99]
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[62:63]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[54:55]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[64:65]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[54:55]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s52
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s51
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s50
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    global_store_dwordx2 v[0:1], a[32:33], off
-; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[66:67]
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[58:59]
-; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_13
-; GLOBALNESS0-NEXT:  ; %bb.22: ; %bb62.i
-; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_15 Depth=2
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[98:99]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[48:49]
+; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_14
+; GLOBALNESS0-NEXT:  ; %bb.23: ; %bb62.i
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS0-NEXT:    global_store_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS0-NEXT:    s_branch .LBB1_13
-; GLOBALNESS0-NEXT:  .LBB1_23: ; %LeafBlock
+; GLOBALNESS0-NEXT:    global_store_dwordx2 v[0:1], v[42:43], off
+; GLOBALNESS0-NEXT:    s_branch .LBB1_14
+; GLOBALNESS0-NEXT:  .LBB1_24: ; %LeafBlock
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s39, 0
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s79, 0
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
 ; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
 ; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_9
-; GLOBALNESS0-NEXT:  .LBB1_24: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_10
+; GLOBALNESS0-NEXT:  .LBB1_25: ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS0-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_3
-; GLOBALNESS0-NEXT:  .LBB1_25: ; %Flow14
+; GLOBALNESS0-NEXT:  .LBB1_26: ; %Flow14
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[36:37]
-; GLOBALNESS0-NEXT:    s_mov_b32 s36, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s37, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s38, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s39, s93
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[40:41]
-; GLOBALNESS0-NEXT:    s_mov_b32 s40, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s41, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s42, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s43, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s44, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s45, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s46, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s47, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s48, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s49, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s50, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s51, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s52, s93
-; GLOBALNESS0-NEXT:    s_mov_b32 s53, s93
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[40:41], s[6:7]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GLOBALNESS0-NEXT:    s_mov_b32 s39, s75
-; GLOBALNESS0-NEXT:    s_mov_b64 s[42:43], s[72:73]
-; GLOBALNESS0-NEXT:  .LBB1_26: ; %Flow15
+; GLOBALNESS0-NEXT:    s_mov_b32 s60, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s61, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s62, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s63, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s64, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s65, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s66, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s67, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s68, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s69, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s70, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s71, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s72, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s73, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s74, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s75, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s76, s81
+; GLOBALNESS0-NEXT:    s_mov_b32 s77, s81
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], s[60:61], s[60:61] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[2:3], s[62:63], s[62:63] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[4:5], s[64:65], s[64:65] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[6:7], s[66:67], s[66:67] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[8:9], s[68:69], s[68:69] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[10:11], s[70:71], s[70:71] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[12:13], s[72:73], s[72:73] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[14:15], s[74:75], s[74:75] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[16:17], s[76:77], s[76:77] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[18:19], s[78:79], s[78:79] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[20:21], s[80:81], s[80:81] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[22:23], s[82:83], s[82:83] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[24:25], s[84:85], s[84:85] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[26:27], s[86:87], s[86:87] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[28:29], s[88:89], s[88:89] op_sel:[0,1]
+; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[30:31], s[90:91], s[90:91] op_sel:[0,1]
+; GLOBALNESS0-NEXT:  .LBB1_27: ; %Flow15
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[70:71]
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[96:97]
+; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[58:59]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[44:45]
 ; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_2
-; GLOBALNESS0-NEXT:  ; %bb.27: ; %bb67.i
+; GLOBALNESS0-NEXT:  ; %bb.28: ; %bb67.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_readlane_b32 s6, v41, 2
-; GLOBALNESS0-NEXT:    v_readlane_b32 s7, v41, 3
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_1
-; GLOBALNESS0-NEXT:  ; %bb.28: ; %bb69.i
+; GLOBALNESS0-NEXT:    s_and_b64 s[6:7], s[94:95], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s6, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s6, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_1
+; GLOBALNESS0-NEXT:  ; %bb.29: ; %bb69.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[32:33], 0, 0
-; GLOBALNESS0-NEXT:    global_store_dwordx2 v[32:33], v[44:45], off
+; GLOBALNESS0-NEXT:    global_store_dwordx2 v[32:33], v[42:43], off
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_1
-; GLOBALNESS0-NEXT:  .LBB1_29: ; %bb73.i
+; GLOBALNESS0-NEXT:  .LBB1_30: ; %bb73.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v45, v44
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v43, v42
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[32:33], 0, 0
-; GLOBALNESS0-NEXT:    global_store_dwordx2 v[32:33], v[44:45], off
+; GLOBALNESS0-NEXT:    global_store_dwordx2 v[32:33], v[42:43], off
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_2
-; GLOBALNESS0-NEXT:  .LBB1_30: ; %loop.exit.guard
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GLOBALNESS0-NEXT:  .LBB1_31: ; %loop.exit.guard
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], -1
-; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_32
-; GLOBALNESS0-NEXT:  ; %bb.31: ; %bb7.i.i
-; GLOBALNESS0-NEXT:    s_add_u32 s8, s60, 40
-; GLOBALNESS0-NEXT:    s_addc_u32 s9, s61, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[62:63]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS0-NEXT:    s_cbranch_scc0 .LBB1_33
+; GLOBALNESS0-NEXT:  ; %bb.32: ; %bb7.i.i
+; GLOBALNESS0-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS0-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s52
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s51
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s50
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
-; GLOBALNESS0-NEXT:  .LBB1_32: ; %Flow
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_34
-; GLOBALNESS0-NEXT:  ; %bb.33: ; %bb11.i.i
-; GLOBALNESS0-NEXT:    s_add_u32 s8, s60, 40
-; GLOBALNESS0-NEXT:    s_addc_u32 s9, s61, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[62:63]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[54:55]
+; GLOBALNESS0-NEXT:  .LBB1_33: ; %Flow
+; GLOBALNESS0-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GLOBALNESS0-NEXT:    s_cselect_b32 s4, 0, 1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s4, 1
+; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_35
+; GLOBALNESS0-NEXT:  ; %bb.34: ; %bb11.i.i
+; GLOBALNESS0-NEXT:    s_add_u32 s8, s38, 40
+; GLOBALNESS0-NEXT:    s_addc_u32 s9, s39, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s100
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s99
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s98
-; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v42
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s52
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s51
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s50
+; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS0-NEXT:  .LBB1_34: ; %UnifiedUnreachableBlock
+; GLOBALNESS0-NEXT:  .LBB1_35: ; %UnifiedUnreachableBlock
 bb:
   store i32 0, i32 addrspace(1)* null, align 4
   %tmp4 = load i32, i32 addrspace(1)* %arg1.global, align 4
   br label %bb5
 
 bb5:                                              ; preds = %bb5.backedge, %bb
   %tmp4.i.sroa.0.0 = phi <9 x double> [ undef, %bb ], [ %tmp4.i.sroa.0.1, %bb5.backedge ]
   %tmp14.1.i = load i32, i32* inttoptr (i64 128 to i32*), align 128

llvm/test/CodeGen/AMDGPU/udiv64.ll

 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
 ; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[16:17], exec
+; GCN-IR-NEXT:    s_cselect_b32 s11, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_cmp_lg_u32 s11, 1
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s12
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
 ; GCN-IR-NEXT:    s_add_u32 s15, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s16, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
 ; GCN-IR-NEXT:    s_add_u32 s2, s2, s14
 ; GCN-IR-NEXT:    s_addc_u32 s3, s3, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0

 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
 ; GCN-NEXT:    buffer_store_short v1, off, s[0:3], 0 offset:4
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv24_i48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xb
-; GCN-IR-NEXT:    s_mov_b32 s11, 0
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_and_b32 s3, s5, 0xffff
 ; GCN-IR-NEXT:    s_and_b32 s2, s4, 0xff000000
 ; GCN-IR-NEXT:    s_and_b32 s5, s7, 0xffff
 ; GCN-IR-NEXT:    s_and_b32 s4, s6, 0xff000000
 ; GCN-IR-NEXT:    s_lshr_b64 s[8:9], s[2:3], 24
 ; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[4:5], 24
 ; GCN-IR-NEXT:    s_and_b32 s9, s9, 0xffff

 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[6:7], s[16:17]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s9
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s8
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[16:17], exec
+; GCN-IR-NEXT:    s_cselect_b32 s11, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_5
+; GCN-IR-NEXT:    s_cmp_lg_u32 s11, 1
+; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB7_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[16:17], 0
-; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], s12
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_4
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s12
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], s6
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[8:9], s16
 ; GCN-IR-NEXT:    s_add_u32 s15, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s16, s3, -1
 ; GCN-IR-NEXT:    s_not_b64 s[4:5], s[10:11]
 ; GCN-IR-NEXT:    s_add_u32 s8, s4, s14
 ; GCN-IR-NEXT:    s_addc_u32 s9, s5, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0

 ; GCN-IR-NEXT:    s_addc_u32 s11, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[10:11], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 63
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[6:7], s[12:13]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[12:13], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, 24
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], exec
+; GCN-IR-NEXT:    s_cselect_b32 s7, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s7, 1
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB8_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s9
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_4
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s10
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s6
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s12
 ; GCN-IR-NEXT:    s_add_u32 s14, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s3, -1
 ; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0

 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[6:7], s[8:9], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[8:9], 63
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[4:5], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GCN-IR-NEXT:    s_cselect_b32 s4, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB11_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB11_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s12, s8, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s8
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB11_4
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s8
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s6
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB11_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[8:9], s[2:3], s12
 ; GCN-IR-NEXT:    s_add_u32 s2, s10, 0xffffffc4
 ; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  .LBB11_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1

llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll

 ; CHECK-NEXT:    s_mov_b32 s2, -1
 ; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], 0 glc
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; CHECK-NEXT:    buffer_store_dword v2, off, s[0:3], 0
 ; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; CHECK-NEXT:    s_waitcnt expcnt(0)
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v1
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; CHECK-NEXT:  .LBB1_1: ; %bb9
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; CHECK-NEXT:    s_cbranch_vccnz .LBB1_1
+; CHECK-NEXT:    s_and_b64 s[2:3], s[0:1], exec
+; CHECK-NEXT:    s_cselect_b32 s2, 0, 1
+; CHECK-NEXT:    s_cmp_lg_u32 s2, 1
+; CHECK-NEXT:    s_cbranch_scc1 .LBB1_1
 ; CHECK-NEXT:  ; %bb.2: ; %bb11
 ; CHECK-NEXT:    s_mov_b32 s3, 0xf000
 ; CHECK-NEXT:    s_mov_b32 s2, -1
 ; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; CHECK-NEXT:    ; return to shader part epilog
 bb:
   %tmp = load volatile i32, i32 addrspace(1)* undef, align 4

llvm/test/CodeGen/AMDGPU/urem64.ll

 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
 ; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[16:17], s[16:17], exec
+; GCN-IR-NEXT:    s_cselect_b32 s11, 0, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_cmp_lg_u32 s11, 1
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
-; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[8:9], exec
+; GCN-IR-NEXT:    s_cselect_b32 s13, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s12
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GCN-IR-NEXT:    s_cmp_lg_u32 s13, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
 ; GCN-IR-NEXT:    s_add_u32 s16, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[10:11]
 ; GCN-IR-NEXT:    s_add_u32 s10, s6, s14
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0

 ; GCN-IR-NEXT:    s_addc_u32 s11, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[10:11], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 63
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[6:7], s[12:13]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[12:13], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, 24
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_and_b64 s[12:13], s[12:13], exec
+; GCN-IR-NEXT:    s_cselect_b32 s7, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s7, 1
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB6_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB6_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s9
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB6_4
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s10
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s6
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB6_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s12
 ; GCN-IR-NEXT:    s_add_u32 s14, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s3, -1
 ; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0

 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[6:7], s[10:11], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 63
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[4:5], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s2
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GCN-IR-NEXT:    s_cselect_b32 s4, 0, 1
+; GCN-IR-NEXT:    s_cmp_lg_u32 s4, 1
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_5
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB7_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s9
-; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_4
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GCN-IR-NEXT:    s_cselect_b32 s9, 0, 1
+; GCN-IR-NEXT:    s_sub_i32 s6, 63, s10
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s6
+; GCN-IR-NEXT:    s_cmp_lg_u32 s9, 1
+; GCN-IR-NEXT:    s_cbranch_scc0 .LBB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[10:11], s[2:3], s12
 ; GCN-IR-NEXT:    s_add_u32 s8, s8, 0xffffffc4
 ; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
 ; GCN-IR-NEXT:  .LBB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1

llvm/test/CodeGen/AMDGPU/v_cmp_gfx11.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-wavefrontsize32,+wavefrontsize64 --global-isel=0 -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK %s
 
 define amdgpu_kernel void @icmp_test() {
 ; CHECK-LABEL: icmp_test:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; CHECK-NEXT:    s_cmp_eq_u64 s[0:1], 0
-; CHECK-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT:    ds_store_b32 v1, v0
+; CHECK-NEXT:    s_cselect_b32 s0, 0, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, s0
+; CHECK-NEXT:    ds_store_b32 v0, v1
 ; CHECK-NEXT:    s_endpgm
 entry:
   %icmp.intr = tail call i64 @llvm.amdgcn.icmp.i64.i16(i16 0, i16 0, i32 32)
   %cmp0 = icmp eq i64 %icmp.intr, 0
   %zext0 = zext i1 %cmp0 to i32
   store i32 %zext0, ptr addrspace(3) null, align 2147483648
   ret void
 }
 
 define amdgpu_kernel void @fcmp_test(half %x, half %y) {
 ; CHECK-LABEL: fcmp_test:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_b32 s0, s[0:1], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshr_b32 s1, s0, 16
 ; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; CHECK-NEXT:    v_cmp_le_f16_e64 s[0:1], s0, s1
 ; CHECK-NEXT:    s_cmp_eq_u64 s[0:1], 0
-; CHECK-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; CHECK-NEXT:    s_cselect_b32 s0, 0, 1
 ; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT:    ds_store_b32 v1, v0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s0
+; CHECK-NEXT:    ds_store_b32 v0, v1
 ; CHECK-NEXT:    s_endpgm
 entry:
   %fcmp.intr = tail call i64 @llvm.amdgcn.fcmp.i64.f16(half %x, half %y, i32 5)
   %cmp0 = icmp eq i64 %fcmp.intr, 0
   %zext0 = zext i1 %cmp0 to i32
   store i32 %zext0, ptr addrspace(3) null, align 2147483648
   ret void
 }

llvm/test/CodeGen/AMDGPU/wqm.ll

 ; GFX9-W64-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX9-W64-NEXT:    s_and_b64 exec, exec, s[0:1]
 ; GFX9-W64-NEXT:    image_store v[4:7], v0, s[0:7] dmask:0xf unorm
 ; GFX9-W64-NEXT:    s_wqm_b64 exec, exec
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-W64-NEXT:    s_mov_b32 s4, 0x40e00000
-; GFX9-W64-NEXT:    s_branch .LBB31_2
+; GFX9-W64-NEXT:    s_branch .LBB31_3
 ; GFX9-W64-NEXT:  .LBB31_1: ; %body
-; GFX9-W64-NEXT:    ; in Loop: Header=BB31_2 Depth=1
+; GFX9-W64-NEXT:    ; in Loop: Header=BB31_3 Depth=1
 ; GFX9-W64-NEXT:    image_sample v[4:7], v0, s[0:7], s[0:3] dmask:0xf
 ; GFX9-W64-NEXT:    v_add_f32_e32 v8, 2.0, v8
-; GFX9-W64-NEXT:    s_cbranch_execz .LBB31_4
-; GFX9-W64-NEXT:  .LBB31_2: ; %loop
+; GFX9-W64-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-W64-NEXT:  .LBB31_2: ; %Flow
+; GFX9-W64-NEXT:    ; in Loop: Header=BB31_3 Depth=1
+; GFX9-W64-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; GFX9-W64-NEXT:    s_cselect_b32 s2, 0, 1
+; GFX9-W64-NEXT:    s_cmp_lg_u32 s2, 1
+; GFX9-W64-NEXT:    s_cbranch_scc0 .LBB31_5
+; GFX9-W64-NEXT:  .LBB31_3: ; %loop
 ; GFX9-W64-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-W64-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v8
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX9-W64-NEXT:    s_cbranch_vccz .LBB31_1
-; GFX9-W64-NEXT:  ; %bb.3:
+; GFX9-W64-NEXT:  ; %bb.4: ; in Loop: Header=BB31_3 Depth=1
+; GFX9-W64-NEXT:    s_mov_b64 s[2:3], -1
 ; GFX9-W64-NEXT:    ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
 ; GFX9-W64-NEXT:    ; implicit-def: $vgpr8
-; GFX9-W64-NEXT:  .LBB31_4: ; %break
+; GFX9-W64-NEXT:    s_branch .LBB31_2
+; GFX9-W64-NEXT:  .LBB31_5: ; %break
 ; GFX9-W64-NEXT:    s_and_b64 exec, exec, s[0:1]
 ; GFX9-W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-W64-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-W32-LABEL: test_loop_vcc:
 ; GFX10-W32:       ; %bb.0: ; %entry
 ; GFX10-W32-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX10-W32-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX10-W32-NEXT:    s_and_b32 exec_lo, exec_lo, s0
 ; GFX10-W32-NEXT:    image_store v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-W32-NEXT:    s_wqm_b32 exec_lo, exec_lo
-; GFX10-W32-NEXT:    s_branch .LBB31_2
+; GFX10-W32-NEXT:    s_branch .LBB31_3
 ; GFX10-W32-NEXT:    .p2align 6
 ; GFX10-W32-NEXT:  .LBB31_1: ; %body
-; GFX10-W32-NEXT:    ; in Loop: Header=BB31_2 Depth=1
+; GFX10-W32-NEXT:    ; in Loop: Header=BB31_3 Depth=1
 ; GFX10-W32-NEXT:    image_sample v[0:3], v4, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D
 ; GFX10-W32-NEXT:    v_add_f32_e32 v8, 2.0, v8
-; GFX10-W32-NEXT:    s_cbranch_execz .LBB31_4
-; GFX10-W32-NEXT:  .LBB31_2: ; %loop
+; GFX10-W32-NEXT:    s_mov_b32 s1, 0
+; GFX10-W32-NEXT:  .LBB31_2: ; %Flow
+; GFX10-W32-NEXT:    ; in Loop: Header=BB31_3 Depth=1
+; GFX10-W32-NEXT:    s_and_b32 s1, s1, exec_lo
+; GFX10-W32-NEXT:    s_cselect_b32 s1, 0, 1
+; GFX10-W32-NEXT:    s_cmp_lg_u32 s1, 1
+; GFX10-W32-NEXT:    s_cbranch_scc0 .LBB31_5
+; GFX10-W32-NEXT:  .LBB31_3: ; %loop
 ; GFX10-W32-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-W32-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 0x40e00000, v8
 ; GFX10-W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v7, v3
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-W32-NEXT:    s_cbranch_vccz .LBB31_1
-; GFX10-W32-NEXT:  ; %bb.3:
+; GFX10-W32-NEXT:  ; %bb.4: ; in Loop: Header=BB31_3 Depth=1
+; GFX10-W32-NEXT:    s_mov_b32 s1, -1
 ; GFX10-W32-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX10-W32-NEXT:    ; implicit-def: $vgpr8
-; GFX10-W32-NEXT:  .LBB31_4: ; %break
+; GFX10-W32-NEXT:    s_branch .LBB31_2
+; GFX10-W32-NEXT:  .LBB31_5: ; %break
 ; GFX10-W32-NEXT:    s_and_b32 exec_lo, exec_lo, s0
 ; GFX10-W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX10-W32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-W32-NEXT:    ; return to shader part epilog

 ; GFX9-W64-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX9-W64-NEXT:    s_wqm_b64 exec, exec
 ; GFX9-W64-NEXT:    s_cmp_lt_i32 s0, 1
 ; GFX9-W64-NEXT:    s_cbranch_scc0 .LBB35_2
 ; GFX9-W64-NEXT:  ; %bb.1: ; %else
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX9-W64-NEXT:    image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf
-; GFX9-W64-NEXT:    s_cbranch_execz .LBB35_3
-; GFX9-W64-NEXT:    s_branch .LBB35_4
+; GFX9-W64-NEXT:    s_mov_b64 s[0:1], 0
+; GFX9-W64-NEXT:    s_branch .LBB35_3
 ; GFX9-W64-NEXT:  .LBB35_2:
+; GFX9-W64-NEXT:    s_mov_b64 s[0:1], -1
 ; GFX9-W64-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-W64-NEXT:  .LBB35_3: ; %if
+; GFX9-W64-NEXT:  .LBB35_3: ; %Flow
+; GFX9-W64-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-W64-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX9-W64-NEXT:    s_cmp_lg_u32 s0, 1
+; GFX9-W64-NEXT:    s_cbranch_scc1 .LBB35_5
+; GFX9-W64-NEXT:  ; %bb.4: ; %if
 ; GFX9-W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-W64-NEXT:    image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf
-; GFX9-W64-NEXT:  .LBB35_4: ; %end
+; GFX9-W64-NEXT:  .LBB35_5: ; %end
 ; GFX9-W64-NEXT:    s_and_b64 exec, exec, s[2:3]
 ; GFX9-W64-NEXT:    v_mov_b32_e32 v5, 1.0
 ; GFX9-W64-NEXT:    buffer_store_dword v5, v4, s[0:3], 0 idxen
 ; GFX9-W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-W64-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-W32-LABEL: test_scc:
 ; GFX10-W32:       ; %bb.0: ; %main_body
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-W32-NEXT:    s_mov_b32 s1, exec_lo
 ; GFX10-W32-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-W32-NEXT:    s_cmp_lt_i32 s0, 1
 ; GFX10-W32-NEXT:    s_cbranch_scc0 .LBB35_2
 ; GFX10-W32-NEXT:  ; %bb.1: ; %else
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX10-W32-NEXT:    image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX10-W32-NEXT:    s_cbranch_execz .LBB35_3
-; GFX10-W32-NEXT:    s_branch .LBB35_4
+; GFX10-W32-NEXT:    s_mov_b32 s0, 0
+; GFX10-W32-NEXT:    s_branch .LBB35_3
 ; GFX10-W32-NEXT:  .LBB35_2:
+; GFX10-W32-NEXT:    s_mov_b32 s0, -1
 ; GFX10-W32-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX10-W32-NEXT:  .LBB35_3: ; %if
+; GFX10-W32-NEXT:  .LBB35_3: ; %Flow
+; GFX10-W32-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX10-W32-NEXT:    s_cselect_b32 s0, 0, 1
+; GFX10-W32-NEXT:    s_cmp_lg_u32 s0, 1
+; GFX10-W32-NEXT:    s_cbranch_scc1 .LBB35_5
+; GFX10-W32-NEXT:  ; %bb.4: ; %if
 ; GFX10-W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-W32-NEXT:    image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-W32-NEXT:  .LBB35_4: ; %end
+; GFX10-W32-NEXT:  .LBB35_5: ; %end
 ; GFX10-W32-NEXT:    s_and_b32 exec_lo, exec_lo, s1
 ; GFX10-W32-NEXT:    v_mov_b32_e32 v5, 1.0
 ; GFX10-W32-NEXT:    buffer_store_dword v5, v4, s[0:3], 0 idxen
 ; GFX10-W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-W32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-W32-NEXT:    ; return to shader part epilog
 main_body:
   %cc = icmp sgt i32 %sel, 0