AMDGPU: Add additional MIR tests for exec mask optimizations

[arsenm]

Summary

Also includes one example of how this transform is unsound. This isn't
verifying the copies are used in the control flow intrinisic patterns.

Also add option to disable exec mask opt pass Since this pass is
unsound, it may be useful to turn it off until it is fixed.

[rampitec]

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

890–895

This should not what when option is disabled.

[arsenm]

Change insert point for memory clause when disabled

[rampitec]

LGTM

[arsenm]

r357091

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

static cl::opt<bool> EnableSROA(
   cl::ReallyHidden,
   cl::init(true));
 
 static cl::opt<bool>
 EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
                         cl::desc("Run early if-conversion"),
                         cl::init(false));
 
+static cl::opt<bool>
+OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
+            cl::desc("Run pre-RA exec mask optimizations"),
+            cl::init(true));
+
 static cl::opt<bool> EnableR600IfConvert(
   "r600-if-convert",
   cl::desc("Use if conversion pass"),
   cl::ReallyHidden,
   cl::init(true));
 
 // Option to disable vectorizer for tests.
 static cl::opt<bool> EnableLoadStoreVectorizer(
void GCNPassConfig::addFastRegAlloc() {
   // This must be run after SILowerControlFlow, since it needs to use the
   // machine-level CFG, but before register allocation.
   insertPass(&SILowerControlFlowID, &SIFixWWMLivenessID, false);
 
   TargetPassConfig::addFastRegAlloc();
 }
 
 void GCNPassConfig::addOptimizedRegAlloc() {
+  if (OptExecMaskPreRA) {
     insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID);
-
     insertPass(&SIOptimizeExecMaskingPreRAID, &SIFormMemoryClausesID);
+  } else {
+    insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
+  }
 
   // This must be run immediately after phi elimination and before
   // TwoAddressInstructions, otherwise the processing of the tied operand of
   // SI_ELSE will introduce a copy of the tied operand source after the else.
   insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
 
   // This must be run after SILowerControlFlow, since it needs to use the
   // machine-level CFG, but before register allocation.

test/CodeGen/AMDGPU/collapse-endcf-broken.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass=si-optimize-exec-masking-pre-ra %s -o - | FileCheck -check-prefix=GXN %s
+
+# FIXME: This is a miscompile, and the s_or_b64s need to be preserved.
+
+---
+name: invalid_end_cf_fold_0
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GXN-LABEL: name: invalid_end_cf_fold_0
+  ; GXN: bb.0:
+  ; GXN:   successors: %bb.1(0x80000000)
+  ; GXN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GXN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GXN:   $exec = S_OR_B64 $exec, [[COPY]], implicit-def $scc
+  ; GXN:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $exec
+  ; GXN: bb.1:
+  ; GXN:   successors: %bb.2(0x80000000)
+  ; GXN: bb.2:
+  ; GXN:   $exec = S_OR_B64 $exec, [[COPY1]], implicit-def $scc
+  ; GXN:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GXN:   [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; GXN:   DS_WRITE_B32 [[DEF]], [[DEF1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GXN:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %0:sgpr_64 = COPY $sgpr0_sgpr1
+    %1:sgpr_64 = COPY $exec
+    $exec = S_OR_B64 $exec, %0, implicit-def $scc
+    %2:sgpr_64 = COPY $exec
+
+  bb.1:
+    $exec = S_OR_B64 $exec, %1, implicit-def $scc
+
+  bb.2:
+    $exec = S_OR_B64 $exec, %2, implicit-def $scc
+
+    %5:vgpr_32 = IMPLICIT_DEF
+    %6:vgpr_32 = IMPLICIT_DEF
+    DS_WRITE_B32 %5, %6, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...

test/CodeGen/AMDGPU/collapse-endcf.ll

-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,ALL %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-opt-exec-mask-pre-ra=0 < %s | FileCheck -enable-var-scope -check-prefixes=DISABLED,ALL %s
 
-; GCN-LABEL: {{^}}simple_nested_if:
+; ALL-LABEL: {{^}}simple_nested_if:
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9_]+]]
 ; GCN-NEXT: s_cbranch_execz [[ENDIF]]
 ; GCN:      s_and_b64 exec, exec, vcc
 ; GCN-NEXT: ; mask branch [[ENDIF]]
 ; GCN-NEXT: {{^BB[0-9_]+}}:
 ; GCN:      store_dword
 ; GCN-NEXT: {{^}}[[ENDIF]]:
 ; GCN-NEXT: s_or_b64 exec, exec, [[SAVEEXEC]]
 ; GCN: ds_write_b32
 ; GCN: s_endpgm
+
+
+; DISABLED: s_or_b64 exec, exec
+; DISABLED: s_or_b64 exec, exec
 define amdgpu_kernel void @simple_nested_if(i32 addrspace(1)* nocapture %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = icmp ugt i32 %tmp, 1
   br i1 %tmp1, label %bb.outer.then, label %bb.outer.end
 
 bb.outer.then:                                    ; preds = %bb
   %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
   store i32 0, i32 addrspace(1)* %tmp4, align 4
   %tmp5 = icmp eq i32 %tmp, 2
   br i1 %tmp5, label %bb.outer.end, label %bb.inner.then
 
 bb.inner.then:                                    ; preds = %bb.outer.then
   %tmp7 = add i32 %tmp, 1
   %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp7
   store i32 1, i32 addrspace(1)* %tmp9, align 4
   br label %bb.outer.end
 
 bb.outer.end:                                     ; preds = %bb.outer.then, %bb.inner.then, %bb
   store i32 3, i32 addrspace(3)* null
   ret void
 }
 
-; GCN-LABEL: {{^}}uncollapsable_nested_if:
+; ALL-LABEL: {{^}}uncollapsable_nested_if:
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[ENDIF_OUTER:BB[0-9_]+]]
 ; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER]]
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC_INNER:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[ENDIF_INNER:BB[0-9_]+]]
 ; GCN-NEXT: {{^BB[0-9_]+}}:
 ; GCN:      store_dword
 ; GCN-NEXT: {{^}}[[ENDIF_INNER]]:
bb.inner.end:                                     ; preds = %bb.inner.then, %bb.outer.then
   store i32 2, i32 addrspace(1)* %tmp10, align 4
   br label %bb.outer.end
 
 bb.outer.end:                                     ; preds = %bb.inner.then, %bb
   store i32 3, i32 addrspace(3)* null
   ret void
 }
 
-; GCN-LABEL: {{^}}nested_if_if_else:
+; ALL-LABEL: {{^}}nested_if_if_else:
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[ENDIF_OUTER:BB[0-9_]+]]
 ; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER]]
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC_INNER:s\[[0-9:]+\]]]
 ; GCN-NEXT: s_xor_b64 [[SAVEEXEC_INNER2:s\[[0-9:]+\]]], exec, [[SAVEEXEC_INNER]]
 ; GCN-NEXT: ; mask branch [[THEN_INNER:BB[0-9_]+]]
 ; GCN-NEXT: {{^BB[0-9_]+}}:
 ; GCN:      store_dword
bb.else:                                             ; preds = %bb.outer.then
   store i32 2, i32 addrspace(1)* %tmp9, align 4
   br label %bb.outer.end
 
 bb.outer.end:                                        ; preds = %bb, %bb.then, %bb.else
   store i32 3, i32 addrspace(3)* null
   ret void
 }
 
-; GCN-LABEL: {{^}}nested_if_else_if:
+; ALL-LABEL: {{^}}nested_if_else_if:
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
 ; GCN-NEXT: s_xor_b64 [[SAVEEXEC_OUTER2:s\[[0-9:]+\]]], exec, [[SAVEEXEC_OUTER]]
 ; GCN-NEXT: ; mask branch [[THEN_OUTER:BB[0-9_]+]]
 ; GCN-NEXT: s_cbranch_execz [[THEN_OUTER]]
 ; GCN-NEXT: {{^BB[0-9_]+}}:
 ; GCN:      store_dword
 ; GCN-NEXT: s_and_saveexec_b64 [[SAVEEXEC_INNER_IF_OUTER_ELSE:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[THEN_OUTER_FLOW:BB[0-9_]+]]
bb.inner.then2:
   store i32 4, i32 addrspace(1)* %tmp5, align 4
   br label %bb.outer.end
 
 bb.outer.end:
   store i32 3, i32 addrspace(3)* null
   ret void
 }
 
-; GCN-LABEL: {{^}}s_endpgm_unsafe_barrier:
+; ALL-LABEL: {{^}}s_endpgm_unsafe_barrier:
 ; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]]
 ; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9_]+]]
 ; GCN-NEXT: {{^BB[0-9_]+}}:
 ; GCN:      store_dword
 ; GCN-NEXT: {{^}}[[ENDIF]]:
 ; GCN-NEXT: s_or_b64 exec, exec, [[SAVEEXEC]]
 ; GCN:      s_barrier
 ; GCN-NEXT: s_endpgm
bb.then:                                          ; preds = %bb
   br label %bb.end
 
 bb.end:                                           ; preds = %bb.then, %bb
   call void @llvm.amdgcn.s.barrier()
   ret void
 }
 
 ; Make sure scc liveness is updated if sor_b64 is removed
-; GCN-LABEL: {{^}}scc_liveness:
+; ALL-LABEL: {{^}}scc_liveness:
 
 ; GCN: [[BB1_LOOP:BB[0-9]+_[0-9]+]]:
 ; GCN: s_andn2_b64 exec, exec,
 ; GCN-NEXT: s_cbranch_execnz [[BB1_LOOP]]
 
 ; GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen
 ; GCN: s_and_b64 exec, exec, {{vcc|s\[[0-9:]+\]}}
 

test/CodeGen/AMDGPU/collapse-endcf.mir

bb.4:
     $exec = S_OR_B64 $exec, %3, implicit-def $scc
     %15:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
     %16:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     $m0 = S_MOV_B32 -1
     DS_WRITE_B32 %16, %15, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
     S_ENDPGM 0
 
 ...
+
+# Empty block separates the collapsable s_or_b64
+---
+name: simple_nested_if_empty_block_between
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: simple_nested_if_empty_block_between
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.5(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.5, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN: bb.4:
+  ; GCN:   successors: %bb.5(0x80000000)
+  ; GCN: bb.5:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    $exec = S_OR_B64 $exec, %12, implicit-def $scc
+
+  bb.5:
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %15:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %16:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %16, %15, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+
+# Effectively empty block separates the collapsable s_or_b64
+---
+name: simple_nested_if_empty_block_dbg_between
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: simple_nested_if_empty_block_dbg_between
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.5(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.5, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN: bb.4:
+  ; GCN:   successors: %bb.5(0x80000000)
+  ; GCN:   DBG_VALUE
+  ; GCN: bb.5:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    $exec = S_OR_B64 $exec, %12, implicit-def $scc
+
+  bb.5:
+    DBG_VALUE
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %15:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %16:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %16, %15, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+
+---
+name: skip_salu_and_meta_insts_find_first
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: skip_salu_and_meta_insts_find_first
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.4(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.4, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GCN:   dead %16:sgpr_32 = S_BREV_B32 [[DEF]]
+  ; GCN:   KILL [[DEF]]
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN: bb.4:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    %15:sgpr_32 = IMPLICIT_DEF
+    %16:sgpr_32 = S_BREV_B32 %15
+    KILL %15
+    $exec = S_OR_B64 $exec, %12, implicit-def $scc
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %17:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %18, %17, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+
+# Make sure SALU instructions, meta instructions, and SGPR->SGPR
+# copies are skipped.
+---
+name: skip_salu_and_meta_insts_after
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: skip_salu_and_meta_insts_after
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.4(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.4, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GCN:   [[S_BREV_B32_:%[0-9]+]]:sgpr_32 = S_BREV_B32 [[DEF]]
+  ; GCN:   KILL [[DEF]]
+  ; GCN:   dead %17:sgpr_32 = COPY [[S_BREV_B32_]]
+  ; GCN: bb.4:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    $exec = S_OR_B64 $exec, %12, implicit-def $scc
+    %15:sgpr_32 = IMPLICIT_DEF
+    %16:sgpr_32 = S_BREV_B32 %15
+    KILL %15
+    %19:sgpr_32 = COPY %16
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %17:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %18, %17, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+
+# SALU instruction depends on exec through a normal operand.
+---
+name: salu_exec_dependency
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: salu_exec_dependency
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.4(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.4, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN:   dead %15:sreg_64 = S_BREV_B64 $exec
+  ; GCN: bb.4:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    $exec = S_OR_B64 $exec, %12,  implicit-def $scc
+    %15:sreg_64 = S_BREV_B64 $exec
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %17:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %18, %17, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+
+# Copy to / from VGPR should be assumed to read exec
+---
+name: copy_no_explicit_exec_dependency
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GCN-LABEL: name: copy_no_explicit_exec_dependency
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.4(0x40000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_]]
+  ; GCN:   SI_MASK_BRANCH %bb.4, implicit $exec
+  ; GCN:   S_BRANCH %bb.1
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; GCN:   undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+  ; GCN:   %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+  ; GCN:   undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+  ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+  ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
+  ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+  ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+  ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
+  ; GCN:   SI_MASK_BRANCH %bb.3, implicit $exec
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.2:
+  ; GCN:   successors: %bb.3(0x80000000)
+  ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
+  ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
+  ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x80000000)
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+  ; GCN:   dead %15:vgpr_32 = COPY %5.sub2
+  ; GCN: bb.4:
+  ; GCN:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+  ; GCN:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+  ; GCN:   [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GCN:   $m0 = S_MOV_B32 -1
+  ; GCN:   DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.4
+    liveins: $vgpr0, $sgpr0_sgpr1
+
+    %1:sgpr_64 = COPY $sgpr0_sgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+    %3:sreg_64 = COPY $exec, implicit-def $exec
+    %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+    $exec = S_MOV_B64_term %4
+    SI_MASK_BRANCH %bb.4, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+    undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+    %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %7:vgpr_32 = COPY %5.sub1
+    undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+    %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+    %5.sub3:sgpr_128 = S_MOV_B32 61440
+    %5.sub2:sgpr_128 = S_MOV_B32 0
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+    %12:sreg_64 = COPY $exec, implicit-def $exec
+    %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+    $exec = S_MOV_B64_term %13
+    SI_MASK_BRANCH %bb.3, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5.sub0:sgpr_128 = COPY %5.sub2
+    %5.sub1:sgpr_128 = COPY %5.sub2
+    %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+  bb.3:
+    $exec = S_OR_B64 $exec, %12,  implicit-def $scc
+    %15:vgpr_32 = COPY %5.sub2
+
+  bb.4:
+    $exec = S_OR_B64 $exec, %3, implicit-def $scc
+    %17:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    $m0 = S_MOV_B32 -1
+    DS_WRITE_B32 %18, %17, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+    S_ENDPGM 0
+
+...
+