[AMDGPU] Enable structurizer workarounds by default

[sameerds]

Summary

Added some reviewers based on the git log of tests that are affected by this change. The change itself is a one-liner that simply switches the default value of a command-line option in the backend. But it affects control flow and each of the four affected tests needs to be checked. It is possible that some tests become redundant due to the way the control flow is now organized.

[cfang]

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

197

Can we come out with a name that is closer to what was implemented, like amdgpu-unify-loop-exits, or even amdgpu-normalize-loops(cfg)? "workarounds" is kind of misleading.

[sameerds]

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

197

This one option controls more than one pass. The intention of naming this option is to indicate to the user that additional things are being done to work around known structurizer limitations. Each of these pass is used to avoid a specific situation where the structurizer is know to fail. We could have one option per pass like the names you suggest, but that sounds like overkill. We might even add more passes to the set of workarounds in the future.

[nhaehnle]

LGTM

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

 
 static cl::opt<bool> EnableScalarIRPasses(
   "amdgpu-scalar-ir-passes",
   cl::desc("Enable scalar IR passes"),
   cl::init(true),
   cl::Hidden);
 
 static cl::opt<bool> EnableStructurizerWorkarounds(
     "amdgpu-enable-structurizer-workarounds",
-    cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(false),
+    cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true),
     cl::Hidden);
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
 
   PassRegistry *PR = PassRegistry::getPassRegistry();

llvm/test/CodeGen/AMDGPU/infinite-loop.ll

 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_branch BB0_1
 ; IR-LABEL: @infinite_loop(
 ; IR-NEXT:  entry:
 ; IR-NEXT:    br label [[LOOP:%.*]]
 ; IR:       loop:
 ; IR-NEXT:    store volatile i32 999, i32 addrspace(1)* [[OUT:%.*]], align 4
 ; IR-NEXT:    br label [[LOOP]]
-;
 entry:
   br label %loop
 
 loop:
   store volatile i32 999, i32 addrspace(1)* %out, align 4
   br label %loop
 }
 

 ; IR-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; IR-NEXT:    [[COND:%.*]] = icmp eq i32 [[TMP]], 1
 ; IR-NEXT:    br i1 [[COND]], label [[LOOP:%.*]], label [[UNIFIEDRETURNBLOCK:%.*]]
 ; IR:       loop:
 ; IR-NEXT:    store volatile i32 999, i32 addrspace(1)* [[OUT:%.*]], align 4
 ; IR-NEXT:    br i1 true, label [[LOOP]], label [[UNIFIEDRETURNBLOCK]]
 ; IR:       UnifiedReturnBlock:
 ; IR-NEXT:    ret void
-;
 entry:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %cond = icmp eq i32 %tmp, 1
   br i1 %cond, label %loop, label %return
 
 loop:
   store volatile i32 999, i32 addrspace(1)* %out, align 4
   br label %loop

 ; IR:       loop1:
 ; IR-NEXT:    store volatile i32 999, i32 addrspace(1)* [[OUT:%.*]], align 4
 ; IR-NEXT:    br i1 true, label [[LOOP1]], label [[DUMMYRETURNBLOCK:%.*]]
 ; IR:       loop2:
 ; IR-NEXT:    store volatile i32 888, i32 addrspace(1)* [[OUT]], align 4
 ; IR-NEXT:    br i1 true, label [[LOOP2]], label [[DUMMYRETURNBLOCK]]
 ; IR:       DummyReturnBlock:
 ; IR-NEXT:    ret void
-;
 entry:
   br i1 undef, label %loop1, label %loop2
 
 loop1:
   store volatile i32 999, i32 addrspace(1)* %out, align 4
   br label %loop1
 
 loop2:
   store volatile i32 888, i32 addrspace(1)* %out, align 4
   br label %loop2
 }
 
 define amdgpu_kernel void @infinite_loop_nest_ret(i32 addrspace(1)* %out) {
 ; SI-LABEL: infinite_loop_nest_ret:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; SI-NEXT:    s_cbranch_execz BB3_5
 ; SI-NEXT:  ; %bb.1: ; %outer_loop.preheader
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
 ; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], 3, v0
-; SI-NEXT:    s_mov_b64 s[2:3], 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:  BB3_2: ; %outer_loop
 ; SI-NEXT:    ; =>This Loop Header: Depth=1
 ; SI-NEXT:    ; Child Loop BB3_3 Depth 2
-; SI-NEXT:    s_and_b64 s[8:9], exec, vcc
-; SI-NEXT:    s_or_b64 s[2:3], s[8:9], s[2:3]
-; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:    s_mov_b64 s[2:3], 0
 ; SI-NEXT:  BB3_3: ; %inner_loop
 ; SI-NEXT:    ; Parent Loop BB3_2 Depth=1
 ; SI-NEXT:    ; => This Inner Loop Header: Depth=2
-; SI-NEXT:    s_and_b64 s[10:11], exec, s[0:1]
-; SI-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; SI-NEXT:    s_and_b64 s[8:9], exec, s[0:1]
+; SI-NEXT:    s_or_b64 s[2:3], s[8:9], s[2:3]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, 0x3e7
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; SI-NEXT:    s_cbranch_execnz BB3_3
-; SI-NEXT:  ; %bb.4: ; %Flow
+; SI-NEXT:  ; %bb.4: ; %loop.exit.guard
 ; SI-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; SI-NEXT:    s_cbranch_execnz BB3_2
+; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_and_b64 vcc, exec, 0
+; SI-NEXT:    s_cbranch_vccz BB3_2
 ; SI-NEXT:  BB3_5: ; %UnifiedReturnBlock
 ; SI-NEXT:    s_endpgm
 ; IR-LABEL: @infinite_loop_nest_ret(
 ; IR-NEXT:  entry:
 ; IR-NEXT:    [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
 ; IR-NEXT:    [[COND1:%.*]] = icmp eq i32 [[TMP]], 1
 ; IR-NEXT:    br i1 [[COND1]], label [[OUTER_LOOP:%.*]], label [[UNIFIEDRETURNBLOCK:%.*]]
 ; IR:       outer_loop:
 ; IR-NEXT:    br label [[INNER_LOOP:%.*]]
 ; IR:       inner_loop:
 ; IR-NEXT:    store volatile i32 999, i32 addrspace(1)* [[OUT:%.*]], align 4
 ; IR-NEXT:    [[COND3:%.*]] = icmp eq i32 [[TMP]], 3
 ; IR-NEXT:    br i1 true, label [[TRANSITIONBLOCK:%.*]], label [[UNIFIEDRETURNBLOCK]]
 ; IR:       TransitionBlock:
 ; IR-NEXT:    br i1 [[COND3]], label [[INNER_LOOP]], label [[OUTER_LOOP]]
 ; IR:       UnifiedReturnBlock:
 ; IR-NEXT:    ret void
-;
 entry:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %cond1 = icmp eq i32 %tmp, 1
   br i1 %cond1, label %outer_loop, label %return
 
 outer_loop:
   ; %cond2 = icmp eq i32 %tmp, 2
   ; br i1 %cond2, label %outer_loop, label %inner_loop

llvm/test/CodeGen/AMDGPU/multilevel-break.ll

 ; OPT-NEXT:    ret void
 ; OPT:       ENDIF:
 ; OPT-NEXT:    [[TMP51]] = icmp eq i32 [[TMP47]], [[CONT:%.*]]
 ; OPT-NEXT:    [[TMP11]] = xor i1 [[TMP51]], true
 ; OPT-NEXT:    br label [[FLOW]]
 ;
 ; GCN-LABEL: multi_else_break:
 ; GCN:       ; %bb.0: ; %main_body
-; GCN-NEXT:    s_mov_b64 s[2:3], 0
+; GCN-NEXT:    s_mov_b64 s[0:1], 0
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_branch BB0_2
-; GCN-NEXT:  BB0_1: ; %Flow2
+; GCN-NEXT:  BB0_1: ; %loop.exit.guard
 ; GCN-NEXT:    ; in Loop: Header=BB0_2 Depth=1
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_and_b64 s[0:1], exec, s[8:9]
-; GCN-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
-; GCN-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; GCN-NEXT:    s_and_b64 s[2:3], exec, s[2:3]
+; GCN-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GCN-NEXT:    s_cbranch_execz BB0_6
 ; GCN-NEXT:  BB0_2: ; %LOOP.outer
 ; GCN-NEXT:    ; =>This Loop Header: Depth=1
 ; GCN-NEXT:    ; Child Loop BB0_4 Depth 2
 ; GCN-NEXT:    ; implicit-def: $sgpr6_sgpr7
-; GCN-NEXT:    ; implicit-def: $sgpr8_sgpr9
+; GCN-NEXT:    ; implicit-def: $sgpr2_sgpr3
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-NEXT:    v_mov_b32_e32 v1, v0
 ; GCN-NEXT:    s_branch BB0_4
 ; GCN-NEXT:  BB0_3: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
-; GCN-NEXT:    s_or_b64 exec, exec, s[10:11]
-; GCN-NEXT:    s_and_b64 s[0:1], exec, s[6:7]
-; GCN-NEXT:    s_or_b64 s[4:5], s[0:1], s[4:5]
+; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GCN-NEXT:    s_and_b64 s[8:9], exec, s[6:7]
+; GCN-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execz BB0_1
 ; GCN-NEXT:  BB0_4: ; %LOOP
 ; GCN-NEXT:    ; Parent Loop BB0_2 Depth=1
 ; GCN-NEXT:    ; => This Inner Loop Header: Depth=2
-; GCN-NEXT:    v_mov_b32_e32 v2, v1
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, 1, v2
-; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, v2, v4
-; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], exec
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
+; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, v1, v4
+; GCN-NEXT:    s_or_b64 s[2:3], s[2:3], exec
 ; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], exec
-; GCN-NEXT:    s_and_saveexec_b64 s[10:11], vcc
+; GCN-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; GCN-NEXT:    s_cbranch_execz BB0_3
 ; GCN-NEXT:  ; %bb.5: ; %ENDIF
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v1
-; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v1
-; GCN-NEXT:    s_andn2_b64 s[8:9], s[8:9], exec
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v0
+; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
 ; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
-; GCN-NEXT:    s_and_b64 s[12:13], vcc, exec
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
-; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[0:1]
-; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    s_and_b64 s[10:11], vcc, exec
+; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
 ; GCN-NEXT:    s_branch BB0_3
 ; GCN-NEXT:  BB0_6: ; %IF
 ; GCN-NEXT:    s_endpgm
 main_body:
   br label %LOOP.outer
 
 LOOP.outer:                                       ; preds = %ENDIF, %main_body
   %tmp43 = phi i32 [ 0, %main_body ], [ %tmp47, %ENDIF ]

 ; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GCN-NEXT:    s_cbranch_execz BB1_9
 ; GCN-NEXT:  BB1_2: ; %bb1
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v1
+; GCN-NEXT:    s_mov_b64 s[6:7], -1
 ; GCN-NEXT:    s_and_b64 vcc, exec, vcc
+; GCN-NEXT:    ; implicit-def: $sgpr8_sgpr9
+; GCN-NEXT:    s_mov_b64 s[10:11], -1
 ; GCN-NEXT:    s_cbranch_vccnz BB1_6
 ; GCN-NEXT:  ; %bb.3: ; %LeafBlock1
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_mov_b64 s[6:7], -1
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
 ; GCN-NEXT:    s_and_b64 vcc, exec, vcc
 ; GCN-NEXT:    s_mov_b64 s[8:9], -1
 ; GCN-NEXT:    s_cbranch_vccz BB1_5
 ; GCN-NEXT:  ; %bb.4: ; %case1
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v2
 ; GCN-NEXT:    s_mov_b64 s[8:9], 0
 ; GCN-NEXT:    s_orn2_b64 s[6:7], vcc, exec
 ; GCN-NEXT:  BB1_5: ; %Flow3
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_mov_b64 s[10:11], 0
+; GCN-NEXT:  BB1_6: ; %Flow
+; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_and_b64 vcc, exec, s[10:11]
 ; GCN-NEXT:    s_cbranch_vccz BB1_1
-; GCN-NEXT:    s_branch BB1_7
-; GCN-NEXT:  BB1_6: ; in Loop: Header=BB1_2 Depth=1
-; GCN-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-NEXT:    s_mov_b64 s[6:7], -1
-; GCN-NEXT:    s_and_b64 vcc, exec, -1
-; GCN-NEXT:    s_cbranch_execz BB1_1
-; GCN-NEXT:  BB1_7: ; %LeafBlock
+; GCN-NEXT:  ; %bb.7: ; %LeafBlock
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GCN-NEXT:    s_and_b64 vcc, exec, vcc
 ; GCN-NEXT:    s_mov_b64 s[8:9], -1
 ; GCN-NEXT:    s_cbranch_vccz BB1_1
 ; GCN-NEXT:  ; %bb.8: ; %case0
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0
 ; GCN-NEXT:    s_mov_b64 s[8:9], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v1
 ; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
 ; GCN-NEXT:    s_and_b64 s[10:11], vcc, exec
 ; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
 ; GCN-NEXT:    s_branch BB1_1
-; GCN-NEXT:  BB1_9: ; %Flow6
+; GCN-NEXT:  BB1_9: ; %loop.exit.guard
 ; GCN-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GCN-NEXT:    s_and_saveexec_b64 s[0:1], s[4:5]
+; GCN-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GCN-NEXT:    s_endpgm
 bb:
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp = sub i32 %id, %arg
   br label %bb1
 
 bb1:
   %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ]

llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow %s | FileCheck -check-prefix=IR %s
 ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; After structurizing, there are 3 levels of loops. The i1 phi
 ; conditions mutually depend on each other, so it isn't safe to delete
 ; the condition that appears to have no uses until the loop is
 ; completely processed.
 
 define amdgpu_kernel void @reduced_nested_loop_conditions(i64 addrspace(3)* nocapture %arg) #0 {
 ; GCN-LABEL: reduced_nested_loop_conditions:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_load_dword s0, s[0:1], 0x9
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GCN-NEXT:    s_mov_b32 m0, -1
+; GCN-NEXT:    s_mov_b64 s[2:3], -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    ds_read_b64 v[0:1], v0
-; GCN-NEXT:    s_mov_b32 s0, 0
-; GCN-NEXT:    s_and_b64 vcc, exec, 0
-; GCN-NEXT:  BB0_1: ; %bb5
+; GCN-NEXT:    s_and_b64 s[0:1], exec, -1
+; GCN-NEXT:    s_branch BB0_2
+; GCN-NEXT:  BB0_1: ; %bb10
+; GCN-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; GCN-NEXT:    s_cbranch_vccz BB0_4
+; GCN-NEXT:  BB0_2: ; %bb5
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_cmp_lg_u32 s0, 1
-; GCN-NEXT:    s_cbranch_scc0 BB0_3
-; GCN-NEXT:  ; %bb.2: ; %bb10
-; GCN-NEXT:    ; in Loop: Header=BB0_1 Depth=1
-; GCN-NEXT:    ; implicit-def: $sgpr0
+; GCN-NEXT:    s_mov_b64 vcc, s[0:1]
 ; GCN-NEXT:    s_cbranch_vccnz BB0_1
-; GCN-NEXT:    s_branch BB0_5
-; GCN-NEXT:  BB0_3: ; %bb8
+; GCN-NEXT:  ; %bb.3: ; in Loop: Header=BB0_2 Depth=1
+; GCN-NEXT:    s_mov_b64 s[4:5], -1
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; GCN-NEXT:    s_cbranch_vccnz BB0_2
+; GCN-NEXT:  BB0_4: ; %loop.exit.guard
+; GCN-NEXT:    s_and_b64 vcc, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_vccz BB0_7
+; GCN-NEXT:  ; %bb.5: ; %bb8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    ds_read_b32 v0, v0
 ; GCN-NEXT:    s_and_b64 vcc, exec, 0
-; GCN-NEXT:  BB0_4: ; %bb9
+; GCN-NEXT:  BB0_6: ; %bb9
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_cbranch_vccz BB0_4
-; GCN-NEXT:  BB0_5: ; %DummyReturnBlock
+; GCN-NEXT:    s_cbranch_vccz BB0_6
+; GCN-NEXT:  BB0_7: ; %DummyReturnBlock
 ; GCN-NEXT:    s_endpgm
 ; IR-LABEL: @reduced_nested_loop_conditions(
 ; IR-NEXT:  bb:
 ; IR-NEXT:    [[MY_TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #4
 ; IR-NEXT:    [[MY_TMP1:%.*]] = getelementptr inbounds i64, i64 addrspace(3)* [[ARG:%.*]], i32 [[MY_TMP]]
 ; IR-NEXT:    [[MY_TMP2:%.*]] = load volatile i64, i64 addrspace(3)* [[MY_TMP1]]
 ; IR-NEXT:    br label [[BB5:%.*]]
 ; IR:       bb3:

 ; IR-NEXT:    br label [[BB20]]
 ; IR:       bb20:
 ; IR-NEXT:    [[MY_TMP21]] = phi i32 [ [[MY_TMP19]], [[BB16]] ], [ 0, [[BB13]] ]
 ; IR-NEXT:    [[MY_TMP22]] = phi i1 [ false, [[BB16]] ], [ [[MY_TMP14]], [[BB13]] ]
 ; IR-NEXT:    br label [[BB9]]
 ; IR:       bb23:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
 ; IR-NEXT:    ret void
-;
 bb:
   %my.tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %my.tmp1 = getelementptr inbounds i64, i64 addrspace(3)* %arg, i32 %my.tmp
   %my.tmp2 = load volatile i64, i64 addrspace(3)* %my.tmp1
   br label %bb5
 
 bb3:                                              ; preds = %bb9
   br i1 true, label %bb4, label %bb13

 ; IR-NEXT:    [[TMP21]] = extractvalue { i1, i64 } [[TMP19]], 1
 ; IR-NEXT:    br i1 [[TMP20]], label [[BB31_LOOPEXIT:%.*]], label [[FLOW3]]
 ; IR:       bb31.loopexit:
 ; IR-NEXT:    br label [[FLOW3]]
 ; IR:       bb31:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
 ; IR-NEXT:    store volatile i32 0, i32 addrspace(1)* undef
 ; IR-NEXT:    ret void
-;
 bb:
   %my.tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %my.tmp1 = zext i32 %my.tmp to i64
   %my.tmp2 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 %my.tmp1
   %my.tmp3 = load i64, i64 addrspace(1)* %my.tmp2, align 16
   %my.tmp932 = load <4 x i32>, <4 x i32> addrspace(1)* undef, align 16
   %my.tmp1033 = extractelement <4 x i32> %my.tmp932, i64 0
   %my.tmp1134 = load volatile i32, i32 addrspace(1)* undef

llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll

 
 define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32 %c3, i32 %x, i32 %y, i1 %arg) nounwind {
 ; SI-LABEL: loop_land_info_assert:
 ; SI:       ; %bb.0: ; %entry
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
-; SI-NEXT:    s_load_dword s4, s[0:1], 0xc
-; SI-NEXT:    s_brev_b32 s5, 44
+; SI-NEXT:    s_load_dword s8, s[0:1], 0xc
+; SI-NEXT:    s_brev_b32 s9, 44
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s2, 0
-; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], s3, 4
-; SI-NEXT:    s_or_b64 s[8:9], s[0:1], s[2:3]
-; SI-NEXT:    s_and_b64 s[0:1], exec, s[2:3]
-; SI-NEXT:    s_and_b64 s[2:3], exec, s[8:9]
+; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], s2, 1
+; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], s3, 4
+; SI-NEXT:    v_cmp_gt_i32_e64 s[2:3], s3, 3
+; SI-NEXT:    s_and_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT:    s_and_b64 s[0:1], exec, s[4:5]
+; SI-NEXT:    s_and_b64 s[2:3], exec, s[2:3]
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_lt_f32_e64 s[8:9], |v0|, s5
+; SI-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s9
+; SI-NEXT:    s_and_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    v_mov_b32_e32 v0, 3
-; SI-NEXT:  BB3_1: ; %while.cond
+; SI-NEXT:    s_branch BB3_4
+; SI-NEXT:  BB3_1: ; %Flow6
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_mov_b64 s[10:11], 0
+; SI-NEXT:  BB3_2: ; %Flow5
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_mov_b64 s[14:15], 0
+; SI-NEXT:  BB3_3: ; %Flow
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_and_b64 vcc, exec, s[12:13]
+; SI-NEXT:    s_cbranch_vccnz BB3_8
+; SI-NEXT:  BB3_4: ; %while.cond
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_mov_b64 s[14:15], -1
+; SI-NEXT:    s_mov_b64 s[10:11], -1
+; SI-NEXT:    s_mov_b64 s[12:13], -1
 ; SI-NEXT:    s_mov_b64 vcc, s[0:1]
-; SI-NEXT:    s_cbranch_vccz BB3_5
-; SI-NEXT:  ; %bb.2: ; %convex.exit
-; SI-NEXT:    ; in Loop: Header=BB3_1 Depth=1
+; SI-NEXT:    s_cbranch_vccz BB3_3
+; SI-NEXT:  ; %bb.5: ; %convex.exit
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_mov_b64 s[10:11], -1
+; SI-NEXT:    s_mov_b64 s[12:13], -1
 ; SI-NEXT:    s_mov_b64 vcc, s[2:3]
-; SI-NEXT:    s_cbranch_vccnz BB3_8
-; SI-NEXT:  ; %bb.3: ; %if.end
-; SI-NEXT:    ; in Loop: Header=BB3_1 Depth=1
-; SI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; SI-NEXT:    s_cbranch_vccnz BB3_1
-; SI-NEXT:  ; %bb.4: ; %if.else
-; SI-NEXT:    ; in Loop: Header=BB3_1 Depth=1
+; SI-NEXT:    s_cbranch_vccz BB3_2
+; SI-NEXT:  ; %bb.6: ; %if.end
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_mov_b64 s[12:13], -1
+; SI-NEXT:    s_mov_b64 vcc, s[4:5]
+; SI-NEXT:    s_cbranch_vccz BB3_1
+; SI-NEXT:  ; %bb.7: ; %if.else
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_mov_b64 s[12:13], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_branch BB3_1
-; SI-NEXT:  BB3_5: ; %for.cond.preheader
+; SI-NEXT:  BB3_8: ; %loop.exit.guard4
+; SI-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; SI-NEXT:    s_and_b64 vcc, exec, s[10:11]
+; SI-NEXT:    s_cbranch_vccz BB3_4
+; SI-NEXT:  ; %bb.9: ; %loop.exit.guard
+; SI-NEXT:    s_and_b64 vcc, exec, s[14:15]
+; SI-NEXT:    s_cbranch_vccz BB3_13
+; SI-NEXT:  ; %bb.10: ; %for.cond.preheader
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, 0x3e8
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s4, v0
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s8, v0
 ; SI-NEXT:    s_and_b64 vcc, exec, vcc
-; SI-NEXT:    s_cbranch_vccz BB3_8
-; SI-NEXT:  ; %bb.6: ; %for.body
+; SI-NEXT:    s_cbranch_vccz BB3_13
+; SI-NEXT:  ; %bb.11: ; %for.body
 ; SI-NEXT:    s_and_b64 vcc, exec, 0
-; SI-NEXT:  BB3_7: ; %self.loop
+; SI-NEXT:  BB3_12: ; %self.loop
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_cbranch_vccz BB3_7
-; SI-NEXT:  BB3_8: ; %DummyReturnBlock
+; SI-NEXT:    s_cbranch_vccz BB3_12
+; SI-NEXT:  BB3_13: ; %DummyReturnBlock
 ; SI-NEXT:    s_endpgm
 ;
 ; FLAT-LABEL: loop_land_info_assert:
 ; FLAT:       ; %bb.0: ; %entry
 ; FLAT-NEXT:    s_mov_b32 s7, 0xf000
 ; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; FLAT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; FLAT-NEXT:    s_load_dword s4, s[0:1], 0x30
-; FLAT-NEXT:    s_brev_b32 s5, 44
+; FLAT-NEXT:    s_load_dword s8, s[0:1], 0x30
+; FLAT-NEXT:    s_brev_b32 s9, 44
 ; FLAT-NEXT:    s_waitcnt lgkmcnt(0)
-; FLAT-NEXT:    v_cmp_gt_i32_e64 s[0:1], s2, 0
-; FLAT-NEXT:    v_cmp_lt_i32_e64 s[2:3], s3, 4
-; FLAT-NEXT:    s_or_b64 s[8:9], s[0:1], s[2:3]
-; FLAT-NEXT:    s_and_b64 s[0:1], exec, s[2:3]
-; FLAT-NEXT:    s_and_b64 s[2:3], exec, s[8:9]
+; FLAT-NEXT:    v_cmp_lt_i32_e64 s[0:1], s2, 1
+; FLAT-NEXT:    v_cmp_lt_i32_e64 s[4:5], s3, 4
+; FLAT-NEXT:    v_cmp_gt_i32_e64 s[2:3], s3, 3
+; FLAT-NEXT:    s_and_b64 s[2:3], s[0:1], s[2:3]
+; FLAT-NEXT:    s_and_b64 s[0:1], exec, s[4:5]
+; FLAT-NEXT:    s_and_b64 s[2:3], exec, s[2:3]
 ; FLAT-NEXT:    s_waitcnt vmcnt(0)
-; FLAT-NEXT:    v_cmp_lt_f32_e64 s[8:9], |v0|, s5
+; FLAT-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s9
+; FLAT-NEXT:    s_and_b64 s[4:5], exec, s[4:5]
 ; FLAT-NEXT:    v_mov_b32_e32 v0, 3
-; FLAT-NEXT:  BB3_1: ; %while.cond
+; FLAT-NEXT:    s_branch BB3_4
+; FLAT-NEXT:  BB3_1: ; %Flow6
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_mov_b64 s[10:11], 0
+; FLAT-NEXT:  BB3_2: ; %Flow5
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_mov_b64 s[14:15], 0
+; FLAT-NEXT:  BB3_3: ; %Flow
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_and_b64 vcc, exec, s[12:13]
+; FLAT-NEXT:    s_cbranch_vccnz BB3_8
+; FLAT-NEXT:  BB3_4: ; %while.cond
 ; FLAT-NEXT:    ; =>This Inner Loop Header: Depth=1
+; FLAT-NEXT:    s_mov_b64 s[14:15], -1
+; FLAT-NEXT:    s_mov_b64 s[10:11], -1
+; FLAT-NEXT:    s_mov_b64 s[12:13], -1
 ; FLAT-NEXT:    s_mov_b64 vcc, s[0:1]
-; FLAT-NEXT:    s_cbranch_vccz BB3_5
-; FLAT-NEXT:  ; %bb.2: ; %convex.exit
-; FLAT-NEXT:    ; in Loop: Header=BB3_1 Depth=1
+; FLAT-NEXT:    s_cbranch_vccz BB3_3
+; FLAT-NEXT:  ; %bb.5: ; %convex.exit
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_mov_b64 s[10:11], -1
+; FLAT-NEXT:    s_mov_b64 s[12:13], -1
 ; FLAT-NEXT:    s_mov_b64 vcc, s[2:3]
-; FLAT-NEXT:    s_cbranch_vccnz BB3_8
-; FLAT-NEXT:  ; %bb.3: ; %if.end
-; FLAT-NEXT:    ; in Loop: Header=BB3_1 Depth=1
-; FLAT-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; FLAT-NEXT:    s_cbranch_vccnz BB3_1
-; FLAT-NEXT:  ; %bb.4: ; %if.else
-; FLAT-NEXT:    ; in Loop: Header=BB3_1 Depth=1
+; FLAT-NEXT:    s_cbranch_vccz BB3_2
+; FLAT-NEXT:  ; %bb.6: ; %if.end
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_mov_b64 s[12:13], -1
+; FLAT-NEXT:    s_mov_b64 vcc, s[4:5]
+; FLAT-NEXT:    s_cbranch_vccz BB3_1
+; FLAT-NEXT:  ; %bb.7: ; %if.else
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_mov_b64 s[12:13], 0
 ; FLAT-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; FLAT-NEXT:    s_branch BB3_1
-; FLAT-NEXT:  BB3_5: ; %for.cond.preheader
+; FLAT-NEXT:  BB3_8: ; %loop.exit.guard4
+; FLAT-NEXT:    ; in Loop: Header=BB3_4 Depth=1
+; FLAT-NEXT:    s_and_b64 vcc, exec, s[10:11]
+; FLAT-NEXT:    s_cbranch_vccz BB3_4
+; FLAT-NEXT:  ; %bb.9: ; %loop.exit.guard
+; FLAT-NEXT:    s_and_b64 vcc, exec, s[14:15]
+; FLAT-NEXT:    s_cbranch_vccz BB3_13
+; FLAT-NEXT:  ; %bb.10: ; %for.cond.preheader
 ; FLAT-NEXT:    v_mov_b32_e32 v0, 0x3e8
-; FLAT-NEXT:    v_cmp_lt_i32_e32 vcc, s4, v0
+; FLAT-NEXT:    v_cmp_lt_i32_e32 vcc, s8, v0
 ; FLAT-NEXT:    s_and_b64 vcc, exec, vcc
-; FLAT-NEXT:    s_cbranch_vccz BB3_8
-; FLAT-NEXT:  ; %bb.6: ; %for.body
+; FLAT-NEXT:    s_cbranch_vccz BB3_13
+; FLAT-NEXT:  ; %bb.11: ; %for.body
 ; FLAT-NEXT:    s_and_b64 vcc, exec, 0
-; FLAT-NEXT:  BB3_7: ; %self.loop
+; FLAT-NEXT:  BB3_12: ; %self.loop
 ; FLAT-NEXT:    ; =>This Inner Loop Header: Depth=1
-; FLAT-NEXT:    s_cbranch_vccz BB3_7
-; FLAT-NEXT:  BB3_8: ; %DummyReturnBlock
+; FLAT-NEXT:    s_cbranch_vccz BB3_12
+; FLAT-NEXT:  BB3_13: ; %DummyReturnBlock
 ; FLAT-NEXT:    s_endpgm
 entry:
   %cmp = icmp sgt i32 %c0, 0
   br label %while.cond.outer
 
 while.cond.outer:
   %tmp = load float, float addrspace(1)* undef
   br label %while.cond