[AMDGPU] Mark s_endpgm convergent

[rampitec]

Summary

This shall prevent tail duplication of blocks with s_endpgm.

[arsenm]

The concept of convergence we have breaks down at the machine level. At the machine level, we don't exactly need to think about the operations treated in the IR as convergent as related to control flow anymore. s_endpgm doesn't really have the convergent properties anyway

[rampitec]

In D62937#1531865, @arsenm wrote:

The concept of convergence we have breaks down at the machine level. At the machine level, we don't exactly need to think about the operations treated in the IR as convergent as related to control flow anymore. s_endpgm doesn't really have the convergent properties anyway

You cannot duplicate it by moving into the control flow. If you move it into the 'if..then' block that will prevent subsequent 'else' from from execution. How do you propose to shield it from duplication?

[nhaehnle]

I also don't understand the purpose of this and what this solves.

Convergent is defined as "may not be modified to have additional control dependencies". However, at the MIR level our branches are all effectively scalar, so adding control dependencies should actually always be okay.

lib/Target/AMDGPU/SOPInstructions.td

 
 def S_NOP : SOPP <0x00000000, (ins i16imm:$simm16), "s_nop $simm16">;
 
 let isTerminator = 1 in {
 
 def S_ENDPGM : SOPP <0x00000001, (ins EndpgmImm:$simm16), "s_endpgm $simm16"> {
   let isBarrier = 1;
   let isReturn = 1;
+  let isConvergent = 1;
 }
 
 def S_ENDPGM_SAVED : SOPP <0x0000001B, (ins), "s_endpgm_saved"> {
   let SubtargetPredicate = isGFX8Plus;
   let simm16 = 0;
   let isBarrier = 1;
   let isReturn = 1;
+  let isConvergent = 1;
 }
 
-let SubtargetPredicate = isGFX9Plus in {
+let SubtargetPredicate = isGFX9Plus, isConvergent = 1 in {
   let isBarrier = 1, isReturn = 1, simm16 = 0 in {
     def S_ENDPGM_ORDERED_PS_DONE :
       SOPP<0x01e, (ins), "s_endpgm_ordered_ps_done">;
   } // End isBarrier = 1, isReturn = 1, simm16 = 0
-} // End SubtargetPredicate = isGFX9Plus
+} // End SubtargetPredicate = isGFX9Plus, isConvergent = 1
 
 let SubtargetPredicate = isGFX10Plus in {
   let isBarrier = 1, isReturn = 1, simm16 = 0 in {
     def S_CODE_END :
       SOPP<0x01f, (ins), "s_code_end">;
   } // End isBarrier = 1, isReturn = 1, simm16 = 0
 } // End SubtargetPredicate = isGFX10Plus
 

test/CodeGen/AMDGPU/fneg-combines.ll

 ; CopyToReg tests
 ; --------------------------------------------------------------------------------
 
 ; GCN-LABEL: {{^}}v_fneg_copytoreg_f32:
 ; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_mul_f32_e32 [[MUL0:v[0-9]+]], [[A]], [[B]]
-; GCN: s_cbranch_scc0
-
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[MUL0]]
-; GCN: s_endpgm
+; GCN: s_cbranch_scc1
 
 ; GCN: v_xor_b32_e32 [[XOR:v[0-9]+]], 0x80000000, [[MUL0]]
 ; GCN: v_mul_f32_e32 [[MUL1:v[0-9]+]], [[XOR]], [[C]]
 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[MUL1]]
 
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[MUL0]]
+; GCN: s_endpgm
+
 define amdgpu_kernel void @v_fneg_copytoreg_f32(float addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr, float addrspace(1)* %c.ptr, i32 %d) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %b.gep = getelementptr inbounds float, float addrspace(1)* %b.ptr, i64 %tid.ext
   %c.gep = getelementptr inbounds float, float addrspace(1)* %c.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep

test/CodeGen/AMDGPU/tail-duplication-convergent.ll

bb2:
   store i32 1, i32 addrspace(1)* %a
   br label %call
 
 call:
   tail call void @convergent_func()
   ret void
 }
 
+; s_endpgm shouldn't be duplicated.
+
+; GCN-LABEL: {{^}}taildup_ret:
+; GCN: s_endpgm
+; GCN-NOT: s_endpgm
+define amdgpu_kernel void @taildup_ret(i32 addrspace(1)* %a, i1 %cond) #0 {
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  store i32 0, i32 addrspace(1)* %a
+  br label %call
+
+bb2:
+  store i32 1, i32 addrspace(1)* %a
+  br label %call
+
+call:
+  br label %ret
+
+ret:
+  ret void
+}
+
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind convergent }

test/CodeGen/AMDGPU/uniform-cfg.ll

 }
 
 
 ; GCN-LABEL: {{^}}uniform_if_else_ret:
 ; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
 ; GCN: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2
-; GCN: buffer_store_dword [[TWO]]
-; GCN: s_endpgm
 
 ; GCN: {{^}}[[IF_LABEL]]:
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @uniform_if_else_ret(i32 addrspace(1)* nocapture %out, i32 %a) {
 entry:
   %cmp = icmp eq i32 %a, 0
ENDIF:                                            ; preds = %IF, %main_body
   ret void
 }
 
 ; GCN-LABEL: {{^}}icmp_users_different_blocks:
 ; GCN: s_load_dwordx2 s{{\[}}[[COND0:[0-9]+]]:[[COND1:[0-9]+]]{{\]}}
 ; GCN: s_cmp_lt_i32 s[[COND0]], 1
 ; GCN: s_cbranch_scc1 [[EXIT:[A-Za-z0-9_]+]]
 ; GCN: v_cmp_gt_i32_e64 {{[^,]*}}, s[[COND1]], 0{{$}}
-; GCN: s_cbranch_vccz [[BODY:[A-Za-z0-9_]+]]
-; GCN: {{^}}[[EXIT]]:
-; GCN: s_endpgm
-; GCN: {{^}}[[BODY]]:
+; GCN: s_cbranch_vccnz [[EXIT:[A-Za-z0-9_]+]]
 ; GCN: buffer_store
+; GCN: {{^}}[[EXIT]]:
 ; GCN: s_endpgm
 define amdgpu_kernel void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, i32 addrspace(1)* %out) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %cmp0 = icmp sgt i32 %cond0, 0
   %cmp1 = icmp sgt i32 %cond1, 0
   br i1 %cmp0, label %bb2, label %bb9
 

 }
 
 ; Test uniform and divergent.
 
 ; GCN-LABEL: {{^}}uniform_inside_divergent:
 ; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}}
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
 ; GCN: s_cmp_lg_u32 {{s[0-9]+}}, 0
-; GCN: s_cbranch_scc0 [[IF_UNIFORM_LABEL:[A-Z0-9_a-z]+]]
-; GCN: s_endpgm
-; GCN: {{^}}[[IF_UNIFORM_LABEL]]:
+; GCN: s_cbranch_scc1 [[ENDIF_UNIFORM_LABEL:[A-Z0-9_a-z]+]]
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
+; GCN: {{^}}[[ENDIF_UNIFORM_LABEL]]:
+; GCN: s_endpgm
 define amdgpu_kernel void @uniform_inside_divergent(i32 addrspace(1)* %out, i32 %cond) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #0
   %d_cmp = icmp ult i32 %tid, 16
   br i1 %d_cmp, label %if, label %endif
 
 if:
   store i32 0, i32 addrspace(1)* %out

 
 ; GCN-LABEL: {{^}}divergent_if_uniform_if:
 ; GCN: v_cmp_eq_u32_e32 vcc, 0, v0
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
 ; GCN: s_or_b64 exec, exec, [[MASK]]
 ; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cbranch_scc0 [[IF_UNIFORM:[A-Z0-9_]+]]
-; GCN: s_endpgm
-; GCN: [[IF_UNIFORM]]:
+; GCN: s_cbranch_scc1 [[ENDIF_UNIFORM:[A-Z0-9_]+]]
 ; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2
 ; GCN: buffer_store_dword [[TWO]]
+; GCN: [[ENDIF_UNIFORM]]:
+; GCN: s_endpgm
 define amdgpu_kernel void @divergent_if_uniform_if(i32 addrspace(1)* %out, i32 %cond) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #0
   %d_cmp = icmp eq i32 %tid, 0
   br i1 %d_cmp, label %if, label %endif
 
 if:
   store i32 1, i32 addrspace(1)* %out