[AMDGPU] Keep skip branch for ds instructions

[sebastian-ne]

Summary

Same as other memory instructions, ds instructions add latency even if
exec is zero. Jumping over them if exec=0 is cheaper than executing
them.
With this change, the branch instruction that skips over a basic block
if exec=0 is not removed when the block contains a ds instruction.

[arsenm]

If I remember my cycle counts correctly, with an untaken branch this will be about breakeven for a single DS op?

[critson]

In D97922#2603323, @arsenm wrote:

If I remember my cycle counts correctly, with an untaken branch this will be about breakeven for a single DS op?

I believe this is about right.
However since we allow 12 instructions and rarely encounter a DS op in isolation, I think the branch is generally cheaper (assuming cache hit).

[sebastian-ne]

Yes, that’s also what Carl concluded. Taking a s_cbranch_execz should be about the same amount of cycles as a ds_ instruction.
If we more than a ds_ instruction, taking the branch should be cheaper.

llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp

for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end();
       if (TII->hasUnwantedEffectsWhenEXECEmpty(*I))
         return true;
 
       if (TII->isKillTerminator(I->getOpcode()))
         return true;
 
       // These instructions are potentially expensive even if EXEC = 0.
       if (TII->isSMRD(*I) || TII->isVMEM(*I) || TII->isFLAT(*I) ||
-          I->getOpcode() == AMDGPU::S_WAITCNT)
+          TII->isDS(*I) || I->getOpcode() == AMDGPU::S_WAITCNT)
         return true;
 
       ++NumInstr;
       if (NumInstr >= SkipThreshold)
         return true;
     }
   }
 

llvm/test/CodeGen/AMDGPU/branch-condition-and.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
 
 ; This used to crash because during intermediate control flow lowering, there
 ; was a sequence
 ;       s_mov_b64 s[0:1], exec
 ;       s_and_b64 s[2:3], s[0:1], s[2:3] ; def & use of the same register pair
 ;       ...
 ;       s_mov_b64_term exec, s[2:3]
 ; that was not treated correctly.
 ;
 ; GCN-LABEL: {{^}}ham:
 ; GCN-DAG: v_cmp_lt_f32_e64 [[OTHERCC:s\[[0-9]+:[0-9]+\]]],
 ; GCN-DAG: v_cmp_lt_f32_e32 vcc,
 ; GCN: s_and_b64 [[AND:s\[[0-9]+:[0-9]+\]]], vcc, [[OTHERCC]]
 ; GCN: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[AND]]
+; GCN-NEXT: s_cbranch_execz BB0_{{[0-9]+}}
 
 ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; %bb4
 ; GCN: ds_write_b32
 
-; GCN: ; %bb.{{[0-9]+}}:
+; GCN: BB0_{{[0-9]+}}: ; %UnifiedReturnBlock
 ; GCN-NEXT: s_endpgm
 ; GCN-NEXT: .Lfunc_end
 define amdgpu_ps void @ham(float %arg, float %arg1) #0 {
 bb:
   %tmp = fcmp ogt float %arg, 0.000000e+00
   %tmp2 = fcmp ogt float %arg1, 0.000000e+00
   %tmp3 = and i1 %tmp, %tmp2
   br i1 %tmp3, label %bb4, label %bb5

llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir

body: |
   bb.1:
     successors: %bb.2
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2:
     S_ENDPGM 0
 ...
+
+---
+
+name: skip_execz_ds
+body: |
+  ; CHECK-LABEL: name: skip_execz_ds
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   SI_MASK_BRANCH %bb.2, implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.2
+    SI_MASK_BRANCH %bb.2, implicit $exec
+
+  bb.1:
+    successors: %bb.2
+    $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+
+  bb.2:
+    S_ENDPGM 0
+...

llvm/test/CodeGen/AMDGPU/ret_jump.ll

 ; GCN: s_cbranch_execnz [[RETURN:BB[0-9]+_[0-9]+]]
 
 ; GCN: ; %UnifiedReturnBlock
 ; GCN-NEXT: s_or_b64 exec, exec
 ; GCN-NEXT: s_waitcnt
 
 ; GCN: BB{{[0-9]+_[0-9]+}}: ; %else
 ; GCN: s_and_saveexec_b64 [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]], vcc
+; GCN-NEXT: s_cbranch_execz BB1_{{[0-9]+}}
 
 ; GCN-NEXT:  ; %unreachable.bb
 ; GCN: ds_write_b32
 ; GCN: ; divergent unreachable
 
 ; GCN: ; %ret.bb
 ; GCN: store_dword
 define amdgpu_ps <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> @uniform_br_nontrivial_ret_divergent_br_nontrivial_unreachable([9 x <4 x i32>] addrspace(4)* inreg %arg, [17 x <4 x i32>] addrspace(4)* inreg %arg1, [17 x <8 x i32>] addrspace(4)* inreg %arg2, i32 addrspace(4)* inreg %arg3, float inreg %arg4, i32 inreg %arg5, <2 x i32> %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <3 x i32> %arg9, <2 x i32> %arg10, <2 x i32> %arg11, <2 x i32> %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, i32 inreg %arg18, i32 %arg19, float %arg20, i32 %arg21) #0 {

llvm/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator:
 ; GCN: v_cmp_eq_u32
 ; GCN: s_and_saveexec_b64
+; GCN-NEXT: s_cbranch_execz BB0_{{[0-9]+}}
 
 ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; %unreachable
 ; GCN: ds_write_b32
 ; GCN: ; divergent unreachable
 
-; GCN-NEXT: ; %bb.{{[0-9]+}}: ; %UnifiedReturnBlock
+; GCN-NEXT: BB0_{{[0-9]+}}: ; %UnifiedReturnBlock
 ; GCN: s_endpgm
 
 define amdgpu_kernel void @lower_control_flow_unreachable_terminator() #0 {
 bb:
   %tmp15 = tail call i32 @llvm.amdgcn.workitem.id.y()
   %tmp63 = icmp eq i32 %tmp15, 32
   br i1 %tmp63, label %unreachable, label %ret
 
 unreachable:
   store volatile i32 0, i32 addrspace(3)* undef, align 4
   unreachable
 
 ret:
   ret void
 }
 
 ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator_swap_block_order:
 ; GCN: v_cmp_ne_u32
 ; GCN: s_and_saveexec_b64
+; GCN-NEXT: s_cbranch_execz BB1_{{[0-9]+}}
 
 ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; %unreachable
 ; GCN: ds_write_b32
 ; GCN: ; divergent unreachable
 
-; GCN: ; %bb.{{[0-9]+}}:
+; GCN: BB1_{{[0-9]+}}:
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @lower_control_flow_unreachable_terminator_swap_block_order() #0 {
 bb:
   %tmp15 = tail call i32 @llvm.amdgcn.workitem.id.y()
   %tmp63 = icmp eq i32 %tmp15, 32
   br i1 %tmp63, label %ret, label %unreachable
 
 ret: