[AMDGPU] Mark s_barrier as having side effects but not accessing memory.

[foad]

Summary

This fixes poor scheduling in a function containing a barrier and a few
load instructions.

Without this fix, ScheduleDAGInstrs::buildSchedGraph adds an artificial
edge in the dependency graph from the barrier instruction to the exit
node representing live-out latency, with a latency of about 500 cycles.
Because of this it thinks the critical path through the graph also has
a latency of about 500 cycles. And because of that it does not think
that any of the load instructions are on the critical path, so it
schedules them with no regard for their (80 cycle) latency, which gives
poor results.

[arsenm]

LGTM

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

209 ↗

(On Diff #218880)

This does end up not adding readnone to the intrinsic declaration, correct?

llvm/test/CodeGen/AMDGPU/schedule-barrier.mir

44 ↗

(On Diff #218880)

You can remove the into and IR fragment

[foad]

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

209 ↗

(On Diff #218880)

The declaration comes out as:

; Function Attrs: convergent nounwind
declare void @llvm.amdgcn.s.barrier() #0

llvm/trunk/include/llvm/IR/IntrinsicsAMDGPU.td

 def int_amdgcn_s_sendmsg : GCCBuiltin<"__builtin_amdgcn_s_sendmsg">,
   Intrinsic <[], [llvm_i32_ty, llvm_i32_ty],
   [ImmArg<0>, IntrNoMem, IntrHasSideEffects]>;
 def int_amdgcn_s_sendmsghalt : GCCBuiltin<"__builtin_amdgcn_s_sendmsghalt">,
   Intrinsic <[], [llvm_i32_ty, llvm_i32_ty],
   [ImmArg<0>, IntrNoMem, IntrHasSideEffects]>;
 
 def int_amdgcn_s_barrier : GCCBuiltin<"__builtin_amdgcn_s_barrier">,
-  Intrinsic<[], [], [IntrConvergent]>;
+  Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent]>;
 
 def int_amdgcn_wave_barrier : GCCBuiltin<"__builtin_amdgcn_wave_barrier">,
   Intrinsic<[], [], [IntrConvergent]>;
 
 def int_amdgcn_s_waitcnt : GCCBuiltin<"__builtin_amdgcn_s_waitcnt">,
   Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]>;
 
 def int_amdgcn_div_scale : Intrinsic<

llvm/trunk/lib/Target/AMDGPU/SOPInstructions.td

 } // End isBranch = 1
 } // End isTerminator = 1
 
 let hasSideEffects = 1 in {
 def S_BARRIER : SOPP <0x0000000a, (ins), "s_barrier",
   [(int_amdgcn_s_barrier)]> {
   let SchedRW = [WriteBarrier];
   let simm16 = 0;
-  let mayLoad = 1;
-  let mayStore = 1;
   let isConvergent = 1;
 }
 
 def S_WAKEUP : SOPP <0x00000003, (ins), "s_wakeup"> {
   let SubtargetPredicate = isGFX8Plus;
   let simm16 = 0;
   let mayLoad = 1;
   let mayStore = 1;

llvm/trunk/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll

 ; VARIANT0-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; VARIANT0-NEXT:    v_not_b32_e32 v3, v0
 ; VARIANT0-NEXT:    s_mov_b32 s7, 0xf000
 ; VARIANT0-NEXT:    s_mov_b32 s6, 0
 ; VARIANT0-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT0-NEXT:    v_mov_b32_e32 v2, 0
 ; VARIANT0-NEXT:    s_waitcnt lgkmcnt(0)
 ; VARIANT0-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
-; VARIANT0-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT0-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; VARIANT0-NEXT:    s_barrier
+; VARIANT0-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT0-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; VARIANT0-NEXT:    v_lshl_b64 v[3:4], v[3:4], 2
 ; VARIANT0-NEXT:    buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
 ; VARIANT0-NEXT:    s_waitcnt vmcnt(0)
 ; VARIANT0-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
 ; VARIANT0-NEXT:    s_endpgm
 ;
 ; VARIANT1-LABEL: test_barrier:
 ; VARIANT1:       ; %bb.0: ; %entry
 ; VARIANT1-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; VARIANT1-NEXT:    s_load_dword s2, s[0:1], 0xb
 ; VARIANT1-NEXT:    v_not_b32_e32 v3, v0
 ; VARIANT1-NEXT:    s_mov_b32 s7, 0xf000
 ; VARIANT1-NEXT:    s_mov_b32 s6, 0
 ; VARIANT1-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT1-NEXT:    v_mov_b32_e32 v2, 0
 ; VARIANT1-NEXT:    s_waitcnt lgkmcnt(0)
 ; VARIANT1-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
-; VARIANT1-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT1-NEXT:    s_barrier
+; VARIANT1-NEXT:    v_add_i32_e32 v3, vcc, s2, v3
 ; VARIANT1-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; VARIANT1-NEXT:    v_lshl_b64 v[3:4], v[3:4], 2
 ; VARIANT1-NEXT:    s_waitcnt expcnt(0)
 ; VARIANT1-NEXT:    buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
 ; VARIANT1-NEXT:    s_waitcnt vmcnt(0)
 ; VARIANT1-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
 ; VARIANT1-NEXT:    s_endpgm
 ;
 ; VARIANT2-LABEL: test_barrier:
 ; VARIANT2:       ; %bb.0: ; %entry
 ; VARIANT2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; VARIANT2-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT2-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VARIANT2-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
 ; VARIANT2-NEXT:    s_waitcnt lgkmcnt(0)
-; VARIANT2-NEXT:    v_mov_b32_e32 v2, s3
-; VARIANT2-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v1
-; VARIANT2-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; VARIANT2-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT2-NEXT:    v_mov_b32_e32 v4, s3
+; VARIANT2-NEXT:    v_xad_u32 v1, v0, -1, s0
+; VARIANT2-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; VARIANT2-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
+; VARIANT2-NEXT:    v_lshlrev_b64 v[1:2], 2, v[1:2]
+; VARIANT2-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; VARIANT2-NEXT:    global_store_dword v[3:4], v0, off
+; VARIANT2-NEXT:    v_mov_b32_e32 v5, s3
+; VARIANT2-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v1
+; VARIANT2-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v2, vcc
 ; VARIANT2-NEXT:    s_waitcnt vmcnt(0)
 ; VARIANT2-NEXT:    s_barrier
-; VARIANT2-NEXT:    v_xad_u32 v3, v0, -1, s0
-; VARIANT2-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; VARIANT2-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
-; VARIANT2-NEXT:    v_mov_b32_e32 v0, s3
-; VARIANT2-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
-; VARIANT2-NEXT:    v_addc_co_u32_e32 v4, vcc, v0, v4, vcc
-; VARIANT2-NEXT:    global_load_dword v0, v[3:4], off
+; VARIANT2-NEXT:    global_load_dword v0, v[0:1], off
 ; VARIANT2-NEXT:    s_waitcnt vmcnt(0)
-; VARIANT2-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT2-NEXT:    global_store_dword v[3:4], v0, off
 ; VARIANT2-NEXT:    s_endpgm
 ;
 ; VARIANT3-LABEL: test_barrier:
 ; VARIANT3:       ; %bb.0: ; %entry
 ; VARIANT3-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; VARIANT3-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; VARIANT3-NEXT:    s_load_dword s0, s[0:1], 0x2c
+; VARIANT3-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
 ; VARIANT3-NEXT:    s_waitcnt lgkmcnt(0)
-; VARIANT3-NEXT:    v_mov_b32_e32 v2, s3
-; VARIANT3-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v1
-; VARIANT3-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; VARIANT3-NEXT:    global_store_dword v[1:2], v0, off
-; VARIANT3-NEXT:    s_barrier
-; VARIANT3-NEXT:    v_xad_u32 v3, v0, -1, s0
-; VARIANT3-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; VARIANT3-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
-; VARIANT3-NEXT:    v_mov_b32_e32 v0, s3
+; VARIANT3-NEXT:    v_mov_b32_e32 v4, s3
+; VARIANT3-NEXT:    v_xad_u32 v1, v0, -1, s0
+; VARIANT3-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; VARIANT3-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v3
-; VARIANT3-NEXT:    v_addc_co_u32_e32 v4, vcc, v0, v4, vcc
-; VARIANT3-NEXT:    global_load_dword v0, v[3:4], off
+; VARIANT3-NEXT:    v_lshlrev_b64 v[1:2], 2, v[1:2]
+; VARIANT3-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; VARIANT3-NEXT:    global_store_dword v[3:4], v0, off
+; VARIANT3-NEXT:    v_mov_b32_e32 v5, s3
+; VARIANT3-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v1
+; VARIANT3-NEXT:    v_addc_co_u32_e32 v1, vcc, v5, v2, vcc
+; VARIANT3-NEXT:    s_barrier
+; VARIANT3-NEXT:    global_load_dword v0, v[0:1], off
 ; VARIANT3-NEXT:    s_waitcnt vmcnt(0)
-; VARIANT3-NEXT:    global_store_dword v[1:2], v0, off
+; VARIANT3-NEXT:    global_store_dword v[3:4], v0, off
 ; VARIANT3-NEXT:    s_endpgm
 entry:
   %tmp = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = getelementptr i32, i32 addrspace(1)* %out, i32 %tmp
   store i32 %tmp, i32 addrspace(1)* %tmp1
   call void @llvm.amdgcn.s.barrier()
   %tmp3 = sub i32 %size, 1
   %tmp4 = sub i32 %tmp3, %tmp

llvm/trunk/test/CodeGen/AMDGPU/local-memory.amdgcn.ll

 ; Check that the LDS size emitted correctly
 ; EG: .long 166120
 ; EG-NEXT: .long 8
 ; GCN: .long 47180
 ; GCN-NEXT: .long 32900
 
 ; GCN-LABEL: {{^}}local_memory_two_objects:
 ; GCN: v_lshlrev_b32_e32 [[ADDRW:v[0-9]+]], 2, v0
+; CI-DAG: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 0, [[ADDRW]]
 ; CI-DAG: ds_write2_b32 [[ADDRW]], {{v[0-9]+}}, {{v[0-9]+}} offset1:4
 ; SI-DAG: ds_write2_b32 [[ADDRW]], {{v[0-9]+}}, {{v[0-9]+}} offset1:4
-
-; GCN: s_barrier
-
 ; SI-DAG: v_sub_i32_e32 [[SUB0:v[0-9]+]], vcc, 28, [[ADDRW]]
 ; SI-DAG: v_sub_i32_e32 [[SUB1:v[0-9]+]], vcc, 12, [[ADDRW]]
 
+; GCN: s_barrier
+
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[SUB0]]
 ; SI-DAG: ds_read_b32 v{{[0-9]+}}, [[SUB1]]
-
-; CI: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 0, [[ADDRW]]
 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, [[SUB]] offset0:3 offset1:7
 
 define amdgpu_kernel void @local_memory_two_objects(i32 addrspace(1)* %out) #0 {
 entry:
   %x.i = call i32 @llvm.amdgcn.workitem.id.x()
   %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(3)* @local_memory_two_objects.local_mem0, i32 0, i32 %x.i
   store i32 %x.i, i32 addrspace(3)* %arrayidx, align 4
   %mul = shl nsw i32 %x.i, 1

llvm/trunk/test/CodeGen/AMDGPU/schedule-barrier.mir

+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=machine-scheduler -o - %s | FileCheck %s
+
+---
+# Check that the high latency loads are both scheduled first, before the
+# multiplies, despite the presence of a barrier in the function.
+# CHECK: BUFFER_LOAD_DWORD_OFFSET
+# CHECK: BUFFER_LOAD_DWORD_OFFSET
+# CHECK: V_MUL_LO_U32
+# CHECK: V_MUL_LO_U32
+name: test
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+
+    undef %43.sub3:vreg_128 = COPY $vgpr9
+    undef %42.sub2:vreg_128 = COPY $vgpr8
+    undef %41.sub1:vreg_128 = COPY $vgpr7
+    undef %26.sub0:vreg_128 = COPY $vgpr6
+    undef %46.sub3:vreg_128 = COPY $vgpr5
+    undef %45.sub2:vreg_128 = COPY $vgpr4
+    undef %44.sub1:vreg_128 = COPY $vgpr3
+    undef %32.sub0:vreg_128 = COPY $vgpr2
+    undef %38.sub1:vreg_64 = COPY $vgpr1
+    %38.sub0:vreg_64 = COPY $vgpr0
+
+    S_BARRIER
+
+    undef %33.sub0:sgpr_128 = V_READFIRSTLANE_B32 %32.sub0, implicit $exec
+    %33.sub1:sgpr_128 = V_READFIRSTLANE_B32 %44.sub1, implicit $exec
+    %33.sub2:sgpr_128 = V_READFIRSTLANE_B32 %45.sub2, implicit $exec
+    %33.sub3:sgpr_128 = V_READFIRSTLANE_B32 %46.sub3, implicit $exec
+    %15:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %33, 0, 0, 0, 0, 0, 0, implicit $exec
+    %39:vgpr_32 = V_MUL_LO_U32 %15, %15, implicit $exec
+
+    undef %27.sub0:sgpr_128 = V_READFIRSTLANE_B32 %26.sub0, implicit $exec
+    %27.sub1:sgpr_128 = V_READFIRSTLANE_B32 %41.sub1, implicit $exec
+    %27.sub2:sgpr_128 = V_READFIRSTLANE_B32 %42.sub2, implicit $exec
+    %27.sub3:sgpr_128 = V_READFIRSTLANE_B32 %43.sub3, implicit $exec
+    %19:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %27, 0, 0, 0, 0, 0, 0, implicit $exec
+    %40:vgpr_32 = V_MUL_LO_U32 %19, %19, implicit $exec
+
+    %23:vgpr_32 = V_ADD_U32_e32 %39, %40, implicit $exec
+    GLOBAL_STORE_DWORD %38, %23, 0, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+...

llvm/trunk/test/CodeGen/AMDGPU/v_mac_f16.ll

 }
 
 ; GCN-LABEL: {{^}}mac_v2f16:
 ; GCN: {{buffer|flat}}_load_dword v[[A_V2_F16:[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword v[[B_V2_F16:[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword v[[C_V2_F16:[0-9]+]]
 
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
-; SI:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
-; SI:  v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
-; SI:  v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
-; SI:  v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]]
+; SI-DAG:  v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+
+; SI-DAG:  v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
+
+; SI-DAG:  v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]]
 ; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]]
+
 ; SI-DAG:  v_mac_f32_e32 v[[C_F32_0]], v[[A_F32_0]], v[[B_F32_0]]
 ; SI-DAG:  v_cvt_f16_f32_e32 v[[R_F16_LO:[0-9]+]], v[[C_F32_0]]
 ; SI-DAG:  v_mac_f32_e32 v[[C_F32_1]], v[[A_F32_1]], v[[B_F32_1]]
 ; SI-DAG:  v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[C_F32_1]]
 ; SI:  v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
 ; VI-NOT: and
 ; SI:  v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_LO]], v[[R_F16_HI]]
 

llvm/trunk/test/CodeGen/AMDGPU/waitcnt-vscnt.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx802  -asm-verbose=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8_9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900  -asm-verbose=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9_10,GFX8_9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -asm-verbose=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX9_10 %s
 
 ; GCN-LABEL: barrier_vmcnt_global:
 ; GFX8:         flat_load_dword
 ; GFX9_10:      global_load_dword
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9_10-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX8:         s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9_10:      s_waitcnt vmcnt(0){{$}}
 ; GCN-NEXT:     s_barrier
 define amdgpu_kernel void @barrier_vmcnt_global(i32 addrspace(1)* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp1
   %tmp4 = load i32, i32 addrspace(1)* %tmp3, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier()
   fence syncscope("singlethread") acquire
   %tmp5 = add nuw nsw i64 %tmp2, 4294967296
   %tmp6 = lshr exact i64 %tmp5, 32
   %tmp7 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp6
   store i32 %tmp4, i32 addrspace(1)* %tmp7, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vscnt_global:
 ; GFX8:       flat_store_dword
 ; GFX9_10:    global_store_dword
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9:       s_waitcnt vmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vscnt_global(i32 addrspace(1)* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = add nuw nsw i64 %tmp2, 8589934592
   %tmp4 = lshr exact i64 %tmp3, 32
   %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp4
   store i32 0, i32 addrspace(1)* %tmp5, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier() #3
   fence syncscope("singlethread") acquire
   %tmp6 = add nuw nsw i64 %tmp2, 4294967296
   %tmp7 = lshr exact i64 %tmp6, 32
   %tmp8 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp7
   store i32 1, i32 addrspace(1)* %tmp8, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vmcnt_vscnt_global:
 ; GFX8:         flat_load_dword
 ; GFX9_10:      global_load_dword
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX9_10-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT:   s_waitcnt_vscnt null, 0x0
+; GFX8:         s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX9_10:      s_waitcnt vmcnt(0){{$}}
+; GFX10:        s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:     s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_global(i32 addrspace(1)* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = add nuw nsw i64 %tmp2, 8589934592
   %tmp4 = lshr exact i64 %tmp3, 32
   %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp4
   store i32 0, i32 addrspace(1)* %tmp5, align 4
   %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp1
   %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier()
   fence syncscope("singlethread") acquire
   %tmp8 = add nuw nsw i64 %tmp2, 4294967296
   %tmp9 = lshr exact i64 %tmp8, 32
   %tmp10 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp9
   store i32 %tmp7, i32 addrspace(1)* %tmp10, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vmcnt_flat:
 ; GCN:      flat_load_dword
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN-NEXT: s_barrier
 define amdgpu_kernel void @barrier_vmcnt_flat(i32* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = getelementptr inbounds i32, i32* %arg, i64 %tmp1
   %tmp4 = load i32, i32* %tmp3, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier()
   fence syncscope("singlethread") acquire
   %tmp5 = add nuw nsw i64 %tmp2, 4294967296
   %tmp6 = lshr exact i64 %tmp5, 32
   %tmp7 = getelementptr inbounds i32, i32* %arg, i64 %tmp6
   store i32 %tmp4, i32* %tmp7, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vscnt_flat:
 ; GCN:         flat_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_barrier
 define amdgpu_kernel void @barrier_vscnt_flat(i32* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = add nuw nsw i64 %tmp2, 8589934592
   %tmp4 = lshr exact i64 %tmp3, 32
   %tmp5 = getelementptr inbounds i32, i32* %arg, i64 %tmp4
   store i32 0, i32* %tmp5, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier() #3
   fence syncscope("singlethread") acquire
   %tmp6 = add nuw nsw i64 %tmp2, 4294967296
   %tmp7 = lshr exact i64 %tmp6, 32
   %tmp8 = getelementptr inbounds i32, i32* %arg, i64 %tmp7
   store i32 1, i32* %tmp8, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vmcnt_vscnt_flat:
 ; GCN:        flat_load_dword
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_flat(i32* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = add nuw nsw i64 %tmp2, 8589934592
   %tmp4 = lshr exact i64 %tmp3, 32
   %tmp5 = getelementptr inbounds i32, i32* %arg, i64 %tmp4
   store i32 0, i32* %tmp5, align 4
   %tmp6 = getelementptr inbounds i32, i32* %arg, i64 %tmp1
   %tmp7 = load i32, i32* %tmp6, align 4
   fence syncscope("singlethread") release
   tail call void @llvm.amdgcn.s.barrier()
   fence syncscope("singlethread") acquire
   %tmp8 = add nuw nsw i64 %tmp2, 4294967296
   %tmp9 = lshr exact i64 %tmp8, 32
   %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %tmp9
   store i32 %tmp7, i32* %tmp10, align 4
   ret void
 }
 
 ; GCN-LABEL: barrier_vmcnt_vscnt_flat_workgroup:
 ; GCN:        flat_load_dword
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_barrier
 define amdgpu_kernel void @barrier_vmcnt_vscnt_flat_workgroup(i32* %arg) {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = zext i32 %tmp to i64
   %tmp2 = shl nuw nsw i64 %tmp1, 32
   %tmp3 = add nuw nsw i64 %tmp2, 8589934592
   %tmp4 = lshr exact i64 %tmp3, 32
bb:
   %tmp6 = lshr exact i64 %tmp5, 32
   %tmp7 = getelementptr inbounds i32, i32* %arg, i64 %tmp6
   store i32 %tmp4, i32* %tmp7, align 4
   ret void
 }
 
 ; GCN-LABEL: store_vscnt_private:
 ; GCN:         buffer_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0)
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_private(i32 addrspace(5)* %p) {
   store i32 0, i32 addrspace(5)* %p
   ret void
 }
 
 ; GCN-LABEL: store_vscnt_global:
 ; GFX8:        flat_store_dword
 ; GFX9_10:     global_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0)
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_global(i32 addrspace(1)* %p) {
   store i32 0, i32 addrspace(1)* %p
   ret void
 }
 
 ; GCN-LABEL: store_vscnt_flat:
 ; GCN:         flat_store_dword
-; GFX8_9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0
+; GFX8_9:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    s_setpc_b64
 define void @store_vscnt_flat(i32* %p) {
   store i32 0, i32* %p
   ret void
 }
 
 ; GCN-LABEL: function_prologue:
-; GCN-NEXT:   s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GCN:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:   s_setpc_b64
 define void @function_prologue() {
   ret void
 }
 
 declare void @llvm.amdgcn.s.barrier()
 declare i32 @llvm.amdgcn.workitem.id.x()