[AMDGPU] Assembler: Swap operands of flat_store instructions to match AMD assembler

[nhaustov]

Summary

Historically, AMD internal sp3 assembler has flat_store* addr, data format. To match existing code and to enable reuse, change LLVM definitions to match.
Also update MC and CodeGen tests.

[arsenm]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

You should leave these trailing commas. These are to prevent doing something like matching v1 and succeeding on v10.

[nhaustov]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

This [[RESULT]] is the last operand of flat_store_dword after the change. I did not find any better way to match it, any suggestions?

[arsenm]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

You can use {{$}} to match the end of line

[nhaustov]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

Then it won't match for buffer, because it's not last operand.

[arsenm]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

We can use a different lines with a different check prefix for SI/VI, although it's probably not worth it for this. This is good enough

[nhaustov]

test/CodeGen/AMDGPU/sint_to_fp.ll

106

If this is good enough, could you ok the review?

lib/Target/AMDGPU/CIInstructions.td

 def : FlatLoadPat <FLAT_LOAD_USHORT, flat_az_extloadi16, i32>;
 def : FlatLoadPat <FLAT_LOAD_SSHORT, flat_sextloadi16, i32>;
 def : FlatLoadPat <FLAT_LOAD_DWORD, flat_load, i32>;
 def : FlatLoadPat <FLAT_LOAD_DWORDX2, flat_load, v2i32>;
 def : FlatLoadPat <FLAT_LOAD_DWORDX4, flat_load, v4i32>;
 
 class FlatStorePat <FLAT inst, SDPatternOperator node, ValueType vt> : Pat <
   (node vt:$data, i64:$addr),
-  (inst $data, $addr, 0, 0, 0)
+  (inst $addr, $data, 0, 0, 0)
 >;
 
 def : FlatStorePat <FLAT_STORE_BYTE, flat_truncstorei8, i32>;
 def : FlatStorePat <FLAT_STORE_SHORT, flat_truncstorei16, i32>;
 def : FlatStorePat <FLAT_STORE_DWORD, flat_store, i32>;
 def : FlatStorePat <FLAT_STORE_DWORDX2, flat_store, v2i32>;
 def : FlatStorePat <FLAT_STORE_DWORDX4, flat_store, v4i32>;
 

lib/Target/AMDGPU/SIInstrInfo.td

let data = 0, mayLoad = 1 in {
 
     def _vi : FLAT_Real_vi <op.VI, NAME, outs, ins, asm>;
   }
 }
 
 multiclass FLAT_Store_Helper <flat op, string asm_name,
     RegisterClass vdataClass,
     dag outs = (outs),
-    dag ins = (ins vdataClass:$data, VReg_64:$addr, glc_flat:$glc,
+    dag ins = (ins VReg_64:$addr, vdataClass:$data, glc_flat:$glc,
                    slc_flat:$slc, tfe_flat:$tfe),
-    string asm = asm_name#" $data, $addr"#"$glc"#"$slc"#"$tfe"> {
+    string asm = asm_name#" $addr, $data"#"$glc"#"$slc"#"$tfe"> {
 
   let mayLoad = 0, mayStore = 1, vdst = 0 in {
 
     def "" : FLAT_Pseudo <NAME, outs, ins, []>;
 
     def _ci : FLAT_Real_ci <op.CI, NAME, outs, ins, asm>;
 
     def _vi : FLAT_Real_vi <op.VI, NAME, outs, ins, asm>;

test/CodeGen/AMDGPU/ctlz.ll

 ; SI-DAG: v_cmp_eq_i32_e64 vcc, 0, s[[HI]]
 ; SI-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]]
 ; SI-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32
 ; SI-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]]
 ; SI-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[FFBH_LO]]
 ; SI-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]]
 ; SI-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]]
 ; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}}
-; SI: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
+; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
 define void @s_ctlz_i64(i64 addrspace(1)* noalias %out, i64 %val) nounwind {
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
   store i64 %ctlz, i64 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_ctlz_i64_trunc:
 define void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64 %val) nounwind {

 ; SI-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]]
 ; SI-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
 ; SI-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]]
 ; SI-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[ADD]], [[CMPHI]]
 ; SI-DAG: v_or_b32_e32 [[OR:v[0-9]+]], v[[LO]], v[[HI]]
 ; SI-DAG: v_cmp_eq_i32_e32 vcc, 0, [[OR]]
 ; SI-DAG: v_cndmask_b32_e64 v[[CLTZ_LO:[0-9]+]], v[[CTLZ:[0-9]+]], 64, vcc
 ; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}}
-; SI: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CLTZ_LO]]:[[CTLZ_HI]]{{\]}}
+; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CLTZ_LO]]:[[CTLZ_HI]]{{\]}}
 define void @v_ctlz_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
   store i64 %ctlz, i64 addrspace(1)* %out.gep
   ret void

test/CodeGen/AMDGPU/ctlz_zero_undef.ll

 ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i64:
 ; SI: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
 ; SI-DAG: v_cmp_eq_i32_e64 [[CMPHI:s\[[0-9]+:[0-9]+\]]], 0, v[[HI]]
 ; SI-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]]
 ; SI-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
 ; SI-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]]
 ; SI-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[FFBH_LO]]
 ; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}}
-; SI: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
+; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}}
 define void @v_ctlz_zero_undef_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
   store i64 %ctlz, i64 addrspace(1)* %out.gep
   ret void

test/CodeGen/AMDGPU/flat-address-space.ll

 
 ; CHECK-LABEL: {{^}}store_flat_i32:
 ; CHECK-DAG: s_load_dwordx2 s{{\[}}[[LO_SREG:[0-9]+]]:[[HI_SREG:[0-9]+]]],
 ; CHECK-DAG: s_load_dword s[[SDATA:[0-9]+]],
 ; CHECK: s_waitcnt lgkmcnt(0)
 ; CHECK-DAG: v_mov_b32_e32 v[[DATA:[0-9]+]], s[[SDATA]]
 ; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], s[[LO_SREG]]
 ; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], s[[HI_SREG]]
-; CHECK: flat_store_dword v[[DATA]], v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}}
+; CHECK: flat_store_dword v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}}, v[[DATA]]
 define void @store_flat_i32(i32 addrspace(1)* %gptr, i32 %x) #0 {
   %fptr = addrspacecast i32 addrspace(1)* %gptr to i32 addrspace(4)*
   store i32 %x, i32 addrspace(4)* %fptr, align 4
   ret void
 }
 
 ; CHECK-LABEL: {{^}}store_flat_i64:
 ; CHECK: flat_store_dwordx2

test/CodeGen/AMDGPU/hsa.ll

 ; HSA: .end_amd_kernel_code_t
 ; HSA: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x0
 
 ; Make sure we are setting the ATC bit:
 ; HSA-CI: s_mov_b32 s[[HI:[0-9]]], 0x100f000
 ; On VI+ we also need to set MTYPE = 2
 ; HSA-VI: s_mov_b32 s[[HI:[0-9]]], 0x1100f000
 ; Make sure we generate flat store for HSA
-; HSA: flat_store_dword v{{[0-9]+}}
+; HSA: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
 ; HSA: .Lfunc_end0:
 ; HSA: .size   simple, .Lfunc_end0-simple
 
 define void @simple(i32 addrspace(1)* %out) {
 entry:
   store i32 0, i32 addrspace(1)* %out
   ret void
 }

test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll

 ; HSA: enable_sgpr_grid_workgroup_count_y = 0
 ; HSA: enable_sgpr_grid_workgroup_count_z = 0
 ; HSA: .end_amd_kernel_code_t
 
 ; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s2{{$}}
 ; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s6{{$}}
 
 ; ALL-NOT: [[VCOPY]]
-; ALL: {{buffer|flat}}_store_dword [[VCOPY]]
+; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]]
 
 ; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6
 ; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2
 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 0
 ; ALL: COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0
 define void @test_workgroup_id_x(i32 addrspace(1)* %out) #1 {

 ; HSA: enable_sgpr_grid_workgroup_count_x = 0
 ; HSA: enable_sgpr_grid_workgroup_count_y = 0
 ; HSA: enable_sgpr_grid_workgroup_count_z = 0
 
 ; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}}
 ; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s7{{$}}
 
 ; ALL-NOT: [[VCOPY]]
-; ALL: {{buffer|flat}}_store_dword [[VCOPY]]
+; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]]
 
 ; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6
 ; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2
 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 1
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 0
 ; ALL: COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0
 define void @test_workgroup_id_y(i32 addrspace(1)* %out) #1 {

 ; HSA: enable_sgpr_grid_workgroup_count_x = 0
 ; HSA: enable_sgpr_grid_workgroup_count_y = 0
 ; HSA: enable_sgpr_grid_workgroup_count_z = 0
 
 ; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}}
 ; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s7{{$}}
 
 ; ALL-NOT: [[VCOPY]]
-; ALL: {{buffer|flat}}_store_dword [[VCOPY]]
+; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]]
 
 ; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6
 ; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2
 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0
 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 1
 ; ALL: COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0
 define void @test_workgroup_id_z(i32 addrspace(1)* %out) #1 {
   %id = call i32 @llvm.amdgcn.workgroup.id.z()
   store i32 %id, i32 addrspace(1)* %out
   ret void
 }
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }

test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll

 ; MESA: .section .AMDGPU.config
 ; MESA: .long 47180
 ; MESA-NEXT: .long 132{{$}}
 
 ; ALL-LABEL {{^}}test_workitem_id_x:
 ; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 0
 
 ; ALL-NOT: v0
-; ALL: {{buffer|flat}}_store_dword v0
+; ALL: {{buffer|flat}}_store_dword {{.*}}v0
 define void @test_workitem_id_x(i32 addrspace(1)* %out) #1 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   store i32 %id, i32 addrspace(1)* %out
   ret void
 }
 
 ; MESA: .section .AMDGPU.config
 ; MESA: .long 47180
 ; MESA-NEXT: .long 2180{{$}}
 
 ; ALL-LABEL {{^}}test_workitem_id_y:
 ; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 1
 
 ; ALL-NOT: v1
-; ALL: {{buffer|flat}}_store_dword v1
+; ALL: {{buffer|flat}}_store_dword {{.*}}v1
 define void @test_workitem_id_y(i32 addrspace(1)* %out) #1 {
   %id = call i32 @llvm.amdgcn.workitem.id.y()
   store i32 %id, i32 addrspace(1)* %out
   ret void
 }
 
 ; MESA: .section .AMDGPU.config
 ; MESA: .long 47180
 ; MESA-NEXT: .long 4228{{$}}
 
 ; ALL-LABEL {{^}}test_workitem_id_z:
 ; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 2
 
 ; ALL-NOT: v2
-; ALL: {{buffer|flat}}_store_dword v2
+; ALL: {{buffer|flat}}_store_dword {{.*}}v2
 define void @test_workitem_id_z(i32 addrspace(1)* %out) #1 {
   %id = call i32 @llvm.amdgcn.workitem.id.z()
   store i32 %id, i32 addrspace(1)* %out
   ret void
 }
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }

test/CodeGen/AMDGPU/salu-to-valu.ll

 }
 
 ; Test moving an SMRD instruction to the VALU
 
 ; GCN-LABEL: {{^}}smrd_valu:
 ; FIXME: We should be using flat load for HSA.
 ; GCN: buffer_load_dword [[OUT:v[0-9]+]]
 ; GCN-NOHSA: buffer_store_dword [[OUT]]
-; GCN-HSA: flat_store_dword [[OUT]]
+; GCN-HSA: flat_store_dword {{.*}}, [[OUT]]
 define void @smrd_valu(i32 addrspace(2)* addrspace(1)* %in, i32 %a, i32 %b, i32 addrspace(1)* %out) #1 {
 entry:
   %tmp = icmp ne i32 %a, 0
   br i1 %tmp, label %if, label %else
 
 if:                                               ; preds = %entry
   %tmp1 = load i32 addrspace(2)*, i32 addrspace(2)* addrspace(1)* %in
   br label %endif

 ; GCN-LABEL: {{^}}smrd_valu_ci_offset:
 ; GCN-NOHSA-NOT: v_add
 ; GCN-NOHSA: s_movk_i32 [[OFFSET:s[0-9]+]], 0x4e20{{$}}
 ; GCN-NOHSA-NOT: v_add
 ; GCN-NOHSA: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET]] addr64{{$}}
 ; GCN-NOHSA: v_add_i32_e32
 ; GCN-NOHSA: buffer_store_dword
 ; GCN-HSA: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
-; GCN-HSA: flat_store_dword v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN-HSA: flat_store_dword v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}
 define void @smrd_valu_ci_offset(i32 addrspace(1)* %out, i32 addrspace(2)* %in, i32 %c) #1 {
 entry:
   %tmp = call i32 @llvm.r600.read.tidig.x() #0
   %tmp2 = getelementptr i32, i32 addrspace(2)* %in, i32 %tmp
   %tmp3 = getelementptr i32, i32 addrspace(2)* %tmp2, i32 5000
   %tmp4 = load i32, i32 addrspace(2)* %tmp3
   %tmp5 = add i32 %tmp4, %c
   store i32 %tmp5, i32 addrspace(1)* %out
entry:
   ret void
 }
 
 ; GCN-LABEL: {{^}}smrd_valu2_salu_user:
 ; GCN-NOHSA: buffer_load_dword [[MOVED:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
 ; GCN-HSA: flat_load_dword [[MOVED:v[0-9]+]], v[{{[0-9+:[0-9]+}}]
 ; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, s{{[0-9]+}}, [[MOVED]]
 ; GCN-NOHSA: buffer_store_dword [[ADD]]
-; GCN-HSA: flat_store_dword [[ADD]]
+; GCN-HSA: flat_store_dword {{.*}}, [[ADD]]
 define void @smrd_valu2_salu_user(i32 addrspace(1)* %out, [8 x i32] addrspace(2)* %in, i32 %a) #1 {
 entry:
   %tmp = call i32 @llvm.r600.read.tidig.x() #0
   %tmp1 = add i32 %tmp, 4
   %tmp2 = getelementptr [8 x i32], [8 x i32] addrspace(2)* %in, i32 %tmp, i32 4
   %tmp3 = load i32, i32 addrspace(2)* %tmp2
   %tmp4 = add i32 %tmp3, %a
   store i32 %tmp4, i32 addrspace(1)* %out

test/CodeGen/AMDGPU/sint_to_fp.i64.ll

 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_i64
 ; GCN-DAG: v_cmp_lt_u64
 
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 ; GCN: v_cndmask_b32_e32 [[SIGN_SEL:v[0-9]+]],
-; GCN: {{buffer|flat}}_store_dword [[SIGN_SEL]]
+; GCN: {{buffer|flat}}_store_dword {{.*}}[[SIGN_SEL]]
 define void @v_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep
   %result = sitofp i64 %val to float
   store float %result, float addrspace(1)* %out.gep
   ret void

test/CodeGen/AMDGPU/sint_to_fp.ll

define void @s_sint_to_fp_i1_f32_load(float addrspace(1)* %out, i1 %in) #0 {
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_sint_to_fp_i1_f32_load:
 ; SI: {{buffer|flat}}_load_ubyte
 ; SI: v_and_b32_e32 {{v[0-9]+}}, 1, {{v[0-9]+}}
 ; SI: v_cmp_eq_i32
 ; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1.0
-; SI: {{buffer|flat}}_store_dword [[RESULT]],
+; SI: {{buffer|flat}}_store_dword {{.*}}[[RESULT]]
 ; SI: s_endpgm
 define void @v_sint_to_fp_i1_f32_load(float addrspace(1)* %out, i1 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i1, i1 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i1, i1 addrspace(1)* %in.gep
   %fp = sitofp i1 %val to float
   store float %fp, float addrspace(1)* %out.gep
   ret void
 }
 
 declare i32 @llvm.r600.read.tidig.x() #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }

test/CodeGen/AMDGPU/sra.ll

define void @s_ashr_32_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
   store i64 %add, i64 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_ashr_32_i64:
 ; SI: buffer_load_dword v[[HI:[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; VI: flat_load_dword v[[HI:[0-9]+]]
 ; GCN: v_ashrrev_i32_e32 v[[SHIFT:[0-9]+]], 31, v[[HI]]
-; GCN: {{buffer|flat}}_store_dwordx2 v{{\[}}[[HI]]:[[SHIFT]]{{\]}}
+; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[HI]]:[[SHIFT]]{{\]}}
 define void @v_ashr_32_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() #0
   %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
   %a = load i64, i64 addrspace(1)* %gep.in
   %result = ashr i64 %a, 32
   store i64 %result, i64 addrspace(1)* %gep.out
   ret void
define void @s_ashr_63_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_ashr_63_i64:
 ; SI: buffer_load_dword v[[HI:[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; VI: flat_load_dword v[[HI:[0-9]+]]
 ; GCN: v_ashrrev_i32_e32 v[[SHIFT:[0-9]+]], 31, v[[HI]]
 ; GCN: v_mov_b32_e32 v[[COPY:[0-9]+]], v[[SHIFT]]
-; GCN: {{buffer|flat}}_store_dwordx2 v{{\[}}[[SHIFT]]:[[COPY]]{{\]}}
+; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[SHIFT]]:[[COPY]]{{\]}}
 define void @v_ashr_63_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) {
   %tid = call i32 @llvm.r600.read.tidig.x() #0
   %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %gep.out = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
   %a = load i64, i64 addrspace(1)* %gep.in
   %result = ashr i64 %a, 63
   store i64 %result, i64 addrspace(1)* %gep.out
   ret void
 }
 
 attributes #0 = { nounwind readnone }

test/CodeGen/AMDGPU/uint_to_fp.i64.ll

 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_i64
 ; GCN-DAG: v_cmp_lt_u64
 
 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
-; GCN: {{buffer|flat}}_store_dword [[VR]]
+; GCN: {{buffer|flat}}_store_dword {{.*}}[[VR]]
 define void @v_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep
   %result = uitofp i64 %val to float
   store float %result, float addrspace(1)* %out.gep
   ret void

test/CodeGen/AMDGPU/uint_to_fp.ll

define void @s_uint_to_fp_i1_to_f32_load(float addrspace(1)* %out, i1 %in) #0 {
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_uint_to_fp_i1_f32_load:
 ; SI: {{buffer|flat}}_load_ubyte
 ; SI: v_and_b32_e32 {{v[0-9]+}}, 1, {{v[0-9]+}}
 ; SI: v_cmp_eq_i32
 ; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0
-; SI: {{buffer|flat}}_store_dword [[RESULT]],
+; SI: {{buffer|flat}}_store_dword {{.*}}[[RESULT]]
 ; SI: s_endpgm
 define void @v_uint_to_fp_i1_f32_load(float addrspace(1)* %out, i1 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i1, i1 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i1, i1 addrspace(1)* %in.gep
   %fp = uitofp i1 %val to float
   store float %fp, float addrspace(1)* %out.gep

test/MC/AMDGPU/flat.s

 // CI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
 // VI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x53,0xdc,0x03,0x00,0x80,0x01]
 
 flat_load_dword v1, v[3:4] tfe slc glc
 // NOSI: error:
 // CI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x33,0xdc,0x03,0x00,0x80,0x01]
 // VI: flat_load_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x53,0xdc,0x03,0x00,0x80,0x01]
 
-flat_store_dword v1, v[3:4]
+flat_store_dword v[3:4], v1
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4] glc
+flat_store_dword v[3:4], v1 glc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4] glc slc
+flat_store_dword v[3:4], v1 glc slc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4] glc tfe
+flat_store_dword v[3:4], v1 glc tfe
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] glc slc tfe
+flat_store_dword v[3:4], v1 glc slc tfe
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] glc tfe slc
+flat_store_dword v[3:4], v1 glc tfe slc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] slc
+flat_store_dword v[3:4], v1 slc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] slc ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 slc ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4] slc glc
+flat_store_dword v[3:4], v1 slc glc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4] slc tfe
+flat_store_dword v[3:4], v1 slc tfe
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] slc glc tfe
+flat_store_dword v[3:4], v1 slc glc tfe
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] slc tfe glc
+flat_store_dword v[3:4], v1 slc tfe glc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] tfe
+flat_store_dword v[3:4], v1 tfe
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] tfe ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 tfe ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] tfe glc
+flat_store_dword v[3:4], v1 tfe glc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc tfe ; encoding: [0x00,0x00,0x71,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] tfe slc
+flat_store_dword v[3:4], v1 tfe slc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 slc tfe ; encoding: [0x00,0x00,0x72,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] tfe glc slc
+flat_store_dword v[3:4], v1 tfe glc slc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
-flat_store_dword v1, v[3:4] tfe slc glc
+flat_store_dword v[3:4], v1 tfe slc glc
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
+// CIVI: flat_store_dword v[3:4], v1 glc slc tfe ; encoding: [0x00,0x00,0x73,0xdc,0x03,0x01,0x80,0x00]
 
 // FIXME: For atomic instructions, glc must be placed immediately following
 // the data regiser.  These forms aren't currently supported:
 // flat_atomic_add v1, v[3:4], v5 slc glc
 // flat_atomic_add v1, v[3:4], v5 slc glc tfe
 // flat_atomic_add v1, v[3:4], v5 slc tfe glc
 // flat_atomic_add v1, v[3:4], v5 tfe glc
 // flat_atomic_add v[3:4], v5 tfe glc

 // CI: flat_load_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x38,0xdc,0x03,0x00,0x00,0x05]
 // VI: flat_load_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x5c,0xdc,0x03,0x00,0x00,0x05]
 
 flat_load_dwordx3 v[5:7], v[3:4]
 // NOSI: error:
 // CI: flat_load_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x3c,0xdc,0x03,0x00,0x00,0x05]
 // VI: flat_load_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x58,0xdc,0x03,0x00,0x00,0x05]
 
-flat_store_byte v1, v[3:4]
+flat_store_byte v[3:4], v1
 // NOSI: error:
-// CIVI: flat_store_byte v1, v[3:4] ; encoding: [0x00,0x00,0x60,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_byte v[3:4], v1 ; encoding: [0x00,0x00,0x60,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_short v1, v[3:4]
+flat_store_short v[3:4], v1
 // NOSI: error:
-// CIVI: flat_store_short v1, v[3:4] ; encoding: [0x00,0x00,0x68,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_short v[3:4], v1 ; encoding: [0x00,0x00,0x68,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dword v1, v[3:4]
+flat_store_dword v[3:4], v1
 // NOSI: error:
-// CIVI: flat_store_dword v1, v[3:4] ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dword v[3:4], v1 ; encoding: [0x00,0x00,0x70,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dwordx2 v[1:2], v[3:4]
+flat_store_dwordx2 v[3:4], v[1:2]
 // NOSI: error:
-// CIVI: flat_store_dwordx2 v[1:2], v[3:4] ; encoding: [0x00,0x00,0x74,0xdc,0x03,0x01,0x00,0x00]
+// CIVI: flat_store_dwordx2 v[3:4], v[1:2] ; encoding: [0x00,0x00,0x74,0xdc,0x03,0x01,0x00,0x00]
 
-flat_store_dwordx4 v[5:8], v[3:4]
+flat_store_dwordx4 v[3:4], v[5:8]
 // NOSI: error:
-// CI: flat_store_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x78,0xdc,0x03,0x05,0x00,0x00]
-// VI: flat_store_dwordx4 v[5:8], v[3:4] ; encoding: [0x00,0x00,0x7c,0xdc,0x03,0x05,0x00,0x00]
+// CI: flat_store_dwordx4 v[3:4], v[5:8] ; encoding: [0x00,0x00,0x78,0xdc,0x03,0x05,0x00,0x00]
+// VI: flat_store_dwordx4 v[3:4], v[5:8] ; encoding: [0x00,0x00,0x7c,0xdc,0x03,0x05,0x00,0x00]
 
-flat_store_dwordx3 v[5:7], v[3:4]
+flat_store_dwordx3 v[3:4], v[5:7]
 // NOSI: error:
-// CI: flat_store_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x7c,0xdc,0x03,0x05,0x00,0x00]
-// VI: flat_store_dwordx3 v[5:7], v[3:4] ; encoding: [0x00,0x00,0x78,0xdc,0x03,0x05,0x00,0x00]
+// CI: flat_store_dwordx3 v[3:4], v[5:7] ; encoding: [0x00,0x00,0x7c,0xdc,0x03,0x05,0x00,0x00]
+// VI: flat_store_dwordx3 v[3:4], v[5:7] ; encoding: [0x00,0x00,0x78,0xdc,0x03,0x05,0x00,0x00]
 
 flat_atomic_swap v[3:4], v5
 // NOSI: error:
 // CI: flat_atomic_swap v[3:4], v5 ; encoding: [0x00,0x00,0xc0,0xdc,0x03,0x05,0x00,0x00]
 // VI: flat_atomic_swap v[3:4], v5 ; encoding: [0x00,0x00,0x00,0xdd,0x03,0x05,0x00,0x00]
 
 flat_atomic_swap v1, v[3:4], v5 glc
 // NOSI: error: