diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -531,37 +531,47 @@ } // End let AddressSpaces } // End foreach as - multiclass ret_noret_binary_atomic_op { + let PredicateCode = [{ return (SDValue(N, 0).use_empty()); }], + GISelPredicateCode = [{ + return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); + }] + in { + defm "_noret" : binary_atomic_op; + } + + let PredicateCode = [{ return !(SDValue(N, 0).use_empty()); }], + GISelPredicateCode = [{ + return !MRI.use_nodbg_empty(MI.getOperand(0).getReg()); + }] + in { + defm "_ret" : binary_atomic_op; + } +} + +multiclass binary_atomic_op_all_as { foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in { let AddressSpaces = !cast("LoadAddress_"#as).AddrSpaces in { defm "_"#as : binary_atomic_op; - - let PredicateCode = [{return (SDValue(N, 0).use_empty());}] in { - defm "_"#as#"_noret" : binary_atomic_op; - } - - let PredicateCode = [{return !(SDValue(N, 0).use_empty());}] in { - defm "_"#as#"_ret" : binary_atomic_op; - } + defm "_"#as : ret_noret_binary_atomic_op; } } } -defm atomic_swap : ret_noret_binary_atomic_op; -defm atomic_load_add : ret_noret_binary_atomic_op; -defm atomic_load_and : ret_noret_binary_atomic_op; -defm atomic_load_max : ret_noret_binary_atomic_op; -defm atomic_load_min : ret_noret_binary_atomic_op; -defm atomic_load_or : ret_noret_binary_atomic_op; -defm atomic_load_sub : ret_noret_binary_atomic_op; -defm atomic_load_umax : ret_noret_binary_atomic_op; -defm atomic_load_umin : ret_noret_binary_atomic_op; -defm atomic_load_xor : ret_noret_binary_atomic_op; -defm atomic_load_fadd : ret_noret_binary_atomic_op; +defm atomic_swap : binary_atomic_op_all_as; +defm atomic_load_add : binary_atomic_op_all_as; +defm atomic_load_and : binary_atomic_op_all_as; +defm atomic_load_max : binary_atomic_op_all_as; +defm atomic_load_min : binary_atomic_op_all_as; +defm atomic_load_or : binary_atomic_op_all_as; +defm atomic_load_sub : binary_atomic_op_all_as; +defm atomic_load_umax : binary_atomic_op_all_as; +defm atomic_load_umin : binary_atomic_op_all_as; +defm atomic_load_xor : binary_atomic_op_all_as; +defm atomic_load_fadd : binary_atomic_op_all_as; let MemoryVT = v2f16 in -defm atomic_load_fadd_v2f16 : ret_noret_binary_atomic_op; -defm AMDGPUatomic_cmp_swap : ret_noret_binary_atomic_op; +defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as; +defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as; def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>, Aligned<8> { diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -103,7 +103,7 @@ let has_vdst = 0; } -class DS_1A1D_NORET +class DS_1A1D_WRITE : DS_Pseudo.ret:$data0, offset:$offset, gds:$gds), @@ -114,6 +114,28 @@ let IsAtomicNoRet = 1; } +multiclass DS_1A1D_WRITE_mc { + def "" : DS_1A1D_WRITE, + AtomicNoRet; + + let has_m0_read = 0 in { + def _gfx9 : DS_1A1D_WRITE, + AtomicNoRet; + } +} + +class DS_1A1D_NORET.ret> +: DS_Pseudo.ret:$data0, offset:$offset, gds:$gds), + " $addr, $data0$offset$gds"> { + + let has_data1 = 0; + let has_vdst = 0; + let IsAtomicNoRet = 1; +} + multiclass DS_1A1D_NORET_mc { def "" : DS_1A1D_NORET, AtomicNoRet; @@ -436,9 +458,9 @@ defm DS_MAX_F32 : DS_1A1D_NORET_mc<"ds_max_f32">; let mayLoad = 0 in { -defm DS_WRITE_B8 : DS_1A1D_NORET_mc<"ds_write_b8">; -defm DS_WRITE_B16 : DS_1A1D_NORET_mc<"ds_write_b16">; -defm DS_WRITE_B32 : DS_1A1D_NORET_mc<"ds_write_b32">; +defm DS_WRITE_B8 : DS_1A1D_WRITE_mc<"ds_write_b8">; +defm DS_WRITE_B16 : DS_1A1D_WRITE_mc<"ds_write_b16">; +defm DS_WRITE_B32 : DS_1A1D_WRITE_mc<"ds_write_b32">; defm DS_WRITE2_B32 : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">; defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">; @@ -446,8 +468,8 @@ let has_m0_read = 0 in { let SubtargetPredicate = HasD16LoadStore in { -def DS_WRITE_B8_D16_HI : DS_1A1D_NORET<"ds_write_b8_d16_hi">; -def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">; +def DS_WRITE_B8_D16_HI : DS_1A1D_WRITE<"ds_write_b8_d16_hi">; +def DS_WRITE_B16_D16_HI : DS_1A1D_WRITE<"ds_write_b16_d16_hi">; } } // End has_m0_read = 0 @@ -481,7 +503,7 @@ defm DS_XOR_B64 : DS_1A1D_NORET_mc<"ds_xor_b64", VReg_64>; defm DS_MSKOR_B64 : DS_1A2D_NORET_mc<"ds_mskor_b64", VReg_64>; let mayLoad = 0 in { -defm DS_WRITE_B64 : DS_1A1D_NORET_mc<"ds_write_b64", VReg_64>; +defm DS_WRITE_B64 : DS_1A1D_WRITE_mc<"ds_write_b64", VReg_64>; defm DS_WRITE2_B64 : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VReg_64>; defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VReg_64>; } @@ -641,8 +663,8 @@ } // End mayStore = 0 let mayLoad = 0 in { -defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", VReg_96>; -defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", VReg_128>; +defm DS_WRITE_B96 : DS_1A1D_WRITE_mc<"ds_write_b96", VReg_96>; +defm DS_WRITE_B128 : DS_1A1D_WRITE_mc<"ds_write_b128", VReg_128>; } // End mayLoad = 0 def DS_NOP : DS_VOID<"ds_nop">; @@ -892,6 +914,9 @@ (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 gds)) >; +// FIXME: (How) Should we write a GCNPat for DSAtomicNoRetPat? Currently noret +// atomic patterns are derived from DSAtomicRetPat. + multiclass DSAtomicRetPat_mc { let OtherPredicates = [LDSRequiresM0Init] in { def : DSAtomicRetPat(frag#"_local_m0_"#vt.Size)>; @@ -905,6 +930,24 @@ def : DSAtomicRetPat(frag#"_region_m0_"#vt.Size), 1>; } +multiclass DSAtomicRetNoRetPat_mc { + let OtherPredicates = [LDSRequiresM0Init] in { + def : DSAtomicRetPat(frag#"_local_m0_ret_"#vt.Size)>; + def : DSAtomicRetPat(frag#"_local_m0_noret_"#vt.Size)>; + } + + let OtherPredicates = [NotLDSRequiresM0Init] in { + def : DSAtomicRetPat(!cast(inst)#"_gfx9"), vt, + !cast(frag#"_local_ret_"#vt.Size)>; + def : DSAtomicRetPat(!cast(noRetInst)#"_gfx9"), vt, + !cast(frag#"_local_noret_"#vt.Size)>; + } + + def : DSAtomicRetPat(frag#"_region_m0_ret_"#vt.Size), 1>; + def : DSAtomicRetPat(frag#"_region_m0_noret_"#vt.Size), 1>; +} + class DSAtomicCmpXChg : GCNPat < @@ -929,45 +972,46 @@ // 32-bit atomics. defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; defm : DSAtomicCmpXChg_mc; let SubtargetPredicate = HasLDSFPAtomicAdd in { -defm : DSAtomicRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; } // 64-bit atomics. defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; -defm : DSAtomicRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; +defm : DSAtomicRetNoRetPat_mc; defm : DSAtomicCmpXChg_mc; let SubtargetPredicate = isGFX90APlus in { -def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; } def : Pat < diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -291,19 +291,10 @@ // PatFrags for global memory operations //===----------------------------------------------------------------------===// -foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in { -let AddressSpaces = !cast("LoadAddress_"#as).AddrSpaces in { - - -defm atomic_inc_#as : binary_atomic_op; -defm atomic_dec_#as : binary_atomic_op; -defm atomic_load_fmin_#as : binary_atomic_op; -defm atomic_load_fmax_#as : binary_atomic_op; - - -} // End let AddressSpaces = ... -} // End foreach AddrSpace - +defm atomic_inc : binary_atomic_op_all_as; +defm atomic_dec : binary_atomic_op_all_as; +defm atomic_load_fmin : binary_atomic_op_all_as; +defm atomic_load_fmax : binary_atomic_op_all_as; //===----------------------------------------------------------------------===// // SDNodes PatFrags for loads/stores with a glue input. @@ -686,10 +677,12 @@ let AddressSpaces = StoreAddress_local.AddrSpaces in { defm _local_m0 : binary_atomic_op (NAME#"_glue"), IsInt>; + defm _local_m0 : ret_noret_binary_atomic_op (NAME#"_glue"), IsInt>; } let AddressSpaces = StoreAddress_region.AddrSpaces in { defm _region_m0 : binary_atomic_op (NAME#"_glue"), IsInt>; + defm _region_m0 : ret_noret_binary_atomic_op (NAME#"_glue"), IsInt>; } } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll @@ -530,7 +530,7 @@ ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s4 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] -; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v2, v[0:1] +; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_endpgm main_body: %ret = call double @llvm.amdgcn.ds.fadd.f64(double addrspace(3)* %ptr, double %data, i32 0, i32 0, i1 0) @@ -560,7 +560,7 @@ ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v2, v[0:1] +; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_endpgm main_body: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir @@ -1,11 +1,11 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s -# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s +# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX10 %s # GFX6/7 selection should fail. # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s -# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s +# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX7 %s --- name: atomicrmw_fadd_s32_local @@ -18,24 +18,42 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_local ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) - ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX9-LABEL: name: atomicrmw_fadd_s32_local ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) - ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] + ; GFX10-LABEL: name: atomicrmw_fadd_s32_local + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX10-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] ; GFX6-LABEL: name: atomicrmw_fadd_s32_local ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) - ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_local + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX7-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (s32), addrspace 3) @@ -54,21 +72,37 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_local_noret ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) ; GFX9-LABEL: name: atomicrmw_fadd_s32_local_noret ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX10-LABEL: name: atomicrmw_fadd_s32_local_noret + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) ; GFX6-LABEL: name: atomicrmw_fadd_s32_local_noret ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_local_noret + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (s32), addrspace 3) @@ -86,26 +120,46 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_local_gep4 ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) - ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX9-LABEL: name: atomicrmw_fadd_s32_local_gep4 ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 4, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) - ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 4, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] + ; GFX10-LABEL: name: atomicrmw_fadd_s32_local_gep4 + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 4, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3) + ; GFX10-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]] ; GFX6-LABEL: name: atomicrmw_fadd_s32_local_gep4 ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 - ; GFX6: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[C]](s32) - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) - ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 + ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[C]](s32) + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_local_gep4 + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 + ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[C]](s32) + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3) + ; GFX7-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_CONSTANT i32 4 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir @@ -1,11 +1,11 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s -# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s +# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX10 %s # GFX6/7 selection should fail. # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s -# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s +# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX7 %s --- name: atomicrmw_fadd_s32_region @@ -18,24 +18,42 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_region ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) - ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX9-LABEL: name: atomicrmw_fadd_s32_region ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) - ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX10-LABEL: name: atomicrmw_fadd_s32_region + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX10-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX6-LABEL: name: atomicrmw_fadd_s32_region ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) - ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_region + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX7-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) %0:vgpr(p2) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (s32), addrspace 2) @@ -54,21 +72,37 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_noret ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_noret ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX10-LABEL: name: atomicrmw_fadd_s32_region_noret + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_noret ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_region_noret + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) %0:vgpr(p2) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (s32), addrspace 2) @@ -86,26 +120,46 @@ ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_gep4 ; GFX8: liveins: $vgpr0, $vgpr1 - ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8: $m0 = S_MOV_B32 -1 - ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) - ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX8-NEXT: {{ $}} + ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_gep4 ; GFX9: liveins: $vgpr0, $vgpr1 - ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) - ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX9-NEXT: {{ $}} + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] + ; GFX10-LABEL: name: atomicrmw_fadd_s32_region_gep4 + ; GFX10: liveins: $vgpr0, $vgpr1 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2) + ; GFX10-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]] ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_gep4 ; GFX6: liveins: $vgpr0, $vgpr1 - ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 - ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; GFX6: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 - ; GFX6: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](s32) - ; GFX6: $m0 = S_MOV_B32 -1 - ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) - ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX6-NEXT: {{ $}} + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 + ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](s32) + ; GFX6-NEXT: $m0 = S_MOV_B32 -1 + ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; GFX7-LABEL: name: atomicrmw_fadd_s32_region_gep4 + ; GFX7: liveins: $vgpr0, $vgpr1 + ; GFX7-NEXT: {{ $}} + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 + ; GFX7-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4 + ; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](s32) + ; GFX7-NEXT: $m0 = S_MOV_B32 -1 + ; GFX7-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2) + ; GFX7-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) %0:vgpr(p2) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = G_CONSTANT i32 4 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll @@ -139,7 +139,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v1, s0 -; CI-NEXT: ds_dec_rtn_u32 v0, v1, v0 +; CI-NEXT: ds_dec_u32 v1, v0 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_dec_noret_i32: @@ -149,7 +149,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s0 -; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0 +; VI-NEXT: ds_dec_u32 v1, v0 ; VI-NEXT: s_endpgm %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false) ret void @@ -163,7 +163,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v1, s0 -; CI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16 +; CI-NEXT: ds_dec_u32 v1, v0 offset:16 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_dec_noret_i32_offset: @@ -173,7 +173,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s0 -; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16 +; VI-NEXT: ds_dec_u32 v1, v0 offset:16 ; VI-NEXT: s_endpgm ; GFX9-LABEL: lds_atomic_dec_noret_i32_offset: ; GFX9: ; %bb.0: @@ -1291,7 +1291,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v2, s0 -; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] +; CI-NEXT: ds_dec_u64 v2, v[0:1] ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_dec_noret_i64: @@ -1302,7 +1302,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v2, s0 -; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] +; VI-NEXT: ds_dec_u64 v2, v[0:1] ; VI-NEXT: s_endpgm ; GFX9-LABEL: lds_atomic_dec_noret_i64: ; GFX9: ; %bb.0: @@ -1326,7 +1326,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v2, s0 -; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; CI-NEXT: ds_dec_u64 v2, v[0:1] offset:32 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_dec_noret_i64_offset: @@ -1337,7 +1337,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v2, s0 -; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; VI-NEXT: ds_dec_u64 v2, v[0:1] offset:32 ; VI-NEXT: s_endpgm ; GFX9-LABEL: lds_atomic_dec_noret_i64_offset: ; GFX9: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll @@ -153,7 +153,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v1, s0 -; CI-NEXT: ds_inc_rtn_u32 v0, v1, v0 +; CI-NEXT: ds_inc_u32 v1, v0 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_inc_noret_i32: @@ -163,7 +163,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s0 -; VI-NEXT: ds_inc_rtn_u32 v0, v1, v0 +; VI-NEXT: ds_inc_u32 v1, v0 ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: lds_atomic_inc_noret_i32: @@ -172,7 +172,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s0 -; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v1 +; GFX9-NEXT: ds_inc_u32 v0, v1 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: lds_atomic_inc_noret_i32: @@ -181,7 +181,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v1 +; GFX10-NEXT: ds_inc_u32 v0, v1 ; GFX10-NEXT: s_endpgm %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false) ret void @@ -195,7 +195,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v1, s0 -; CI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 +; CI-NEXT: ds_inc_u32 v1, v0 offset:16 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_inc_noret_i32_offset: @@ -205,7 +205,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s0 -; VI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 +; VI-NEXT: ds_inc_u32 v1, v0 offset:16 ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: lds_atomic_inc_noret_i32_offset: @@ -214,7 +214,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, s0 -; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 +; GFX9-NEXT: ds_inc_u32 v1, v0 offset:16 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: lds_atomic_inc_noret_i32_offset: @@ -223,7 +223,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, s0 -; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 +; GFX10-NEXT: ds_inc_u32 v1, v0 offset:16 ; GFX10-NEXT: s_endpgm %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %gep, i32 42, i32 0, i32 0, i1 false) @@ -773,7 +773,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v2, s0 -; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] +; CI-NEXT: ds_inc_u64 v2, v[0:1] ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_inc_noret_i64: @@ -784,7 +784,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v2, s0 -; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] +; VI-NEXT: ds_inc_u64 v2, v[0:1] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: lds_atomic_inc_noret_i64: @@ -794,7 +794,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v2, s0 -; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] +; GFX9-NEXT: ds_inc_u64 v2, v[0:1] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: lds_atomic_inc_noret_i64: @@ -804,7 +804,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v2, s0 -; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] +; GFX10-NEXT: ds_inc_u64 v2, v[0:1] ; GFX10-NEXT: s_endpgm %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3i64(i64 addrspace(3)* %ptr, i64 42, i32 0, i32 0, i1 false) ret void @@ -819,7 +819,7 @@ ; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_mov_b32_e32 v2, s0 -; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; CI-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; CI-NEXT: s_endpgm ; ; VI-LABEL: lds_atomic_inc_noret_i64_offset: @@ -830,7 +830,7 @@ ; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v2, s0 -; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; VI-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: lds_atomic_inc_noret_i64_offset: @@ -840,7 +840,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v2, s0 -; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; GFX9-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: lds_atomic_inc_noret_i64_offset: @@ -850,7 +850,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v2, s0 -; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 +; GFX10-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; GFX10-NEXT: s_endpgm %gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3i64(i64 addrspace(3)* %gep, i64 42, i32 0, i32 0, i1 false) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fadd.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fadd.ll @@ -68,21 +68,21 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_add_f32 v0, v1 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fadd_f32_ss_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v1, s3 -; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_add_f32 v0, v1 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: ds_fadd_f32_ss_nortn: ; GFX10: ; %bb.0: ; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v1, s3 -; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX10-NEXT: ds_add_f32 v0, v1 ; GFX10-NEXT: s_endpgm %unused = call float @llvm.amdgcn.ds.fadd(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret void @@ -94,21 +94,21 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s3 ; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_add_rtn_f32 v0, v1, v0 offset:512 +; GFX8-NEXT: ds_add_f32 v1, v0 offset:512 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fadd_f32_ss_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s3 ; GFX9-NEXT: v_mov_b32_e32 v1, s2 -; GFX9-NEXT: ds_add_rtn_f32 v0, v1, v0 offset:512 +; GFX9-NEXT: ds_add_f32 v1, v0 offset:512 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: ds_fadd_f32_ss_offset_nortn: ; GFX10: ; %bb.0: ; GFX10-NEXT: v_mov_b32_e32 v0, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: ds_add_rtn_f32 v0, v1, v0 offset:512 +; GFX10-NEXT: ds_add_f32 v1, v0 offset:512 ; GFX10-NEXT: s_endpgm %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %unused = call float @llvm.amdgcn.ds.fadd(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) @@ -175,14 +175,14 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_add_f32 v0, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fadd_f32_vv_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_add_f32 v0, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -190,7 +190,7 @@ ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v1 +; GFX10-NEXT: ds_add_f32 v0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] %ret = call float @llvm.amdgcn.ds.fadd(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) @@ -202,14 +202,14 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:512 +; GFX8-NEXT: ds_add_f32 v0, v1 offset:512 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fadd_f32_vv_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:512 +; GFX9-NEXT: ds_add_f32 v0, v1 offset:512 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -217,7 +217,7 @@ ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:512 +; GFX10-NEXT: ds_add_f32 v0, v1 offset:512 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmax.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmax.ll @@ -27,25 +27,27 @@ ; GFX9-NEXT: ; return to shader part epilog ; GFX8-MIR-LABEL: name: ds_fmax_f32_ss ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $sgpr2, $sgpr3 - ; GFX8-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX8-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY2]], [[COPY3]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX8-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] - ; GFX8-MIR: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; GFX8-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY2]], [[COPY3]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX8-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] + ; GFX8-MIR-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_ss ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $sgpr2, $sgpr3 - ; GFX9-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY2]], [[COPY3]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX9-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] - ; GFX9-MIR: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; GFX9-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX9-MIR-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY2]], [[COPY3]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX9-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] + ; GFX9-MIR-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret float %ret } @@ -69,25 +71,27 @@ ; GFX9-NEXT: ; return to shader part epilog ; GFX8-MIR-LABEL: name: ds_fmax_f32_ss_offset ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $sgpr2, $sgpr3 - ; GFX8-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY3]], [[COPY2]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX8-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] - ; GFX8-MIR: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; GFX8-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX8-MIR-NEXT: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY3]], [[COPY2]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX8-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] + ; GFX8-MIR-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_ss_offset ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $sgpr2, $sgpr3 - ; GFX9-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY3]], [[COPY2]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX9-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] - ; GFX9-MIR: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; GFX9-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX9-MIR-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY3]], [[COPY2]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX9-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] + ; GFX9-MIR-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) ret float %ret @@ -99,34 +103,36 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_max_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_max_f32 v0, v1 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fmax_f32_ss_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v1, s3 -; GFX9-NEXT: ds_max_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_max_f32 v0, v1 ; GFX9-NEXT: s_endpgm ; GFX8-MIR-LABEL: name: ds_fmax_f32_ss_nortn ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $sgpr2, $sgpr3 - ; GFX8-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX8-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY2]], [[COPY3]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX8-MIR: S_ENDPGM 0 + ; GFX8-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_F32 [[COPY2]], [[COPY3]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX8-MIR-NEXT: S_ENDPGM 0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_ss_nortn ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $sgpr2, $sgpr3 - ; GFX9-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY2]], [[COPY3]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX9-MIR: S_ENDPGM 0 + ; GFX9-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX9-MIR-NEXT: [[DS_MAX_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_F32_gfx9 [[COPY2]], [[COPY3]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX9-MIR-NEXT: S_ENDPGM 0 %unused = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret void } @@ -137,34 +143,36 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s3 ; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_max_rtn_f32 v0, v1, v0 offset:512 +; GFX8-NEXT: ds_max_f32 v1, v0 offset:512 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fmax_f32_ss_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s3 ; GFX9-NEXT: v_mov_b32_e32 v1, s2 -; GFX9-NEXT: ds_max_rtn_f32 v0, v1, v0 offset:512 +; GFX9-NEXT: ds_max_f32 v1, v0 offset:512 ; GFX9-NEXT: s_endpgm ; GFX8-MIR-LABEL: name: ds_fmax_f32_ss_offset_nortn ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $sgpr2, $sgpr3 - ; GFX8-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY3]], [[COPY2]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX8-MIR: S_ENDPGM 0 + ; GFX8-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX8-MIR-NEXT: [[DS_MAX_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_F32 [[COPY3]], [[COPY2]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX8-MIR-NEXT: S_ENDPGM 0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_ss_offset_nortn ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $sgpr2, $sgpr3 - ; GFX9-MIR: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY3]], [[COPY2]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX9-MIR: S_ENDPGM 0 + ; GFX9-MIR-NEXT: liveins: $sgpr2, $sgpr3 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]] + ; GFX9-MIR-NEXT: [[DS_MAX_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_F32_gfx9 [[COPY3]], [[COPY2]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX9-MIR-NEXT: S_ENDPGM 0 %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %unused = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) ret void @@ -187,25 +195,27 @@ ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX8-MIR-LABEL: name: ds_fmax_f32_vv ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX8-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX8-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] - ; GFX8-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX8-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX8-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX8-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX8-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_vv ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX9-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX9-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX9-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX9-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX9-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX9-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret float %ret } @@ -227,25 +237,27 @@ ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX8-MIR-LABEL: name: ds_fmax_f32_vv_offset ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX8-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX8-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] - ; GFX8-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX8-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX8-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX8-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX8-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_vv_offset ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX9-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX9-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX9-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX9-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX9-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX9-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) ret float %ret @@ -256,35 +268,37 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_max_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_max_f32 v0, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fmax_f32_vv_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_max_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_max_f32 v0, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX8-MIR-LABEL: name: ds_fmax_f32_vv_nortn ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX8-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX8-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX8-MIR: S_SETPC_B64_return [[COPY3]] + ; GFX8-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX8-MIR-NEXT: S_SETPC_B64_return [[COPY3]] ; GFX9-MIR-LABEL: name: ds_fmax_f32_vv_nortn ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX9-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) - ; GFX9-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX9-MIR: S_SETPC_B64_return [[COPY3]] + ; GFX9-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: [[DS_MAX_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store (s32) on %ir.ptr, addrspace 3) + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX9-MIR-NEXT: S_SETPC_B64_return [[COPY3]] %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret void } @@ -294,35 +308,37 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_max_rtn_f32 v0, v0, v1 offset:512 +; GFX8-NEXT: ds_max_f32 v0, v1 offset:512 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fmax_f32_vv_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_max_rtn_f32 v0, v0, v1 offset:512 +; GFX9-NEXT: ds_max_f32 v0, v1 offset:512 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX8-MIR-LABEL: name: ds_fmax_f32_vv_offset_nortn ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX8-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX8-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX8-MIR: S_SETPC_B64_return [[COPY3]] + ; GFX8-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_F32 [[COPY]], [[COPY1]], 512, 0, implicit $m0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX8-MIR-NEXT: S_SETPC_B64_return [[COPY3]] ; GFX9-MIR-LABEL: name: ds_fmax_f32_vv_offset_nortn ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX9-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) - ; GFX9-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX9-MIR: S_SETPC_B64_return [[COPY3]] + ; GFX9-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: [[DS_MAX_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_F32_gfx9 [[COPY]], [[COPY1]], 512, 0, implicit $exec :: (load store (s32) on %ir.gep, addrspace 3) + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX9-MIR-NEXT: S_SETPC_B64_return [[COPY3]] %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) ret void @@ -345,25 +361,27 @@ ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX8-MIR-LABEL: name: ds_fmax_f32_vv_volatile ; GFX8-MIR: bb.1 (%ir-block.0): - ; GFX8-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX8-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX8-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX8-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX8-MIR: $m0 = S_MOV_B32 -1 - ; GFX8-MIR: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (volatile load store (s32) on %ir.ptr, addrspace 3) - ; GFX8-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] - ; GFX8-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX8-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX8-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: {{ $}} + ; GFX8-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX8-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX8-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX8-MIR-NEXT: $m0 = S_MOV_B32 -1 + ; GFX8-MIR-NEXT: [[DS_MAX_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (volatile load store (s32) on %ir.ptr, addrspace 3) + ; GFX8-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_]] + ; GFX8-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX8-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 ; GFX9-MIR-LABEL: name: ds_fmax_f32_vv_volatile ; GFX9-MIR: bb.1 (%ir-block.0): - ; GFX9-MIR: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 - ; GFX9-MIR: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9-MIR: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GFX9-MIR: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 - ; GFX9-MIR: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (volatile load store (s32) on %ir.ptr, addrspace 3) - ; GFX9-MIR: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] - ; GFX9-MIR: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] - ; GFX9-MIR: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 + ; GFX9-MIR-NEXT: liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: {{ $}} + ; GFX9-MIR-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX9-MIR-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX9-MIR-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; GFX9-MIR-NEXT: [[DS_MAX_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_MAX_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (volatile load store (s32) on %ir.ptr, addrspace 3) + ; GFX9-MIR-NEXT: $vgpr0 = COPY [[DS_MAX_RTN_F32_gfx9_]] + ; GFX9-MIR-NEXT: [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]] + ; GFX9-MIR-NEXT: S_SETPC_B64_return [[COPY3]], implicit $vgpr0 %ret = call float @llvm.amdgcn.ds.fmax(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 true) ret float %ret } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmin.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ds.fmin.ll @@ -68,21 +68,21 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_min_f32 v0, v1 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fmin_f32_ss_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v1, s3 -; GFX9-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_min_f32 v0, v1 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: ds_fmin_f32_ss_nortn: ; GFX10: ; %bb.0: ; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v1, s3 -; GFX10-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX10-NEXT: ds_min_f32 v0, v1 ; GFX10-NEXT: s_endpgm %unused = call float @llvm.amdgcn.ds.fmin(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) ret void @@ -94,21 +94,21 @@ ; GFX8-NEXT: v_mov_b32_e32 v0, s3 ; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_min_rtn_f32 v0, v1, v0 offset:512 +; GFX8-NEXT: ds_min_f32 v1, v0 offset:512 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: ds_fmin_f32_ss_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v0, s3 ; GFX9-NEXT: v_mov_b32_e32 v1, s2 -; GFX9-NEXT: ds_min_rtn_f32 v0, v1, v0 offset:512 +; GFX9-NEXT: ds_min_f32 v1, v0 offset:512 ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: ds_fmin_f32_ss_offset_nortn: ; GFX10: ; %bb.0: ; GFX10-NEXT: v_mov_b32_e32 v0, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: ds_min_rtn_f32 v0, v1, v0 offset:512 +; GFX10-NEXT: ds_min_f32 v1, v0 offset:512 ; GFX10-NEXT: s_endpgm %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 %unused = call float @llvm.amdgcn.ds.fmin(float addrspace(3)* %gep, float %val, i32 0, i32 0, i1 false) @@ -175,14 +175,14 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX8-NEXT: ds_min_f32 v0, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fmin_f32_vv_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX9-NEXT: ds_min_f32 v0, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -190,7 +190,7 @@ ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: ds_min_rtn_f32 v0, v0, v1 +; GFX10-NEXT: ds_min_f32 v0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] %ret = call float @llvm.amdgcn.ds.fmin(float addrspace(3)* %ptr, float %val, i32 0, i32 0, i1 false) @@ -202,14 +202,14 @@ ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 m0, -1 -; GFX8-NEXT: ds_min_rtn_f32 v0, v0, v1 offset:512 +; GFX8-NEXT: ds_min_f32 v0, v1 offset:512 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: ds_fmin_f32_vv_offset_nortn: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_min_rtn_f32 v0, v0, v1 offset:512 +; GFX9-NEXT: ds_min_f32 v0, v1 offset:512 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -217,7 +217,7 @@ ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: ds_min_rtn_f32 v0, v0, v1 offset:512 +; GFX10-NEXT: ds_min_f32 v0, v1 offset:512 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr float, float addrspace(3)* %ptr, i32 128 diff --git a/llvm/test/CodeGen/AMDGPU/lds-atomic-fadd.ll b/llvm/test/CodeGen/AMDGPU/lds-atomic-fadd.ll --- a/llvm/test/CodeGen/AMDGPU/lds-atomic-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-atomic-fadd.ll @@ -18,8 +18,8 @@ ; VI-NEXT: s_lshl_b32 s3, s3, 4 ; VI-NEXT: v_mov_b32_e32 v2, s3 ; VI-NEXT: ds_add_f32 v2, v0 offset:64 +; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s2 -; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: ds_add_rtn_f32 v2, v0, v1 ; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v1, s1 @@ -39,8 +39,8 @@ ; GFX9-NEXT: s_lshl_b32 s0, s3, 4 ; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: ds_add_f32 v2, v0 offset:64 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) ; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll --- a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll @@ -40,8 +40,8 @@ ; SI-NEXT: s_add_i32 s1, s1, 64 ; SI-NEXT: v_mov_b32_e32 v2, s1 ; SI-NEXT: ds_min_f32 v2, v0 +; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_mov_b32_e32 v0, s0 -; SI-NEXT: s_waitcnt lgkmcnt(1) ; SI-NEXT: ds_min_rtn_f32 v0, v0, v1 ; SI-NEXT: v_mov_b32_e32 v1, s3 ; SI-NEXT: s_waitcnt lgkmcnt(0) @@ -67,8 +67,8 @@ ; GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; GFX7-NEXT: v_mov_b32_e32 v2, s2 ; GFX7-NEXT: ds_min_f32 v2, v0 offset:64 +; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s1 -; GFX7-NEXT: s_waitcnt lgkmcnt(1) ; GFX7-NEXT: ds_min_rtn_f32 v0, v0, v1 ; GFX7-NEXT: v_mov_b32_e32 v1, s0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) @@ -94,8 +94,8 @@ ; VI-NEXT: s_lshl_b32 s2, s2, 4 ; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: ds_min_f32 v2, v0 offset:64 +; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s1 -; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: ds_min_rtn_f32 v0, v0, v1 ; VI-NEXT: v_mov_b32_e32 v1, s0 ; VI-NEXT: s_waitcnt lgkmcnt(0) @@ -120,8 +120,8 @@ ; GFX9-NEXT: s_lshl_b32 s0, s4, 4 ; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: ds_min_f32 v2, v0 offset:64 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) ; GFX9-NEXT: ds_min_rtn_f32 v0, v0, v1 ; GFX9-NEXT: v_mov_b32_e32 v1, s2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) @@ -148,7 +148,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: ds_min_rtn_f32 v1, v1, v0 offset:32 ; GFX10-NEXT: ds_min_f32 v2, v0 offset:64 -; GFX10-NEXT: s_waitcnt lgkmcnt(1) +; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: ds_min_rtn_f32 v0, v3, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -175,7 +175,7 @@ ; G_SI-NEXT: ds_min_rtn_f32 v1, v1, v0 ; G_SI-NEXT: s_lshl_b32 s1, s2, 4 ; G_SI-NEXT: v_mov_b32_e32 v2, s1 -; G_SI-NEXT: ds_min_rtn_f32 v0, v2, v0 +; G_SI-NEXT: ds_min_f32 v2, v0 ; G_SI-NEXT: s_waitcnt lgkmcnt(0) ; G_SI-NEXT: v_mov_b32_e32 v0, s3 ; G_SI-NEXT: ds_min_rtn_f32 v0, v0, v1 @@ -203,7 +203,7 @@ ; G_GFX7-NEXT: ds_min_rtn_f32 v1, v1, v0 ; G_GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; G_GFX7-NEXT: v_mov_b32_e32 v2, s2 -; G_GFX7-NEXT: ds_min_rtn_f32 v0, v2, v0 +; G_GFX7-NEXT: ds_min_f32 v2, v0 ; G_GFX7-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX7-NEXT: v_mov_b32_e32 v0, s1 ; G_GFX7-NEXT: ds_min_rtn_f32 v0, v0, v1 @@ -231,7 +231,7 @@ ; G_VI-NEXT: ds_min_rtn_f32 v1, v1, v0 ; G_VI-NEXT: s_lshl_b32 s2, s2, 4 ; G_VI-NEXT: v_mov_b32_e32 v2, s2 -; G_VI-NEXT: ds_min_rtn_f32 v0, v2, v0 +; G_VI-NEXT: ds_min_f32 v2, v0 ; G_VI-NEXT: s_waitcnt lgkmcnt(0) ; G_VI-NEXT: v_mov_b32_e32 v0, s1 ; G_VI-NEXT: ds_min_rtn_f32 v0, v0, v1 @@ -258,7 +258,7 @@ ; G_GFX9-NEXT: ds_min_rtn_f32 v0, v0, v1 ; G_GFX9-NEXT: s_lshl_b32 s0, s0, 4 ; G_GFX9-NEXT: v_mov_b32_e32 v2, s0 -; G_GFX9-NEXT: ds_min_rtn_f32 v1, v2, v1 +; G_GFX9-NEXT: ds_min_f32 v2, v1 ; G_GFX9-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX9-NEXT: v_mov_b32_e32 v1, s7 ; G_GFX9-NEXT: ds_min_rtn_f32 v0, v1, v0 @@ -286,7 +286,7 @@ ; G_GFX10-NEXT: v_mov_b32_e32 v2, s2 ; G_GFX10-NEXT: v_mov_b32_e32 v3, s1 ; G_GFX10-NEXT: ds_min_rtn_f32 v0, v0, v1 -; G_GFX10-NEXT: ds_min_rtn_f32 v1, v2, v1 +; G_GFX10-NEXT: ds_min_f32 v2, v1 ; G_GFX10-NEXT: s_waitcnt lgkmcnt(1) ; G_GFX10-NEXT: ds_min_rtn_f32 v0, v3, v0 ; G_GFX10-NEXT: s_waitcnt lgkmcnt(1) @@ -329,8 +329,8 @@ ; SI-NEXT: s_add_i32 s1, s1, 64 ; SI-NEXT: v_mov_b32_e32 v2, s1 ; SI-NEXT: ds_max_f32 v2, v0 +; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_mov_b32_e32 v0, s0 -; SI-NEXT: s_waitcnt lgkmcnt(1) ; SI-NEXT: ds_max_rtn_f32 v0, v0, v1 ; SI-NEXT: v_mov_b32_e32 v1, s3 ; SI-NEXT: s_waitcnt lgkmcnt(0) @@ -356,8 +356,8 @@ ; GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; GFX7-NEXT: v_mov_b32_e32 v2, s2 ; GFX7-NEXT: ds_max_f32 v2, v0 offset:64 +; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s1 -; GFX7-NEXT: s_waitcnt lgkmcnt(1) ; GFX7-NEXT: ds_max_rtn_f32 v0, v0, v1 ; GFX7-NEXT: v_mov_b32_e32 v1, s0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) @@ -383,8 +383,8 @@ ; VI-NEXT: s_lshl_b32 s2, s2, 4 ; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: ds_max_f32 v2, v0 offset:64 +; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s1 -; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: ds_max_rtn_f32 v0, v0, v1 ; VI-NEXT: v_mov_b32_e32 v1, s0 ; VI-NEXT: s_waitcnt lgkmcnt(0) @@ -409,8 +409,8 @@ ; GFX9-NEXT: s_lshl_b32 s0, s4, 4 ; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: ds_max_f32 v2, v0 offset:64 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) ; GFX9-NEXT: ds_max_rtn_f32 v0, v0, v1 ; GFX9-NEXT: v_mov_b32_e32 v1, s2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) @@ -437,7 +437,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: ds_max_rtn_f32 v1, v1, v0 offset:32 ; GFX10-NEXT: ds_max_f32 v2, v0 offset:64 -; GFX10-NEXT: s_waitcnt lgkmcnt(1) +; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: ds_max_rtn_f32 v0, v3, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -464,7 +464,7 @@ ; G_SI-NEXT: ds_max_rtn_f32 v1, v1, v0 ; G_SI-NEXT: s_lshl_b32 s1, s2, 4 ; G_SI-NEXT: v_mov_b32_e32 v2, s1 -; G_SI-NEXT: ds_max_rtn_f32 v0, v2, v0 +; G_SI-NEXT: ds_max_f32 v2, v0 ; G_SI-NEXT: s_waitcnt lgkmcnt(0) ; G_SI-NEXT: v_mov_b32_e32 v0, s3 ; G_SI-NEXT: ds_max_rtn_f32 v0, v0, v1 @@ -492,7 +492,7 @@ ; G_GFX7-NEXT: ds_max_rtn_f32 v1, v1, v0 ; G_GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; G_GFX7-NEXT: v_mov_b32_e32 v2, s2 -; G_GFX7-NEXT: ds_max_rtn_f32 v0, v2, v0 +; G_GFX7-NEXT: ds_max_f32 v2, v0 ; G_GFX7-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX7-NEXT: v_mov_b32_e32 v0, s1 ; G_GFX7-NEXT: ds_max_rtn_f32 v0, v0, v1 @@ -520,7 +520,7 @@ ; G_VI-NEXT: ds_max_rtn_f32 v1, v1, v0 ; G_VI-NEXT: s_lshl_b32 s2, s2, 4 ; G_VI-NEXT: v_mov_b32_e32 v2, s2 -; G_VI-NEXT: ds_max_rtn_f32 v0, v2, v0 +; G_VI-NEXT: ds_max_f32 v2, v0 ; G_VI-NEXT: s_waitcnt lgkmcnt(0) ; G_VI-NEXT: v_mov_b32_e32 v0, s1 ; G_VI-NEXT: ds_max_rtn_f32 v0, v0, v1 @@ -547,7 +547,7 @@ ; G_GFX9-NEXT: ds_max_rtn_f32 v0, v0, v1 ; G_GFX9-NEXT: s_lshl_b32 s0, s0, 4 ; G_GFX9-NEXT: v_mov_b32_e32 v2, s0 -; G_GFX9-NEXT: ds_max_rtn_f32 v1, v2, v1 +; G_GFX9-NEXT: ds_max_f32 v2, v1 ; G_GFX9-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX9-NEXT: v_mov_b32_e32 v1, s7 ; G_GFX9-NEXT: ds_max_rtn_f32 v0, v1, v0 @@ -575,7 +575,7 @@ ; G_GFX10-NEXT: v_mov_b32_e32 v2, s2 ; G_GFX10-NEXT: v_mov_b32_e32 v3, s1 ; G_GFX10-NEXT: ds_max_rtn_f32 v0, v0, v1 -; G_GFX10-NEXT: ds_max_rtn_f32 v1, v2, v1 +; G_GFX10-NEXT: ds_max_f32 v2, v1 ; G_GFX10-NEXT: s_waitcnt lgkmcnt(1) ; G_GFX10-NEXT: ds_max_rtn_f32 v0, v3, v0 ; G_GFX10-NEXT: s_waitcnt lgkmcnt(1) @@ -621,8 +621,8 @@ ; SI-NEXT: s_add_i32 s0, s5, 64 ; SI-NEXT: v_mov_b32_e32 v4, s0 ; SI-NEXT: ds_min_f64 v4, v[0:1] +; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: s_waitcnt lgkmcnt(1) ; SI-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] ; SI-NEXT: s_add_i32 s0, s3, 4 ; SI-NEXT: v_mov_b32_e32 v2, s0 @@ -655,8 +655,8 @@ ; GFX7-NEXT: s_lshl_b32 s2, s4, 4 ; GFX7-NEXT: v_mov_b32_e32 v4, s2 ; GFX7-NEXT: ds_min_f64 v4, v[0:1] offset:64 +; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s1 -; GFX7-NEXT: s_waitcnt lgkmcnt(1) ; GFX7-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] ; GFX7-NEXT: s_add_i32 s1, s0, 4 ; GFX7-NEXT: v_mov_b32_e32 v3, s1 @@ -688,8 +688,8 @@ ; VI-NEXT: s_lshl_b32 s2, s4, 4 ; VI-NEXT: v_mov_b32_e32 v4, s2 ; VI-NEXT: ds_min_f64 v4, v[0:1] offset:64 +; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s1 -; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] ; VI-NEXT: s_add_i32 s1, s0, 4 ; VI-NEXT: v_mov_b32_e32 v3, s1 @@ -721,7 +721,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v5, s0 ; GFX9-NEXT: v_mov_b32_e32 v4, s3 ; GFX9-NEXT: ds_min_f64 v5, v[0:1] offset:64 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: ds_min_rtn_f64 v[0:1], v4, v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) @@ -752,7 +752,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v5, s3 ; GFX10-NEXT: ds_min_rtn_f64 v[2:3], v2, v[0:1] offset:32 ; GFX10-NEXT: ds_min_f64 v4, v[0:1] offset:64 -; GFX10-NEXT: s_waitcnt lgkmcnt(1) +; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: ds_min_rtn_f64 v[0:1], v5, v[2:3] ; GFX10-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -783,7 +783,7 @@ ; G_SI-NEXT: ds_min_rtn_f64 v[2:3], v2, v[0:1] ; G_SI-NEXT: s_lshl_b32 s1, s4, 4 ; G_SI-NEXT: v_mov_b32_e32 v4, s1 -; G_SI-NEXT: ds_min_rtn_f64 v[0:1], v4, v[0:1] +; G_SI-NEXT: ds_min_f64 v4, v[0:1] ; G_SI-NEXT: s_waitcnt lgkmcnt(0) ; G_SI-NEXT: v_mov_b32_e32 v0, s5 ; G_SI-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] @@ -818,7 +818,7 @@ ; G_GFX7-NEXT: ds_min_rtn_f64 v[2:3], v2, v[0:1] ; G_GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; G_GFX7-NEXT: v_mov_b32_e32 v4, s2 -; G_GFX7-NEXT: ds_min_rtn_f64 v[0:1], v4, v[0:1] +; G_GFX7-NEXT: ds_min_f64 v4, v[0:1] ; G_GFX7-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX7-NEXT: v_mov_b32_e32 v0, s1 ; G_GFX7-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] @@ -852,7 +852,7 @@ ; G_VI-NEXT: ds_min_rtn_f64 v[2:3], v2, v[0:1] ; G_VI-NEXT: s_lshl_b32 s2, s2, 4 ; G_VI-NEXT: v_mov_b32_e32 v4, s2 -; G_VI-NEXT: ds_min_rtn_f64 v[0:1], v4, v[0:1] +; G_VI-NEXT: ds_min_f64 v4, v[0:1] ; G_VI-NEXT: s_waitcnt lgkmcnt(0) ; G_VI-NEXT: v_mov_b32_e32 v0, s1 ; G_VI-NEXT: ds_min_rtn_f64 v[0:1], v0, v[2:3] @@ -886,7 +886,7 @@ ; G_GFX9-NEXT: s_lshl_b32 s0, s0, 4 ; G_GFX9-NEXT: v_mov_b32_e32 v5, s0 ; G_GFX9-NEXT: v_mov_b32_e32 v4, s7 -; G_GFX9-NEXT: ds_min_rtn_f64 v[0:1], v5, v[0:1] +; G_GFX9-NEXT: ds_min_f64 v5, v[0:1] ; G_GFX9-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX9-NEXT: ds_min_rtn_f64 v[0:1], v4, v[2:3] ; G_GFX9-NEXT: v_mov_b32_e32 v2, s6 @@ -918,7 +918,7 @@ ; G_GFX10-NEXT: v_mov_b32_e32 v2, s3 ; G_GFX10-NEXT: v_mov_b32_e32 v4, s0 ; G_GFX10-NEXT: ds_min_rtn_f64 v[2:3], v2, v[0:1] -; G_GFX10-NEXT: ds_min_rtn_f64 v[0:1], v4, v[0:1] +; G_GFX10-NEXT: ds_min_f64 v4, v[0:1] ; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX10-NEXT: ds_min_rtn_f64 v[0:1], v5, v[2:3] ; G_GFX10-NEXT: v_mov_b32_e32 v2, s6 @@ -964,8 +964,8 @@ ; SI-NEXT: s_add_i32 s0, s5, 64 ; SI-NEXT: v_mov_b32_e32 v4, s0 ; SI-NEXT: ds_max_f64 v4, v[0:1] +; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: s_waitcnt lgkmcnt(1) ; SI-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] ; SI-NEXT: s_add_i32 s0, s3, 4 ; SI-NEXT: v_mov_b32_e32 v2, s0 @@ -998,8 +998,8 @@ ; GFX7-NEXT: s_lshl_b32 s2, s4, 4 ; GFX7-NEXT: v_mov_b32_e32 v4, s2 ; GFX7-NEXT: ds_max_f64 v4, v[0:1] offset:64 +; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s1 -; GFX7-NEXT: s_waitcnt lgkmcnt(1) ; GFX7-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] ; GFX7-NEXT: s_add_i32 s1, s0, 4 ; GFX7-NEXT: v_mov_b32_e32 v3, s1 @@ -1031,8 +1031,8 @@ ; VI-NEXT: s_lshl_b32 s2, s4, 4 ; VI-NEXT: v_mov_b32_e32 v4, s2 ; VI-NEXT: ds_max_f64 v4, v[0:1] offset:64 +; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s1 -; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] ; VI-NEXT: s_add_i32 s1, s0, 4 ; VI-NEXT: v_mov_b32_e32 v3, s1 @@ -1064,7 +1064,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v5, s0 ; GFX9-NEXT: v_mov_b32_e32 v4, s3 ; GFX9-NEXT: ds_max_f64 v5, v[0:1] offset:64 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: ds_max_rtn_f64 v[0:1], v4, v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) @@ -1095,7 +1095,7 @@ ; GFX10-NEXT: v_mov_b32_e32 v5, s3 ; GFX10-NEXT: ds_max_rtn_f64 v[2:3], v2, v[0:1] offset:32 ; GFX10-NEXT: ds_max_f64 v4, v[0:1] offset:64 -; GFX10-NEXT: s_waitcnt lgkmcnt(1) +; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: ds_max_rtn_f64 v[0:1], v5, v[2:3] ; GFX10-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -1126,7 +1126,7 @@ ; G_SI-NEXT: ds_max_rtn_f64 v[2:3], v2, v[0:1] ; G_SI-NEXT: s_lshl_b32 s1, s4, 4 ; G_SI-NEXT: v_mov_b32_e32 v4, s1 -; G_SI-NEXT: ds_max_rtn_f64 v[0:1], v4, v[0:1] +; G_SI-NEXT: ds_max_f64 v4, v[0:1] ; G_SI-NEXT: s_waitcnt lgkmcnt(0) ; G_SI-NEXT: v_mov_b32_e32 v0, s5 ; G_SI-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] @@ -1161,7 +1161,7 @@ ; G_GFX7-NEXT: ds_max_rtn_f64 v[2:3], v2, v[0:1] ; G_GFX7-NEXT: s_lshl_b32 s2, s2, 4 ; G_GFX7-NEXT: v_mov_b32_e32 v4, s2 -; G_GFX7-NEXT: ds_max_rtn_f64 v[0:1], v4, v[0:1] +; G_GFX7-NEXT: ds_max_f64 v4, v[0:1] ; G_GFX7-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX7-NEXT: v_mov_b32_e32 v0, s1 ; G_GFX7-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] @@ -1195,7 +1195,7 @@ ; G_VI-NEXT: ds_max_rtn_f64 v[2:3], v2, v[0:1] ; G_VI-NEXT: s_lshl_b32 s2, s2, 4 ; G_VI-NEXT: v_mov_b32_e32 v4, s2 -; G_VI-NEXT: ds_max_rtn_f64 v[0:1], v4, v[0:1] +; G_VI-NEXT: ds_max_f64 v4, v[0:1] ; G_VI-NEXT: s_waitcnt lgkmcnt(0) ; G_VI-NEXT: v_mov_b32_e32 v0, s1 ; G_VI-NEXT: ds_max_rtn_f64 v[0:1], v0, v[2:3] @@ -1229,7 +1229,7 @@ ; G_GFX9-NEXT: s_lshl_b32 s0, s0, 4 ; G_GFX9-NEXT: v_mov_b32_e32 v5, s0 ; G_GFX9-NEXT: v_mov_b32_e32 v4, s7 -; G_GFX9-NEXT: ds_max_rtn_f64 v[0:1], v5, v[0:1] +; G_GFX9-NEXT: ds_max_f64 v5, v[0:1] ; G_GFX9-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX9-NEXT: ds_max_rtn_f64 v[0:1], v4, v[2:3] ; G_GFX9-NEXT: v_mov_b32_e32 v2, s6 @@ -1261,7 +1261,7 @@ ; G_GFX10-NEXT: v_mov_b32_e32 v2, s3 ; G_GFX10-NEXT: v_mov_b32_e32 v4, s0 ; G_GFX10-NEXT: ds_max_rtn_f64 v[2:3], v2, v[0:1] -; G_GFX10-NEXT: ds_max_rtn_f64 v[0:1], v4, v[0:1] +; G_GFX10-NEXT: ds_max_f64 v4, v[0:1] ; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) ; G_GFX10-NEXT: ds_max_rtn_f64 v[0:1], v5, v[2:3] ; G_GFX10-NEXT: v_mov_b32_e32 v2, s6 diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll --- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll @@ -79,8 +79,8 @@ ; GFX9-NEXT: s_lshl_b32 s0, s3, 4 ; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: ds_add_f32 v2, v0 offset:64 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NEXT: s_waitcnt lgkmcnt(1) ; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)