Index: lib/Target/AMDGPU/AMDGPU.td =================================================================== --- lib/Target/AMDGPU/AMDGPU.td +++ lib/Target/AMDGPU/AMDGPU.td @@ -316,6 +316,12 @@ "Has unpacked d16 vmem instructions" >; +def FeatureD16HIInsts : SubtargetFeature<"d16-hi-insts", + "HasD16HIInsts", + "true", + "Has D16_HI instructions" +>; + def FeatureDLInsts : SubtargetFeature<"dl-insts", "HasDLInsts", "true", @@ -608,20 +614,23 @@ def FeatureISAVersion9_0_0 : SubtargetFeatureISAVersion <9,0,0, [FeatureGFX9, FeatureMadMixInsts, - FeatureLDSBankCount32 + FeatureLDSBankCount32, + FeatureD16HIInsts ]>; def FeatureISAVersion9_0_2 : SubtargetFeatureISAVersion <9,0,2, [FeatureGFX9, FeatureMadMixInsts, FeatureLDSBankCount32, - FeatureXNACK + FeatureXNACK, + FeatureD16HIInsts ]>; def FeatureISAVersion9_0_4 : SubtargetFeatureISAVersion <9,0,4, [FeatureGFX9, FeatureLDSBankCount32, - FeatureFmaMixInsts]>; + FeatureFmaMixInsts, + FeatureD16HIInsts]>; def FeatureISAVersion9_0_6 : SubtargetFeatureISAVersion <9,0,6, [FeatureGFX9, @@ -764,6 +773,9 @@ def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">, AssemblerPredicate<"FeatureGFX9Insts">; +def HasD16HIInsts : Predicate<"Subtarget->hasD16HIInsts()">, + AssemblerPredicate<"FeatureGFX9Insts,FeatureD16HIInsts">; + def HasUnpackedD16VMem : Predicate<"Subtarget->hasUnpackedD16VMem()">, AssemblerPredicate<"FeatureUnpackedD16VMem">; def HasPackedD16VMem : Predicate<"!Subtarget->hasUnpackedD16VMem()">, Index: lib/Target/AMDGPU/AMDGPUSubtarget.h =================================================================== --- lib/Target/AMDGPU/AMDGPUSubtarget.h +++ lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -172,6 +172,7 @@ bool FlatScratchInsts; bool AddNoCarryInsts; bool HasUnpackedD16VMem; + bool HasD16HIInsts; bool R600ALUInst; bool CaymanISA; bool CFALUBug; @@ -493,6 +494,10 @@ return HasUnpackedD16VMem; } + bool hasD16HIInsts() const { + return HasD16HIInsts; + } + bool isMesaKernel(const MachineFunction &MF) const { return isMesa3DOS() && !AMDGPU::isShader(MF.getFunction().getCallingConv()); } Index: lib/Target/AMDGPU/AMDGPUSubtarget.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -168,6 +168,7 @@ FlatScratchInsts(false), AddNoCarryInsts(false), HasUnpackedD16VMem(false), + HasD16HIInsts(false), R600ALUInst(false), CaymanISA(false), Index: lib/Target/AMDGPU/BUFInstructions.td =================================================================== --- lib/Target/AMDGPU/BUFInstructions.td +++ lib/Target/AMDGPU/BUFInstructions.td @@ -921,22 +921,26 @@ "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 >; -defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < - "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 ->; - defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 >; -defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < - "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 ->; - defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 >; +} // End SubtargetPredicate = HasD16LoadStore + +let SubtargetPredicate = HasD16HIInsts in { + +defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < + "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 +>; + +defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < + "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 +>; + defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 >; @@ -952,11 +956,12 @@ defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < "buffer_load_format_d16_hi_x", VGPR_32 >; + defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < "buffer_store_format_d16_hi_x", VGPR_32 >; -} // End HasD16LoadStore +} // End SubtargetPredicate = HasD16HIInsts def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", int_amdgcn_buffer_wbinvl1>; @@ -1375,14 +1380,16 @@ defm : MUBUFScratchLoadPat ; let OtherPredicates = [HasD16LoadStore] in { -defm : MUBUFScratchLoadPat_Hi16; -defm : MUBUFScratchLoadPat_Hi16; -defm : MUBUFScratchLoadPat_Hi16; - defm : MUBUFScratchLoadPat_Lo16; defm : MUBUFScratchLoadPat_Lo16; defm : MUBUFScratchLoadPat_Lo16; -} +} // End OtherPredicates = [HasD16LoadStore] + +let OtherPredicates = [HasD16HIInsts] in { +defm : MUBUFScratchLoadPat_Hi16; +defm : MUBUFScratchLoadPat_Hi16; +defm : MUBUFScratchLoadPat_Hi16; +} // End OtherPredicates = [HasD16HIInsts] // BUFFER_LOAD_DWORD*, addr64=0 multiclass MUBUF_Load_Dword ; -let OtherPredicates = [HasD16LoadStore] in { +let OtherPredicates = [HasD16HIInsts] in { // Hiding the extract high pattern in the PatFrag seems to not // automatically increase the complexity. let AddedComplexity = 1 in { defm : MUBUFScratchStorePat ; defm : MUBUFScratchStorePat ; -} -} +} // End AddedComplexity = 1 +} // End OtherPredicates = [HasD16HIInsts] //===----------------------------------------------------------------------===// // MTBUF Patterns Index: lib/Target/AMDGPU/DSInstructions.td =================================================================== --- lib/Target/AMDGPU/DSInstructions.td +++ lib/Target/AMDGPU/DSInstructions.td @@ -385,10 +385,10 @@ let has_m0_read = 0 in { -let SubtargetPredicate = HasD16LoadStore in { +let SubtargetPredicate = HasD16HIInsts in { def DS_WRITE_B8_D16_HI : DS_1A1D_NORET<"ds_write_b8_d16_hi">; def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">; -} +} // End SubtargetPredicate = HasD16HIInsts let SubtargetPredicate = HasDSAddTid in { def DS_WRITE_ADDTID_B32 : DS_1A1D_NORET<"ds_write_addtid_b32">; @@ -529,12 +529,15 @@ let has_m0_read = 0 in { let SubtargetPredicate = HasD16LoadStore in { def DS_READ_U8_D16 : DS_1A_RET_Tied<"ds_read_u8_d16">; -def DS_READ_U8_D16_HI : DS_1A_RET_Tied<"ds_read_u8_d16_hi">; def DS_READ_I8_D16 : DS_1A_RET_Tied<"ds_read_i8_d16">; -def DS_READ_I8_D16_HI : DS_1A_RET_Tied<"ds_read_i8_d16_hi">; def DS_READ_U16_D16 : DS_1A_RET_Tied<"ds_read_u16_d16">; +} // End SubtargetPredicate = HasD16LoadStore + +let SubtargetPredicate = HasD16HIInsts in { +def DS_READ_U8_D16_HI : DS_1A_RET_Tied<"ds_read_u8_d16_hi">; +def DS_READ_I8_D16_HI : DS_1A_RET_Tied<"ds_read_i8_d16_hi">; def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">; -} +} // End SubtargetPredicate = HasD16HIInsts let SubtargetPredicate = HasDSAddTid in { def DS_READ_ADDTID_B32 : DS_1A_RET<"ds_read_addtid_b32">; @@ -655,18 +658,19 @@ } // End AddedComplexity = 100 -let OtherPredicates = [HasD16LoadStore] in { let AddedComplexity = 100 in { -defm : DSReadPat_Hi16; -defm : DSReadPat_Hi16; -defm : DSReadPat_Hi16; - +let OtherPredicates = [HasD16LoadStore] in { defm : DSReadPat_Lo16; defm : DSReadPat_Lo16; defm : DSReadPat_Lo16; +} // End OtherPredicates = [HasD16LoadStore] -} -} +let OtherPredicates = [HasD16HIInsts] in { +defm : DSReadPat_Hi16; +defm : DSReadPat_Hi16; +defm : DSReadPat_Hi16; +} // End OtherPredicates = [HasD16HIInsts] +} // End AddedComplexity = 100 class DSWritePat : GCNPat < (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)), @@ -689,10 +693,10 @@ defm : DSWritePat_mc ; defm : DSWritePat_mc ; -let OtherPredicates = [HasD16LoadStore] in { +let OtherPredicates = [HasD16HIInsts] in { def : DSWritePat ; def : DSWritePat ; -} +} // End OtherPredicates = [HasD16HIInsts] class DS64Bit4ByteAlignedReadPat : GCNPat < Index: lib/Target/AMDGPU/FLATInstructions.td =================================================================== --- lib/Target/AMDGPU/FLATInstructions.td +++ lib/Target/AMDGPU/FLATInstructions.td @@ -367,15 +367,17 @@ let SubtargetPredicate = HasD16LoadStore in { def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>; -def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>; -def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>; -def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; +} // End SubtargetPredicate = HasD16LoadStore +let SubtargetPredicate = HasD16HIInsts in { +def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; +def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; +def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; -} +} // End SubtargetPredicate = HasD16HIInsts defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", VGPR_32, i32, atomic_cmp_swap_flat, @@ -479,6 +481,14 @@ } // End SubtargetPredicate = isCI +let SubtargetPredicate = HasD16HIInsts in { +defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; +defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; +defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; +defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; +defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; +} // End SubtargetPredicate = HasD16HIInsts + let SubtargetPredicate = HasFlatGlobalInsts in { defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; @@ -490,11 +500,8 @@ defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>; -defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>; -defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>; -defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>; defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>; @@ -503,9 +510,6 @@ defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; -defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; -defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; - let is_flat_global = 1 in { defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", VGPR_32, i32, AMDGPUatomic_cmp_swap_global, @@ -591,6 +595,14 @@ } // End SubtargetPredicate = HasFlatGlobalInsts +let SubtargetPredicate = HasD16HIInsts in { +defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>; +defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>; +defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>; +defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; +defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; +} // End SubtargetPredicate = HasD16HIInsts + let SubtargetPredicate = HasFlatScratchInsts in { defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; @@ -602,11 +614,8 @@ defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32>; -defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>; defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32>; -defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>; defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32>; -defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>; defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>; defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>; @@ -615,9 +624,6 @@ defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; -defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; -defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; - } // End SubtargetPredicate = HasFlatScratchInsts //===----------------------------------------------------------------------===// @@ -780,7 +786,7 @@ def : FlatStorePat ; def : FlatStorePat ; -let OtherPredicates = [HasD16LoadStore] in { +let OtherPredicates = [HasD16HIInsts] in { def : FlatStorePat ; def : FlatStorePat ; @@ -788,14 +794,16 @@ defm : FlatLoadPat_Hi16 ; defm : FlatLoadPat_Hi16 ; defm : FlatLoadPat_Hi16 ; -} +} // End AddedComplexity = 3 +} // End OtherPredicates = [HasD16HIInsts] +let OtherPredicates = [HasD16LoadStore] in { let AddedComplexity = 9 in { defm : FlatLoadPat_Lo16 ; defm : FlatLoadPat_Lo16 ; defm : FlatLoadPat_Lo16 ; -} -} +} // End AddedComplexity = 9 +} // End OtherPredicates = [HasD16LoadStore] } // End OtherPredicates = [HasFlatAddressSpace] @@ -824,19 +832,20 @@ def : FlatStoreSignedPat ; def : FlatStoreSignedPat ; -let OtherPredicates = [HasD16LoadStore] in { +let OtherPredicates = [HasD16HIInsts] in { def : FlatStoreSignedPat ; def : FlatStoreSignedPat ; defm : FlatSignedLoadPat_Hi16 ; defm : FlatSignedLoadPat_Hi16 ; defm : FlatSignedLoadPat_Hi16 ; +} // End OtherPredicates = [HasD16HIInsts] +let OtherPredicates = [HasD16LoadStore] in { defm : FlatSignedLoadPat_Lo16 ; defm : FlatSignedLoadPat_Lo16 ; defm : FlatSignedLoadPat_Lo16 ; - -} +} // End OtherPredicates = [HasD16LoadStore] def : FlatStoreSignedAtomicPat ; def : FlatStoreSignedAtomicPat ; Index: test/CodeGen/AMDGPU/load-hi16.ll =================================================================== --- test/CodeGen/AMDGPU/load-hi16.ll +++ test/CodeGen/AMDGPU/load-hi16.ll @@ -1,13 +1,14 @@ -; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s -; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s +; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX906,NO-D16-HI %s +; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX803,NO-D16-HI %s ; GCN-LABEL: {{^}}load_local_hi_v2i16_undeflo: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16_d16_hi v0, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16_d16_hi v0, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v +; NO-D16-HI: ds_read_u16 v define <2 x i16> @load_local_hi_v2i16_undeflo(i16 addrspace(3)* %in) #0 { entry: %load = load i16, i16 addrspace(3)* %in @@ -17,12 +18,12 @@ ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v +; NO-D16-HI: ds_read_u16 v define <2 x i16> @load_local_hi_v2i16_reglo(i16 addrspace(3)* %in, i16 %reg) #0 { entry: %load = load i16, i16 addrspace(3)* %in @@ -34,13 +35,13 @@ ; Show that we get reasonable regalloc without physreg constraints. ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v +; NO-D16-HI: ds_read_u16 v define void @load_local_hi_v2i16_reglo_vreg(i16 addrspace(3)* %in, i16 %reg) #0 { entry: %load = load i16, i16 addrspace(3)* %in @@ -52,13 +53,13 @@ ; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo: ; GCN: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v1, 0 -; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: v_mov_b32_e32 v1, 0 +; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v +; NO-D16-HI: ds_read_u16 v define <2 x i16> @load_local_hi_v2i16_zerolo(i16 addrspace(3)* %in) #0 { entry: %load = load i16, i16 addrspace(3)* %in @@ -69,13 +70,13 @@ ; FIXME: Remove m0 initialization ; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo_shift: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16 v0, v0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16 v0, v0 +; GFX900-NEXT: s_waitcnt lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v -; VI: v_lshlrev_b32_e32 v0, 16, v0 +; NO-D16-HI: ds_read_u16 v +; NO-D16-HI: v_lshlrev_b32_e32 v0, 16, v0 define i32 @load_local_hi_v2i16_zerolo_shift(i16 addrspace(3)* %in) #0 { entry: %load = load i16, i16 addrspace(3)* %in @@ -86,13 +87,13 @@ ; GCN-LABEL: {{^}}load_local_hi_v2f16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u16 v +; NO-D16-HI: ds_read_u16 v define void @load_local_hi_v2f16_reglo_vreg(half addrspace(3)* %in, half %reg) #0 { entry: %load = load half, half addrspace(3)* %in @@ -104,13 +105,13 @@ ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_zexti8: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u8_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u8_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_u8 v +; NO-D16-HI: ds_read_u8 v define void @load_local_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(3)* %in, i16 %reg) #0 { entry: %load = load i8, i8 addrspace(3)* %in @@ -123,13 +124,13 @@ ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_sexti8: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_i8_d16_hi v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_i8_d16_hi v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: ds_read_i8 v +; NO-D16-HI: ds_read_i8 v define void @load_local_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(3)* %in, i16 %reg) #0 { entry: %load = load i8, i8 addrspace(3)* %in @@ -142,11 +143,11 @@ ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 define void @load_global_hi_v2i16_reglo_vreg(i16 addrspace(1)* %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 -2047 @@ -159,11 +160,11 @@ ; GCN-LABEL: {{^}}load_global_hi_v2f16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 define void @load_global_hi_v2f16_reglo_vreg(half addrspace(1)* %in, half %reg) #0 { entry: %gep = getelementptr inbounds half, half addrspace(1)* %in, i64 -2047 @@ -176,11 +177,11 @@ ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_zexti8: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_ubyte_d16_hi v2, v[0:1], off offset:-4095 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_ubyte_d16_hi v2, v[0:1], off offset:-4095 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 define void @load_global_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095 @@ -194,11 +195,11 @@ ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_sexti8: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_sbyte_d16_hi v2, v[0:1], off offset:-4095 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_sbyte_d16_hi v2, v[0:1], off offset:-4095 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 define void @load_global_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095 @@ -212,15 +213,16 @@ ; GCN-LABEL: load_flat_hi_v2i16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1] -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: flat_load_short_d16_hi v2, v[0:1] +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_ushort v{{[0-9]+}} -; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, -; VI: v_or_b32_sdwa +; NO-D16-HI: flat_load_ushort v{{[0-9]+}} +; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, +; GFX803: v_or_b32_sdwa +; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, define void @load_flat_hi_v2i16_reglo_vreg(i16* %in, i16 %reg) #0 { entry: %load = load i16, i16* %in @@ -232,15 +234,16 @@ ; GCN-LABEL: {{^}}load_flat_hi_v2f16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1] -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: flat_load_short_d16_hi v2, v[0:1] +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_ushort v{{[0-9]+}} -; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, -; VI: v_or_b32_sdwa +; NO-D16-HI: flat_load_ushort v{{[0-9]+}} +; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, +; GFX803: v_or_b32_sdwa +; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, define void @load_flat_hi_v2f16_reglo_vreg(half* %in, half %reg) #0 { entry: %load = load half, half* %in @@ -252,15 +255,16 @@ ; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_zexti8: ; GCN: s_waitcnt -; GFX9-NEXT: flat_load_ubyte_d16_hi v2, v[0:1] -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: flat_load_ubyte_d16_hi v2, v[0:1] +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_ubyte v{{[0-9]+}} -; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, -; VI: v_or_b32_sdwa +; NO-D16-HI: flat_load_ubyte v{{[0-9]+}} +; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, +; GFX803: v_or_b32_sdwa +; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, define void @load_flat_hi_v2i16_reglo_vreg_zexti8(i8* %in, i16 %reg) #0 { entry: %load = load i8, i8* %in @@ -273,15 +277,16 @@ ; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_sexti8: ; GCN: s_waitcnt -; GFX9-NEXT: flat_load_sbyte_d16_hi v2, v[0:1] -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v[0:1], v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: flat_load_sbyte_d16_hi v2, v[0:1] +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v[0:1], v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_sbyte v{{[0-9]+}} -; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, -; VI: v_or_b32_sdwa +; NO-D16-HI: flat_load_sbyte v{{[0-9]+}} +; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, +; GFX803: v_or_b32_sdwa +; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, define void @load_flat_hi_v2i16_reglo_vreg_sexti8(i8* %in, i16 %reg) #0 { entry: %load = load i8, i8* %in @@ -294,13 +299,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg: ; GCN: s_waitcnt -; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}} +; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}} define void @load_private_hi_v2i16_reglo_vreg(i16 addrspace(5)* byval %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i64 2045 @@ -313,13 +318,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg: ; GCN: s_waitcnt -; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}} +; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}} define void @load_private_hi_v2f16_reglo_vreg(half addrspace(5)* byval %in, half %reg) #0 { entry: %gep = getelementptr inbounds half, half addrspace(5)* %in, i64 2045 @@ -332,13 +337,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff: ; GCN: s_waitcnt -; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s4 offset:4094{{$}} -; GFX9: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s4 offset:4094{{$}} +; GFX900: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}} +; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}} define void @load_private_hi_v2i16_reglo_vreg_nooff(i16 addrspace(5)* byval %in, i16 %reg) #0 { entry: %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*) @@ -350,13 +355,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_load_short_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: buffer_load_short_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}} +; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}} define void @load_private_hi_v2f16_reglo_vreg_nooff(half addrspace(5)* %in, half %reg) #0 { entry: %load = load volatile half, half addrspace(5)* inttoptr (i32 4094 to half addrspace(5)*) @@ -368,13 +373,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8: ; GCN: s_waitcnt -; GFX9: buffer_load_ubyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_ubyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}} +; NO-D16-HI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}} define void @load_private_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(5)* byval %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4091 @@ -388,13 +393,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8: ; GCN: s_waitcnt -; GFX9: buffer_load_sbyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_sbyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}} +; NO-D16-HI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}} define void @load_private_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(5)* byval %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4091 @@ -408,13 +413,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_zexti8: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}} +; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}} define void @load_private_hi_v2i16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, i16 %reg) #0 { entry: %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*) @@ -427,13 +432,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_sexti8: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_load_sbyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: buffer_load_sbyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}} +; NO-D16-HI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}} define void @load_private_hi_v2i16_reglo_vreg_nooff_sexti8(i8 addrspace(5)* %in, i16 %reg) #0 { entry: %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*) @@ -446,13 +451,13 @@ ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff_zexti8: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}} +; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}} define void @load_private_hi_v2f16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, half %reg) #0 { entry: %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*) @@ -466,13 +471,14 @@ ; GCN-LABEL: {{^}}load_constant_hi_v2i16_reglo_vreg: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_ushort +; GFX803: flat_load_ushort +; GFX906: global_load_ushort define void @load_constant_hi_v2i16_reglo_vreg(i16 addrspace(4)* %in, i16 %reg) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 -2047 @@ -485,13 +491,14 @@ ; GCN-LABEL: load_constant_hi_v2f16_reglo_vreg ; GCN: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_store_dword -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_store_dword +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 -; VI: flat_load_ushort +; GFX803: flat_load_ushort +; GFX906: global_load_ushort define void @load_constant_hi_v2f16_reglo_vreg(half addrspace(4)* %in, half %reg) #0 { entry: %gep = getelementptr inbounds half, half addrspace(4)* %in, i64 -2047 @@ -506,8 +513,8 @@ ; to offset variant. ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_to_offset: -; GFX9: buffer_store_dword -; GFX9-NEXT: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4094 +; GFX900: buffer_store_dword +; GFX900-NEXT: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4094 define void @load_private_hi_v2i16_reglo_vreg_to_offset(i16 %reg) #0 { entry: %obj0 = alloca [10 x i32], align 4, addrspace(5) @@ -523,8 +530,8 @@ } ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8_to_offset: -; GFX9: buffer_store_dword -; GFX9-NEXT: buffer_load_sbyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095 +; GFX900: buffer_store_dword +; GFX900-NEXT: buffer_load_sbyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095 define void @load_private_hi_v2i16_reglo_vreg_sexti8_to_offset(i16 %reg) #0 { entry: %obj0 = alloca [10 x i32], align 4, addrspace(5) @@ -541,8 +548,8 @@ } ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8_to_offset: -; GFX9: buffer_store_dword -; GFX9-NEXT: buffer_load_ubyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095 +; GFX900: buffer_store_dword +; GFX900-NEXT: buffer_load_ubyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095 define void @load_private_hi_v2i16_reglo_vreg_zexti8_to_offset(i16 %reg) #0 { entry: %obj0 = alloca [10 x i32], align 4, addrspace(5) @@ -562,12 +569,12 @@ ; FIXME: Is there a cost to using the extload over not? ; GCN-LABEL: {{^}}load_local_v2i16_split: ; GCN: s_waitcnt -; GFX9-NEXT: ds_read_u16 v1, v0 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 offset:2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v1 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: ds_read_u16 v1, v0 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 offset:2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 define <2 x i16> @load_local_v2i16_split(i16 addrspace(3)* %in) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(3)* %in, i32 1 @@ -581,12 +588,12 @@ ; FIXME: Remove waitcnt between reads ; GCN-LABEL: {{^}}load_global_v2i16_split: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_ushort v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_ushort v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_load_short_d16_hi v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v2 +; GFX900-NEXT: s_setpc_b64 define <2 x i16> @load_global_v2i16_split(i16 addrspace(1)* %in) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 1 @@ -600,12 +607,12 @@ ; FIXME: Remove waitcnt between reads ; GCN-LABEL: {{^}}load_flat_v2i16_split: ; GCN: s_waitcnt -; GFX9-NEXT: flat_load_ushort v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: flat_load_short_d16_hi v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: flat_load_ushort v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: flat_load_short_d16_hi v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v2 +; GFX900-NEXT: s_setpc_b64 define <2 x i16> @load_flat_v2i16_split(i16* %in) #0 { entry: %gep = getelementptr inbounds i16, i16* %in, i64 1 @@ -619,12 +626,12 @@ ; FIXME: Remove waitcnt between reads ; GCN-LABEL: {{^}}load_constant_v2i16_split: ; GCN: s_waitcnt -; GFX9-NEXT: global_load_ushort v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: global_load_short_d16_hi v2 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 +; GFX900-NEXT: global_load_ushort v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: global_load_short_d16_hi v2 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: v_mov_b32_e32 v0, v2 +; GFX900-NEXT: s_setpc_b64 define <2 x i16> @load_constant_v2i16_split(i16 addrspace(4)* %in) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 1 @@ -639,11 +646,11 @@ ; FIXME: Is there a cost to using the extload over not? ; GCN-LABEL: {{^}}load_private_v2i16_split: ; GCN: s_waitcnt -; GFX9: buffer_load_ushort v0, off, s[0:3], s5 offset:4{{$}} -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:6 -; GFX9-NEXT: s_waitcnt -; GFX9-NEXT: s_setpc_b64 +; GFX900: buffer_load_ushort v0, off, s[0:3], s5 offset:4{{$}} +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:6 +; GFX900-NEXT: s_waitcnt +; GFX900-NEXT: s_setpc_b64 define <2 x i16> @load_private_v2i16_split(i16 addrspace(5)* byval %in) #0 { entry: %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i32 1 Index: test/CodeGen/AMDGPU/store-hi16.ll =================================================================== --- test/CodeGen/AMDGPU/store-hi16.ll +++ test/CodeGen/AMDGPU/store-hi16.ll @@ -1,13 +1,15 @@ -; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s -; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s +; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX906,NO-D16-HI %s +; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX803,NO-D16-HI %s ; GCN-LABEL: {{^}}store_global_hi_v2i16: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off +; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-NEXT: flat_store_short v[0:1], v2 +; GFX906-NEXT: global_store_short v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -23,10 +25,11 @@ ; GCN-LABEL: {{^}}store_global_hi_v2f16: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off +; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-NEXT: flat_store_short v[0:1], v2 +; GFX906-NEXT: global_store_short v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -42,10 +45,11 @@ ; GCN-LABEL: {{^}}store_global_hi_i32_shift: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off +; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-NEXT: flat_store_short v[0:1], v2 +; GFX906-NEXT: global_store_short v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -60,10 +64,11 @@ ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off +; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_byte v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-NEXT: flat_store_byte v[0:1], v2 +; GFX906-NEXT: global_store_byte v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -79,10 +84,11 @@ ; GCN-LABEL: {{^}}store_global_hi_i8_shift: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off +; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_byte v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-NEXT: flat_store_byte v[0:1], v2 +; GFX906-NEXT: global_store_byte v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -96,13 +102,16 @@ ; GCN-LABEL: {{^}}store_global_hi_v2i16_max_offset: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:4094 +; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:4094 -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_short v[0:1], v2{{$}} + +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: global_store_short v[0:1], v2, off -; VI: flat_store_short v[0:1], v2{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_global_hi_v2i16_max_offset(i16 addrspace(1)* %out, i32 %arg) #0 { @@ -117,13 +126,16 @@ ; GCN-LABEL: {{^}}store_global_hi_v2i16_min_offset: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:-4096{{$}} +; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:-4096{{$}} + +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_short v[0:1], v{{[0-9]$}} -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: global_store_short v[0:1], v2, off -; VI: flat_store_short v[0:1], v{{[0-9]$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_global_hi_v2i16_min_offset(i16 addrspace(1)* %out, i32 %arg) #0 { @@ -137,12 +149,15 @@ ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_max_offset: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:4095 +; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:4095 -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 -; VI: flat_store_byte v[0:1], v{{[0-9]$}} +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_byte v[0:1], v{{[0-9]$}} + +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: global_store_byte v[0:1], v2, off ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -158,13 +173,16 @@ ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_min_offset: ; GCN: s_waitcnt -; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:-4095 +; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:-4095 + +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_byte v[0:1], v{{[0-9]$}} -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: global_store_byte v[0:1], v2, off -; VI: flat_store_byte v[0:1], v{{[0-9]$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_global_hi_v2i16_i8_min_offset(i8 addrspace(1)* %out, i32 %arg) #0 { @@ -180,10 +198,10 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} +; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; NO-D16-HI-NEXT: flat_store_short v[0:1], v2 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -198,10 +216,10 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2f16: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} +; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; NO-D16-HI-NEXT: flat_store_short v[0:1], v2 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -216,10 +234,10 @@ ; GCN-LABEL: {{^}}store_flat_hi_i32_shift: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} +; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_short v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; NO-D16-HI-NEXT: flat_store_short v[0:1], v2 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -234,10 +252,10 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} +; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_byte v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; NO-D16-HI-NEXT: flat_store_byte v[0:1], v2 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -253,10 +271,10 @@ ; GCN-LABEL: {{^}}store_flat_hi_i8_shift: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} +; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; VI-NEXT: flat_store_byte v[0:1], v2 +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; NO-D16-HI-NEXT: flat_store_byte v[0:1], v2 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -270,12 +288,16 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16_max_offset: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:4094{{$}} +; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:4094{{$}} + +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: flat_store_short v[0:1], v2 offset:4094 + +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_short v[0:1], v2{{$}} -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 -; VI: flat_store_short v[0:1], v2{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_flat_hi_v2i16_max_offset(i16* %out, i32 %arg) #0 { @@ -290,11 +312,16 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16_neg_offset: ; GCN: s_waitcnt ; GCN: v_add{{(_co)?}}_{{i|u}}32_e32 -; VI: v_addc_u32_e32 -; GFX9: v_addc_co_u32_e32 -; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} -; VI: flat_store_short v[0:1], v2{{$}} +; GFX803: v_addc_u32_e32 +; GFX900: v_addc_co_u32_e32 + +; GFX906-NEXT: v_lshrrev_b32_e32 +; GFX906-NEXT: v_addc_co_u32_e32 +; GFX906: flat_store_short v[0:1], v2 + +; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}} +; GFX803: flat_store_short v[0:1], v2{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_flat_hi_v2i16_neg_offset(i16* %out, i32 %arg) #0 { @@ -308,12 +335,16 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_max_offset: ; GCN: s_waitcnt -; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:4095{{$}} +; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:4095{{$}} + +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803-DAG: v_add_u32_e32 +; GFX803-DAG: v_addc_u32_e32 +; GFX803: flat_store_byte v[0:1], v2{{$}} + +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: flat_store_byte v[0:1], v2 offset:4095{{$}} -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 -; VI-DAG: v_add_u32_e32 -; VI-DAG: v_addc_u32_e32 -; VI: flat_store_byte v[0:1], v2{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_flat_hi_v2i16_i8_max_offset(i8* %out, i32 %arg) #0 { @@ -329,12 +360,20 @@ ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_neg_offset: ; GCN: s_waitcnt ; GCN-DAG: v_add{{(_co)?}}_{{i|u}}32_e32 -; VI-DAG: v_addc_u32_e32 -; GFX9-DAG: v_addc_co_u32_e32 -; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} -; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2 -; VI: flat_store_byte v[0:1], v2{{$}} +; GFX803-DAG: v_addc_u32_e32 +; GFX900-DAG: v_addc_co_u32_e32 +; GFX906-DAG: v_add_co_u32_e32 + +; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}} + +; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX906-NEXT: v_addc_co_u32_e32 +; GFX906-NEXT: flat_store_byte v[0:1], v2{{$}} + +; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2 +; GFX803: flat_store_byte v[0:1], v2{{$}} + ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 define void @store_flat_hi_v2i16_i8_neg_offset(i8* %out, i32 %arg) #0 { @@ -350,10 +389,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} +; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -369,10 +408,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2f16: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} +; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -388,10 +427,10 @@ ; GCN-LABEL: {{^}}store_private_hi_i32_shift: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} +; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; VI-NEXT: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI-NEXT: buffer_store_short v1, v0, s[0:3], s4 offen{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -406,10 +445,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}} +; GFX900-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}} +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -425,10 +464,10 @@ ; GCN-LABEL: {{^}}store_private_hi_i8_shift: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}} +; GFX900-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}} +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -442,10 +481,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_max_offset: ; GCN: s_waitcnt -; GFX9: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} +; GFX900: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}} -; VI: v_lshrrev_b32_e32 v0, 16, v0 -; VI-NEXT: buffer_store_short v0, off, s[0:3], s5 offset:4094{{$}} +; NO-D16-HI: v_lshrrev_b32_e32 v0, 16, v0 +; NO-D16-HI-NEXT: buffer_store_short v0, off, s[0:3], s5 offset:4094{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -463,10 +502,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_nooff: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s4{{$}} +; GFX900-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s4{{$}} -; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; VI-NEXT: buffer_store_short v0, off, s[0:3], s4{{$}} +; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; NO-D16-HI-NEXT: buffer_store_short v0, off, s[0:3], s4{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -483,10 +522,10 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_nooff: ; GCN: s_waitcnt -; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s4{{$}} +; GFX900-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s4{{$}} -; VI: v_lshrrev_b32_e32 v0, 16, v0 -; VI: buffer_store_byte v0, off, s[0:3], s4{{$}} +; NO-D16-HI: v_lshrrev_b32_e32 v0, 16, v0 +; NO-D16-HI: buffer_store_byte v0, off, s[0:3], s4{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -502,10 +541,10 @@ ; GCN-LABEL: {{^}}store_local_hi_v2i16: ; GCN: s_waitcnt -; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}} +; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: ds_write_b16 v0, v1 +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: ds_write_b16 v0, v1 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -521,10 +560,10 @@ ; GCN-LABEL: {{^}}store_local_hi_v2f16: ; GCN: s_waitcnt -; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}} +; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: ds_write_b16 v0, v1 +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: ds_write_b16 v0, v1 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -540,10 +579,10 @@ ; GCN-LABEL: {{^}}store_local_hi_i32_shift: ; GCN: s_waitcnt -; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}} +; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: ds_write_b16 v0, v1 +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: ds_write_b16 v0, v1 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -558,10 +597,10 @@ ; GCN-LABEL: {{^}}store_local_hi_v2i16_i8: ; GCN: s_waitcnt -; GFX9-NEXT: ds_write_b8_d16_hi v0, v1{{$}} +; GFX900-NEXT: ds_write_b8_d16_hi v0, v1{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: ds_write_b8 v0, v1 +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: ds_write_b8 v0, v1 ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -576,10 +615,10 @@ ; GCN-LABEL: {{^}}store_local_hi_v2i16_max_offset: ; GCN: s_waitcnt -; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:65534{{$}} +; GFX900-NEXT: ds_write_b16_d16_hi v0, v1 offset:65534{{$}} -; VI: v_lshrrev_b32_e32 v1, 16, v1 -; VI: ds_write_b16 v0, v1 offset:65534{{$}} +; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1 +; NO-D16-HI: ds_write_b16 v0, v1 offset:65534{{$}} ; GCN-NEXT: s_waitcnt ; GCN-NEXT: s_setpc_b64 @@ -595,8 +634,8 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_to_offset: ; GCN: s_waitcnt -; GFX9: buffer_store_dword -; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094 +; GFX900: buffer_store_dword +; GFX900-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094 define void @store_private_hi_v2i16_to_offset(i32 %arg) #0 { entry: %obj0 = alloca [10 x i32], align 4, addrspace(5) @@ -612,8 +651,8 @@ ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_to_offset: ; GCN: s_waitcnt -; GFX9: buffer_store_dword -; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095 +; GFX900: buffer_store_dword +; GFX900-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095 define void @store_private_hi_v2i16_i8_to_offset(i32 %arg) #0 { entry: %obj0 = alloca [10 x i32], align 4, addrspace(5)