AMDGPU: Add D16 instructions preserve unused bits feature

[kzhuravl]

Summary

• Predicate D16 patterns on this new feature
• Added this new feature to gfx900/2/4

[arsenm]

lib/Target/AMDGPU/AMDGPU.td

319–323 ↗

(On Diff #144935)

This isn't the right to handle this. The instructions themselves do exist, so should not be dependent on a subtarget feature. The assembler/disassembler should still accept them. This should be a separately named feature

[arsenm]

lib/Target/AMDGPU/AMDGPU.td

319–323 ↗

(On Diff #144935)

We also still can select them, just under different circumstances

[kzhuravl]

Address review feedback

[kzhuravl]

lib/Target/AMDGPU/AMDGPU.td

319–323 ↗

(On Diff #144935)

Can this be done in a separate change?

[arsenm]

LGTM. Adding run lines with a target without the feature to the assembler/disassembler tests wouldn't hurt

lib/Target/AMDGPU/AMDGPU.td

319–323 ↗

(On Diff #144935)

As far as selecting them, yes

[arsenm]

lib/Target/AMDGPU/AMDGPU.td

329 ↗

(On Diff #145139)

Should probably note the alternative is to zero them, and specify these are memory instructions

[arsenm]

Does this actually apply to DS instructions?

---

From: Konstantin Zhuravlyov via Phabricator <[e-mail]>
Sent: Friday, May 4, 2018 1:11:31 PM
To: Zhuravlyov, Konstantin; Arsenault, Matthew; Mekhanoshin, Stanislav
Cc: [e-mail]; [e-mail]; Liu, Yaxun (Sam); Stuttard, David; [e-mail]; Tye, Tony; [e-mail]
Subject: [PATCH] D46366: AMDGPU: Add D16 instructions preserve unused bits feature

This revision was automatically updated to reflect the committed changes.
kzhuravl marked an inline comment as done.
Closed by commit rL331551: AMDGPU: Add D16 instructions preserve unused bits feature (authored by kzhuravl, committed by ).

Changed prior to commit:

https://reviews.llvm.org/D46366?vs=145139&id=145264#toc

Repository:

rL LLVM

https://reviews.llvm.org/D46366

Files:

llvm/trunk/lib/Target/AMDGPU/AMDGPU.td
llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h
llvm/trunk/lib/Target/AMDGPU/BUFInstructions.td
llvm/trunk/lib/Target/AMDGPU/DSInstructions.td
llvm/trunk/lib/Target/AMDGPU/FLATInstructions.td
llvm/trunk/test/CodeGen/AMDGPU/load-hi16.ll
llvm/trunk/test/CodeGen/AMDGPU/store-hi16.ll
llvm/trunk/test/MC/AMDGPU/gfx9_asm_all.s

[kzhuravl]

Yes.

llvm/trunk/lib/Target/AMDGPU/AMDGPU.td

 >;
 
 def FeatureDLInsts : SubtargetFeature<"dl-insts",
   "HasDLInsts",
   "true",
   "Has deep learning instructions"
 >;
 
+def FeatureD16PreservesUnusedBits : SubtargetFeature<
+  "d16-preserves-unused-bits",
+  "D16PreservesUnusedBits",
+  "true",
+  "D16 memory instructions preserve unused bits rather than zeroing them out"
+>;
+
 //===------------------------------------------------------------===//
 // Subtarget Features (options and debugging)
 //===------------------------------------------------------------===//
 
 // Some instructions do not support denormals despite this flag. Using
 // fp32 denormals also causes instructions to run at the double
 // precision rate for the device.
 def FeatureFP32Denormals : SubtargetFeature<"fp32-denormals",

 def FeatureISAVersion8_1_0 : SubtargetFeatureISAVersion <8,1,0,
   [FeatureVolcanicIslands,
    FeatureLDSBankCount16,
    FeatureXNACK]>;
 
 def FeatureISAVersion9_0_0 : SubtargetFeatureISAVersion <9,0,0,
   [FeatureGFX9,
    FeatureMadMixInsts,
-   FeatureLDSBankCount32
+   FeatureLDSBankCount32,
+   FeatureD16PreservesUnusedBits
    ]>;
 
 def FeatureISAVersion9_0_2 : SubtargetFeatureISAVersion <9,0,2,
   [FeatureGFX9,
    FeatureMadMixInsts,
    FeatureLDSBankCount32,
-   FeatureXNACK
+   FeatureXNACK,
+   FeatureD16PreservesUnusedBits
    ]>;
 
 def FeatureISAVersion9_0_4 : SubtargetFeatureISAVersion <9,0,4,
   [FeatureGFX9,
    FeatureLDSBankCount32,
-   FeatureFmaMixInsts]>;
+   FeatureFmaMixInsts,
+   FeatureD16PreservesUnusedBits]>;
 
 def FeatureISAVersion9_0_6 : SubtargetFeatureISAVersion <9,0,6,
   [FeatureGFX9,
    HalfRate64Ops,
    FeatureFmaMixInsts,
    FeatureLDSBankCount32,
    FeatureDLInsts]>;
 

 def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">,
   AssemblerPredicate<"FeatureGFX9Insts">;
 
 def HasUnpackedD16VMem : Predicate<"Subtarget->hasUnpackedD16VMem()">,
   AssemblerPredicate<"FeatureUnpackedD16VMem">;
 def HasPackedD16VMem : Predicate<"!Subtarget->hasUnpackedD16VMem()">,
   AssemblerPredicate<"!FeatureUnpackedD16VMem">;
 
+def D16PreservesUnusedBits : Predicate<"Subtarget->d16PreservesUnusedBits()">,
+  AssemblerPredicate<"FeatureD16PreservesUnusedBits">;
 
 def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">;
 def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">;
 
 def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">,
   AssemblerPredicate<"FeatureGFX9Insts">;
 
 def HasAddNoCarryInsts : Predicate<"Subtarget->hasAddNoCarryInsts()">,

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h

protected:
   bool HasSDWA;
   bool HasSDWAOmod;
   bool HasSDWAScalar;
   bool HasSDWASdst;
   bool HasSDWAMac;
   bool HasSDWAOutModsVOPC;
   bool HasDPP;
   bool HasDLInsts;
+  bool D16PreservesUnusedBits;
   bool FlatAddressSpace;
   bool FlatInstOffsets;
   bool FlatGlobalInsts;
   bool FlatScratchInsts;
   bool AddNoCarryInsts;
   bool HasUnpackedD16VMem;
   bool R600ALUInst;
   bool CaymanISA;
public:
   bool vmemWriteNeedsExpWaitcnt() const {
     return getGeneration() < SEA_ISLANDS;
   }
 
   bool hasDLInsts() const {
     return HasDLInsts;
   }
 
+  bool d16PreservesUnusedBits() const {
+    return D16PreservesUnusedBits;
+  }
+
   /// Returns the offset in bytes from the start of the input buffer
   ///        of the first explicit kernel argument.
   unsigned getExplicitKernelArgOffset(const MachineFunction &MF) const {
     return isAmdCodeObjectV2(MF) ? 0 : 36;
   }
 
   unsigned getAlignmentForImplicitArgPtr() const {
     return isAmdHsaOS() ? 8 : 4;

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

: AMDGPUGenSubtargetInfo(TT, GPU, FS),
     HasSDWA(false),
     HasSDWAOmod(false),
     HasSDWAScalar(false),
     HasSDWASdst(false),
     HasSDWAMac(false),
     HasSDWAOutModsVOPC(false),
     HasDPP(false),
     HasDLInsts(false),
+    D16PreservesUnusedBits(false),
     FlatAddressSpace(false),
     FlatInstOffsets(false),
     FlatGlobalInsts(false),
     FlatScratchInsts(false),
     AddNoCarryInsts(false),
     HasUnpackedD16VMem(false),
 
     R600ALUInst(false),

llvm/trunk/lib/Target/AMDGPU/BUFInstructions.td

 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>;
 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>;
 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>;
 
 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, i16, load_private>;
 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, i16, az_extloadi8_private>;
 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, i16, sextloadi8_private>;
 }

 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>;
 
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
  // Hiding the extract high pattern in the PatFrag seems to not
  // automatically increase the complexity.
 let AddedComplexity = 1 in {
 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
 }
 }
 

llvm/trunk/lib/Target/AMDGPU/DSInstructions.td

 
 let AddedComplexity = 100 in {
 
 defm : DSReadPat_mc <DS_READ_B64, v2i32, "load_align8_local">;
 defm : DSReadPat_mc <DS_READ_B128, v4i32, "load_align16_local">;
 
 } // End AddedComplexity = 100
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
 let AddedComplexity = 100 in {
 defm : DSReadPat_Hi16<DS_READ_U16_D16_HI, load_local>;
 defm : DSReadPat_Hi16<DS_READ_U8_D16_HI, az_extloadi8_local>;
 defm : DSReadPat_Hi16<DS_READ_I8_D16_HI, sextloadi8_local>;
 
 defm : DSReadPat_Lo16<DS_READ_U16_D16, load_local>;
 defm : DSReadPat_Lo16<DS_READ_U8_D16, az_extloadi8_local>;
 defm : DSReadPat_Lo16<DS_READ_I8_D16, sextloadi8_local>;

 }
 
 defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
 defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
 defm : DSWritePat_mc <DS_WRITE_B32, i32, "store_local">;
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
 def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_local_hi16>;
 def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_local_hi16>;
 }
 
 
 class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, PatFrag frag> : GCNPat <
   (v2i32 (frag (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
   (inst $ptr, $offset0, $offset1, (i1 0))

llvm/trunk/lib/Target/AMDGPU/FLATInstructions.td

 def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_or_global, i64>;
 def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>;
 def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>;
 def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>;
 
 def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
 def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>;
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
 def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>;
 def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>;
 
 let AddedComplexity = 3 in {
 defm : FlatLoadPat_Hi16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_flat>;
 defm : FlatLoadPat_Hi16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_flat>;
 defm : FlatLoadPat_Hi16 <FLAT_LOAD_SHORT_D16_HI, load_flat>;
 }

 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;
 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, store_global, i16>;
 def : FlatStoreSignedPat <GLOBAL_STORE_DWORD, store_global, i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, store_global, v2i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, v4i32>;
 
-let OtherPredicates = [HasD16LoadStore] in {
+let OtherPredicates = [D16PreservesUnusedBits] in {
 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>;
 
 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_global>;
 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_global>;
 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SHORT_D16_HI, load_global>;
 
 defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_global>;

llvm/trunk/test/CodeGen/AMDGPU/load-hi16.ll

-; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s
+; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX906,NO-D16-HI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX803,NO-D16-HI %s
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_undeflo:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16_d16_hi v0, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16_d16_hi v0, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
+; NO-D16-HI: ds_read_u16 v
 define <2 x i16> @load_local_hi_v2i16_undeflo(i16 addrspace(3)* %in) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build = insertelement <2 x i16> undef, i16 %load, i32 1
   ret <2 x i16> %build
 }
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v1
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
+; NO-D16-HI: ds_read_u16 v
 define <2 x i16> @load_local_hi_v2i16_reglo(i16 addrspace(3)* %in, i16 %reg) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   ret <2 x i16> %build1
 }
 
 ; Show that we get reasonable regalloc without physreg constraints.
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
+; NO-D16-HI: ds_read_u16 v
 define void @load_local_hi_v2i16_reglo_vreg(i16 addrspace(3)* %in, i16 %reg) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo:
 ; GCN: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: v_mov_b32_e32 v1, 0
+; GFX900-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v1
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
+; NO-D16-HI: ds_read_u16 v
 define <2 x i16> @load_local_hi_v2i16_zerolo(i16 addrspace(3)* %in) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 1
   ret <2 x i16> %build
 }
 
 ; FIXME: Remove m0 initialization
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo_shift:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16 v0, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16 v0, v0
+; GFX900-NEXT: s_waitcnt lgkmcnt(0)
+; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
-; VI: v_lshlrev_b32_e32 v0, 16, v0
+; NO-D16-HI: ds_read_u16 v
+; NO-D16-HI: v_lshlrev_b32_e32 v0, 16, v0
 define i32 @load_local_hi_v2i16_zerolo_shift(i16 addrspace(3)* %in) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in
   %zext = zext i16 %load to i32
   %shift = shl i32 %zext, 16
   ret i32 %shift
 }
 
 ; GCN-LABEL: {{^}}load_local_hi_v2f16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u16 v
+; NO-D16-HI: ds_read_u16 v
 define void @load_local_hi_v2f16_reglo_vreg(half addrspace(3)* %in, half %reg) #0 {
 entry:
   %load = load half, half addrspace(3)* %in
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_zexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u8_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u8_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_u8 v
+; NO-D16-HI: ds_read_u8 v
 define void @load_local_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_sexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_i8_d16_hi v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_i8_d16_hi v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: ds_read_i8 v
+; NO-D16-HI: ds_read_i8 v
 define void @load_local_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
 entry:
   %load = load i8, i8 addrspace(3)* %in
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 define void @load_global_hi_v2i16_reglo_vreg(i16 addrspace(1)* %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 -2047
   %load = load i16, i16 addrspace(1)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_global_hi_v2f16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 define void @load_global_hi_v2f16_reglo_vreg(half addrspace(1)* %in, half %reg) #0 {
 entry:
   %gep = getelementptr inbounds half, half addrspace(1)* %in, i64 -2047
   %load = load half, half addrspace(1)* %gep
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_zexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_ubyte_d16_hi v2, v[0:1], off offset:-4095
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_ubyte_d16_hi v2, v[0:1], off offset:-4095
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 define void @load_global_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
   %load = load i8, i8 addrspace(1)* %gep
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_sexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_sbyte_d16_hi v2, v[0:1], off offset:-4095
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_sbyte_d16_hi v2, v[0:1], off offset:-4095
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 define void @load_global_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
   %load = load i8, i8 addrspace(1)* %gep
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: load_flat_hi_v2i16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1]
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
-
-; VI: flat_load_ushort v{{[0-9]+}}
-; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_or_b32_sdwa
+; GFX900-NEXT: flat_load_short_d16_hi v2, v[0:1]
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
+
+; NO-D16-HI: flat_load_ushort v{{[0-9]+}}
+; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; GFX803: v_or_b32_sdwa
+; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16,
 define void @load_flat_hi_v2i16_reglo_vreg(i16* %in, i16 %reg) #0 {
 entry:
   %load = load i16, i16* %in
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_flat_hi_v2f16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1]
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
-
-; VI: flat_load_ushort v{{[0-9]+}}
-; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_or_b32_sdwa
+; GFX900-NEXT: flat_load_short_d16_hi v2, v[0:1]
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
+
+; NO-D16-HI: flat_load_ushort v{{[0-9]+}}
+; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; GFX803: v_or_b32_sdwa
+; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16,
 define void @load_flat_hi_v2f16_reglo_vreg(half* %in, half %reg) #0 {
 entry:
   %load = load half, half* %in
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_zexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_load_ubyte_d16_hi v2, v[0:1]
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
-
-; VI: flat_load_ubyte v{{[0-9]+}}
-; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_or_b32_sdwa
+; GFX900-NEXT: flat_load_ubyte_d16_hi v2, v[0:1]
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
+
+; NO-D16-HI: flat_load_ubyte v{{[0-9]+}}
+; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; GFX803: v_or_b32_sdwa
+; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16,
 define void @load_flat_hi_v2i16_reglo_vreg_zexti8(i8* %in, i16 %reg) #0 {
 entry:
   %load = load i8, i8* %in
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_sexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_load_sbyte_d16_hi v2, v[0:1]
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v[0:1], v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
-
-; VI: flat_load_sbyte v{{[0-9]+}}
-; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_or_b32_sdwa
+; GFX900-NEXT: flat_load_sbyte_d16_hi v2, v[0:1]
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v[0:1], v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
+
+; NO-D16-HI: flat_load_sbyte v{{[0-9]+}}
+; GFX803: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; GFX803: v_or_b32_sdwa
+; GFX906: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16,
 define void @load_flat_hi_v2i16_reglo_vreg_sexti8(i8* %in, i16 %reg) #0 {
 entry:
   %load = load i8, i8* %in
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
 define void @load_private_hi_v2i16_reglo_vreg(i16 addrspace(5)* byval %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i64 2045
   %load = load i16, i16 addrspace(5)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
 define void @load_private_hi_v2f16_reglo_vreg(half addrspace(5)* byval %in, half %reg) #0 {
 entry:
   %gep = getelementptr inbounds half, half addrspace(5)* %in, i64 2045
   %load = load half, half addrspace(5)* %gep
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s4 offset:4094{{$}}
-; GFX9: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_short_d16_hi v0, off, s[0:3], s4 offset:4094{{$}}
+; GFX900: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
 define void @load_private_hi_v2i16_reglo_vreg_nooff(i16 addrspace(5)* byval %in, i16 %reg) #0 {
 entry:
   %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*)
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff:
 ; GCN: s_waitcnt
-; GFX9-NEXT: buffer_load_short_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: buffer_load_short_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
 define void @load_private_hi_v2f16_reglo_vreg_nooff(half addrspace(5)* %in, half %reg) #0 {
 entry:
   %load = load volatile half, half addrspace(5)* inttoptr (i32 4094 to half addrspace(5)*)
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_ubyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_ubyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+; NO-D16-HI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
 define void @load_private_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(5)* byval %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4091
   %load = load i8, i8 addrspace(5)* %gep
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_sbyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_sbyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+; NO-D16-HI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
 define void @load_private_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(5)* byval %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i8, i8 addrspace(5)* %in, i64 4091
   %load = load i8, i8 addrspace(5)* %gep
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_zexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
 define void @load_private_hi_v2i16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, i16 %reg) #0 {
 entry:
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_sexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: buffer_load_sbyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: buffer_load_sbyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}}
+; NO-D16-HI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}}
 define void @load_private_hi_v2i16_reglo_vreg_nooff_sexti8(i8 addrspace(5)* %in, i16 %reg) #0 {
 entry:
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff_zexti8:
 ; GCN: s_waitcnt
-; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+; NO-D16-HI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
 define void @load_private_hi_v2f16_reglo_vreg_nooff_zexti8(i8 addrspace(5)* %in, half %reg) #0 {
 entry:
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = zext i8 %load to i16
   %bc.ext = bitcast i16 %ext to half
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bc.ext, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_constant_hi_v2i16_reglo_vreg:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: flat_load_ushort
+; GFX803: flat_load_ushort
+; GFX906: global_load_ushort
 define void @load_constant_hi_v2i16_reglo_vreg(i16 addrspace(4)* %in, i16 %reg) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 -2047
   %load = load i16, i16 addrspace(4)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: load_constant_hi_v2f16_reglo_vreg
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_store_dword
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_store_dword
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 
-; VI: flat_load_ushort
+; GFX803: flat_load_ushort
+; GFX906: global_load_ushort
 define void @load_constant_hi_v2f16_reglo_vreg(half addrspace(4)* %in, half %reg) #0 {
 entry:
   %gep = getelementptr inbounds half, half addrspace(4)* %in, i64 -2047
   %load = load half, half addrspace(4)* %gep
   %build0 = insertelement <2 x half> undef, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
 ; Local object gives known offset, so requires converting from offen
 ; to offset variant.
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_to_offset:
-; GFX9: buffer_store_dword
-; GFX9-NEXT: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4094
+; GFX900: buffer_store_dword
+; GFX900-NEXT: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4094
 define void @load_private_hi_v2i16_reglo_vreg_to_offset(i16 %reg) #0 {
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i16], align 2, addrspace(5)
   %bc = bitcast [10 x i32] addrspace(5)* %obj0 to i32 addrspace(5)*
   store volatile i32 123, i32 addrspace(5)* %bc
   %gep = getelementptr inbounds [4096 x i16], [4096 x i16] addrspace(5)* %obj1, i32 0, i32 2025
   %load = load i16, i16 addrspace(5)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8_to_offset:
-; GFX9: buffer_store_dword
-; GFX9-NEXT: buffer_load_sbyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
+; GFX900: buffer_store_dword
+; GFX900-NEXT: buffer_load_sbyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
 define void @load_private_hi_v2i16_reglo_vreg_sexti8_to_offset(i16 %reg) #0 {
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
   %bc = bitcast [10 x i32] addrspace(5)* %obj0 to i32 addrspace(5)*
   store volatile i32 123, i32 addrspace(5)* %bc
   %gep = getelementptr inbounds [4096 x i8], [4096 x i8] addrspace(5)* %obj1, i32 0, i32 4051
   %load = load i8, i8 addrspace(5)* %gep
   %ext = sext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8_to_offset:
-; GFX9: buffer_store_dword
-; GFX9-NEXT: buffer_load_ubyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
+; GFX900: buffer_store_dword
+; GFX900-NEXT: buffer_load_ubyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
 define void @load_private_hi_v2i16_reglo_vreg_zexti8_to_offset(i16 %reg) #0 {
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
   %bc = bitcast [10 x i32] addrspace(5)* %obj0 to i32 addrspace(5)*
   store volatile i32 123, i32 addrspace(5)* %bc
   %gep = getelementptr inbounds [4096 x i8], [4096 x i8] addrspace(5)* %obj1, i32 0, i32 4051
   %load = load i8, i8 addrspace(5)* %gep
   %ext = zext i8 %load to i16
   %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; FIXME: Remove m0 init and waitcnt between reads
 ; FIXME: Is there a cost to using the extload over not?
 ; GCN-LABEL: {{^}}load_local_v2i16_split:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_read_u16 v1, v0
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 offset:2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: ds_read_u16 v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: ds_read_u16_d16_hi v1, v0 offset:2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v1
+; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @load_local_v2i16_split(i16 addrspace(3)* %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(3)* %in, i32 1
   %load0 = load volatile i16, i16 addrspace(3)* %in
   %load1 = load volatile i16, i16 addrspace(3)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
 
 ; FIXME: Remove waitcnt between reads
 ; GCN-LABEL: {{^}}load_global_v2i16_split:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_ushort v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_ushort v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_load_short_d16_hi v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v2
+; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @load_global_v2i16_split(i16 addrspace(1)* %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 1
   %load0 = load volatile i16, i16 addrspace(1)* %in
   %load1 = load volatile i16, i16 addrspace(1)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
 
 ; FIXME: Remove waitcnt between reads
 ; GCN-LABEL: {{^}}load_flat_v2i16_split:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_load_ushort v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: flat_load_short_d16_hi v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: flat_load_ushort v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: flat_load_short_d16_hi v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v2
+; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @load_flat_v2i16_split(i16* %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16* %in, i64 1
   %load0 = load volatile i16, i16* %in
   %load1 = load volatile i16, i16* %gep
   %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
 
 ; FIXME: Remove waitcnt between reads
 ; GCN-LABEL: {{^}}load_constant_v2i16_split:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_load_ushort v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: global_load_short_d16_hi v2
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-NEXT: s_setpc_b64
+; GFX900-NEXT: global_load_ushort v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: global_load_short_d16_hi v2
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, v2
+; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @load_constant_v2i16_split(i16 addrspace(4)* %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 1
   %load0 = load volatile i16, i16 addrspace(4)* %in
   %load1 = load volatile i16, i16 addrspace(4)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
 
 ; FIXME: Remove m0 init and waitcnt between reads
 ; FIXME: Is there a cost to using the extload over not?
 ; GCN-LABEL: {{^}}load_private_v2i16_split:
 ; GCN: s_waitcnt
-; GFX9: buffer_load_ushort v0, off, s[0:3], s5 offset:4{{$}}
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:6
-; GFX9-NEXT: s_waitcnt
-; GFX9-NEXT: s_setpc_b64
+; GFX900: buffer_load_ushort v0, off, s[0:3], s5 offset:4{{$}}
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:6
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @load_private_v2i16_split(i16 addrspace(5)* byval %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, i16 addrspace(5)* %in, i32 1
   %load0 = load volatile i16, i16 addrspace(5)* %in
   %load1 = load volatile i16, i16 addrspace(5)* %gep
   %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
 
 attributes #0 = { nounwind }

llvm/trunk/test/CodeGen/AMDGPU/store-hi16.ll

-; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900 %s
+; RUN: llc -march=amdgcn -mcpu=gfx906 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX906,NO-D16-HI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX803,NO-D16-HI %s
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-NEXT: flat_store_short v[0:1], v2
+; GFX906-NEXT: global_store_short v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16(i16 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   store i16 %hi, i16 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2f16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-NEXT: flat_store_short v[0:1], v2
+; GFX906-NEXT: global_store_short v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2f16(half addrspace(1)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x half>
   %hi = extractelement <2 x half> %value, i32 1
   store half %hi, half addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_i32_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-NEXT: flat_store_short v[0:1], v2
+; GFX906-NEXT: global_store_short v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_i32_shift(i16 addrspace(1)* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i16
   store i16 %hi, i16 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off
+; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_byte v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-NEXT: flat_store_byte v[0:1], v2
+; GFX906-NEXT: global_store_byte v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16_i8(i8 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   store i8 %trunc, i8 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_i8_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off
+; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_byte v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-NEXT: flat_store_byte v[0:1], v2
+; GFX906-NEXT: global_store_byte v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_i8_shift(i8 addrspace(1)* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i8
   store i8 %hi, i8 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16_max_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:4094
+; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:4094
 
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_short v[0:1], v2{{$}}
+
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: global_store_short v[0:1], v2, off
 
-; VI: flat_store_short v[0:1], v2{{$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16_max_offset(i16 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 2047
   store i16 %hi, i16 addrspace(1)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16_min_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:-4096{{$}}
+; GFX900-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:-4096{{$}}
+
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_short v[0:1], v{{[0-9]$}}
 
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: global_store_short v[0:1], v2, off
 
-; VI: flat_store_short v[0:1], v{{[0-9]$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16_min_offset(i16 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 -2048
   store i16 %hi, i16 addrspace(1)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_max_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:4095
+; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:4095
+
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_byte v[0:1], v{{[0-9]$}}
 
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
-; VI: flat_store_byte v[0:1], v{{[0-9]$}}
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: global_store_byte v[0:1], v2, off
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16_i8_max_offset(i8 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 4095
   store i8 %trunc, i8 addrspace(1)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_min_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:-4095
+; GFX900-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:-4095
 
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_byte v[0:1], v{{[0-9]$}}
+
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: global_store_byte v[0:1], v2, off
 
-; VI: flat_store_byte v[0:1], v{{[0-9]$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_global_hi_v2i16_i8_min_offset(i8 addrspace(1)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   %gep = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 -4095
   store i8 %trunc, i8 addrspace(1)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NO-D16-HI-NEXT: flat_store_short v[0:1], v2
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16(i16* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   store i16 %hi, i16* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2f16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NO-D16-HI-NEXT: flat_store_short v[0:1], v2
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2f16(half* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x half>
   %hi = extractelement <2 x half> %value, i32 1
   store half %hi, half* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_i32_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_short v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NO-D16-HI-NEXT: flat_store_short v[0:1], v2
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_i32_shift(i16* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i16
   store i16 %hi, i16* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_byte v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NO-D16-HI-NEXT: flat_store_byte v[0:1], v2
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16_i8(i8* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   store i8 %trunc, i8* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_i8_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT: flat_store_byte v[0:1], v2
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; NO-D16-HI-NEXT: flat_store_byte v[0:1], v2
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_i8_shift(i8* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i8
   store i8 %hi, i8* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_max_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:4094{{$}}
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:4094{{$}}
+
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: flat_store_short v[0:1], v2 offset:4094
+
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_short v[0:1], v2{{$}}
 
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
-; VI: flat_store_short v[0:1], v2{{$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16_max_offset(i16* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16* %out, i64 2047
   store i16 %hi, i16* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_neg_offset:
 ; GCN: s_waitcnt
 ; GCN: v_add{{(_co)?}}_{{i|u}}32_e32
-; VI: v_addc_u32_e32
-; GFX9: v_addc_co_u32_e32
 
-; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
-; VI: flat_store_short v[0:1], v2{{$}}
+; GFX803: v_addc_u32_e32
+; GFX900: v_addc_co_u32_e32
+
+; GFX906-NEXT: v_lshrrev_b32_e32
+; GFX906-NEXT: v_addc_co_u32_e32
+; GFX906: flat_store_short v[0:1], v2
+
+; GFX900-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; GFX803: flat_store_short v[0:1], v2{{$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16_neg_offset(i16* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16* %out, i64 -1023
   store i16 %hi, i16* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_max_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:4095{{$}}
+; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:4095{{$}}
+
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803-DAG: v_add_u32_e32
+; GFX803-DAG: v_addc_u32_e32
+; GFX803: flat_store_byte v[0:1], v2{{$}}
+
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: flat_store_byte v[0:1], v2 offset:4095{{$}}
 
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
-; VI-DAG: v_add_u32_e32
-; VI-DAG: v_addc_u32_e32
-; VI: flat_store_byte v[0:1], v2{{$}}
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16_i8_max_offset(i8* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   %gep = getelementptr inbounds i8, i8* %out, i64 4095
   store i8 %trunc, i8* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_neg_offset:
 ; GCN: s_waitcnt
 ; GCN-DAG: v_add{{(_co)?}}_{{i|u}}32_e32
-; VI-DAG: v_addc_u32_e32
-; GFX9-DAG: v_addc_co_u32_e32
 
-; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
-; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
-; VI: flat_store_byte v[0:1], v2{{$}}
+; GFX803-DAG: v_addc_u32_e32
+; GFX900-DAG: v_addc_co_u32_e32
+; GFX906-DAG: v_add_co_u32_e32
+
+; GFX900-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+
+; GFX906-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: v_addc_co_u32_e32
+; GFX906-NEXT: flat_store_byte v[0:1], v2{{$}}
+
+; GFX803-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; GFX803: flat_store_byte v[0:1], v2{{$}}
+
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_flat_hi_v2i16_i8_neg_offset(i8* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   %gep = getelementptr inbounds i8, i8* %out, i64 -4095
   store i8 %trunc, i8* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2i16(i16 addrspace(5)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   store i16 %hi, i16 addrspace(5)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2f16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2f16(half addrspace(5)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x half>
   %hi = extractelement <2 x half> %value, i32 1
   store half %hi, half addrspace(5)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_i32_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+; GFX900-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI-NEXT: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_i32_shift(i16 addrspace(5)* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i16
   store i16 %hi, i16 addrspace(5)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+; GFX900-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2i16_i8(i8 addrspace(5)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   store i8 %trunc, i8 addrspace(5)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_i8_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+; GFX900-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_i8_shift(i8 addrspace(5)* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i8
   store i8 %hi, i8 addrspace(5)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_max_offset:
 ; GCN: s_waitcnt
-; GFX9: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX900: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
 
-; VI: v_lshrrev_b32_e32 v0, 16, v0
-; VI-NEXT: buffer_store_short v0, off, s[0:3], s5 offset:4094{{$}}
+; NO-D16-HI: v_lshrrev_b32_e32 v0, 16, v0
+; NO-D16-HI-NEXT: buffer_store_short v0, off, s[0:3], s5 offset:4094{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2i16_max_offset(i16 addrspace(5)* byval %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16 addrspace(5)* %out, i64 2045
   store i16 %hi, i16 addrspace(5)* %gep
   ret void
 }
 
 
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_nooff:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s4{{$}}
+; GFX900-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s4{{$}}
 
-; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; VI-NEXT: buffer_store_short v0, off, s[0:3], s4{{$}}
+; NO-D16-HI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; NO-D16-HI-NEXT: buffer_store_short v0, off, s[0:3], s4{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2i16_nooff(i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   store volatile i16 %hi, i16 addrspace(5)* null
   ret void
 }
 
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_nooff:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s4{{$}}
+; GFX900-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s4{{$}}
 
-; VI: v_lshrrev_b32_e32 v0, 16, v0
-; VI: buffer_store_byte v0, off, s[0:3], s4{{$}}
+; NO-D16-HI: v_lshrrev_b32_e32 v0, 16, v0
+; NO-D16-HI: buffer_store_byte v0, off, s[0:3], s4{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_private_hi_v2i16_i8_nooff(i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   store volatile i8 %trunc, i8 addrspace(5)* null
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_local_hi_v2i16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: ds_write_b16 v0, v1
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: ds_write_b16 v0, v1
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_local_hi_v2i16(i16 addrspace(3)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   store i16 %hi, i16 addrspace(3)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_local_hi_v2f16:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: ds_write_b16 v0, v1
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: ds_write_b16 v0, v1
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_local_hi_v2f16(half addrspace(3)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x half>
   %hi = extractelement <2 x half> %value, i32 1
   store half %hi, half addrspace(3)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_local_hi_i32_shift:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+; GFX900-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: ds_write_b16 v0, v1
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: ds_write_b16 v0, v1
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_local_hi_i32_shift(i16 addrspace(3)* %out, i32 %value) #0 {
 entry:
   %hi32 = lshr i32 %value, 16
   %hi = trunc i32 %hi32 to i16
   store i16 %hi, i16 addrspace(3)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_local_hi_v2i16_i8:
 ; GCN: s_waitcnt
 
-; GFX9-NEXT: ds_write_b8_d16_hi v0, v1{{$}}
+; GFX900-NEXT: ds_write_b8_d16_hi v0, v1{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: ds_write_b8 v0, v1
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: ds_write_b8 v0, v1
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_local_hi_v2i16_i8(i8 addrspace(3)* %out, i32 %arg) #0 {
 entry:
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %trunc = trunc i16 %hi to i8
   store i8 %trunc, i8 addrspace(3)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_local_hi_v2i16_max_offset:
 ; GCN: s_waitcnt
-; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:65534{{$}}
+; GFX900-NEXT: ds_write_b16_d16_hi v0, v1 offset:65534{{$}}
 
-; VI: v_lshrrev_b32_e32 v1, 16, v1
-; VI: ds_write_b16 v0, v1 offset:65534{{$}}
+; NO-D16-HI: v_lshrrev_b32_e32 v1, 16, v1
+; NO-D16-HI: ds_write_b16 v0, v1 offset:65534{{$}}
 
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @store_local_hi_v2i16_max_offset(i16 addrspace(3)* %out, i32 %arg) #0 {
 entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16 addrspace(3)* %out, i64 32767
   store i16 %hi, i16 addrspace(3)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_to_offset:
 ; GCN: s_waitcnt
-; GFX9: buffer_store_dword
-; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094
+; GFX900: buffer_store_dword
+; GFX900-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094
 define void @store_private_hi_v2i16_to_offset(i32 %arg) #0 {
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i16], align 2, addrspace(5)
   %bc = bitcast [10 x i32] addrspace(5)* %obj0 to i32 addrspace(5)*
   store volatile i32 123, i32 addrspace(5)* %bc
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds [4096 x i16], [4096 x i16] addrspace(5)* %obj1, i32 0, i32 2025
   store i16 %hi, i16 addrspace(5)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_to_offset:
 ; GCN: s_waitcnt
-; GFX9: buffer_store_dword
-; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095
+; GFX900: buffer_store_dword
+; GFX900-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095
 define void @store_private_hi_v2i16_i8_to_offset(i32 %arg) #0 {
 entry:
   %obj0 = alloca [10 x i32], align 4, addrspace(5)
   %obj1 = alloca [4096 x i8], align 2, addrspace(5)
   %bc = bitcast [10 x i32] addrspace(5)* %obj0 to i32 addrspace(5)*
   store volatile i32 123, i32 addrspace(5)* %bc
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds [4096 x i8], [4096 x i8] addrspace(5)* %obj1, i32 0, i32 4051
   %trunc = trunc i16 %hi to i8
   store i8 %trunc, i8 addrspace(5)* %gep
   ret void
 }
 
 attributes #0 = { nounwind }

llvm/trunk/test/MC/AMDGPU/gfx9_asm_all.s

 // RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -mattr=+d16-preserves-unused-bits -show-encoding %s | FileCheck %s
 
 ds_add_u32 v1, v2 offset:65535
 // CHECK: [0xff,0xff,0x00,0xd8,0x01,0x02,0x00,0x00]
 
 ds_add_u32 v255, v2 offset:65535
 // CHECK: [0xff,0xff,0x00,0xd8,0xff,0x02,0x00,0x00]
 
 ds_add_u32 v1, v255 offset:65535