AMDGPU: Add most d16 load/store instruction definitions

[arsenm]

Summary

Doesn't include the tied operand necessary for the loads,
but is enough for the assembler to work.

[rampitec]

lib/Target/AMDGPU/BUFInstructions.td

1593

Do you mind to align this block as around?

lib/Target/AMDGPU/DSInstructions.td

467

And also align here and below.

[arsenm]

Format

[rampitec]

lib/Target/AMDGPU/DSInstructions.td

468

Here as well...

lib/Target/AMDGPU/FLATInstructions.td

917

Here as well...

923

Here as well...

978

Here as well...

1022

Here as well...

[arsenm]

Formatting

[rampitec]

lib/Target/AMDGPU/FLATInstructions.td

1023

The last place ;)

[arsenm]

Format

lib/Target/AMDGPU/FLATInstructions.td

1023

The problem with these is the new instructions are longer, so it requires reformatting everything else around it (which is why I didn't want to have any of these aligned in the first place)

[rampitec]

LGTM

[arsenm]

r312347

lib/Target/AMDGPU/AMDGPU.td

 
 def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">,
   AssemblerPredicate<"FeatureFlatAddressSpace">;
 
 def HasFlatGlobalInsts : Predicate<"Subtarget->hasFlatGlobalInsts()">,
   AssemblerPredicate<"FeatureFlatGlobalInsts">;
 def HasFlatScratchInsts : Predicate<"Subtarget->hasFlatScratchInsts()">,
   AssemblerPredicate<"FeatureFlatScratchInsts">;
+def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">,
+  AssemblerPredicate<"FeatureGFX9Insts">;
 
 def HasAddNoCarryInsts : Predicate<"Subtarget->hasAddNoCarryInsts()">,
   AssemblerPredicate<"FeatureAddNoCarryInsts">;
 
 def NotHasAddNoCarryInsts : Predicate<"!Subtarget->hasAddNoCarryInsts()">,
   AssemblerPredicate<"!FeatureAddNoCarryInsts">;
 
 def Has16BitInsts : Predicate<"Subtarget->has16BitInsts()">,

lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   bool hasFlatGlobalInsts() const {
     return FlatGlobalInsts;
   }
 
   bool hasFlatScratchInsts() const {
     return FlatScratchInsts;
   }
 
+  bool hasD16LoadStore() const {
+    return getGeneration() >= GFX9;
+  }
+
   bool hasAddNoCarry() const {
     return AddNoCarryInsts;
   }
 
   bool isMesaKernel(const MachineFunction &MF) const {
     return isMesa3DOS() && !AMDGPU::isShader(MF.getFunction()->getCallingConv());
   }
 

lib/Target/AMDGPU/BUFInstructions.td

 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 */
 
 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
                                           int_amdgcn_buffer_wbinvl1_sc>;
 }
 
+let SubtargetPredicate = HasD16LoadStore in {
+
+defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
+  "buffer_load_ubyte_d16", VGPR_32, i32
+>;
+
+defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
+  "buffer_load_ubyte_d16_hi", VGPR_32, i32
+>;
+
+defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
+  "buffer_load_sbyte_d16", VGPR_32, i32
+>;
+
+defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
+  "buffer_load_sbyte_d16_hi", VGPR_32, i32
+>;
+
+defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
+  "buffer_load_short_d16", VGPR_32, i32
+>;
+
+defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
+  "buffer_load_short_d16_hi", VGPR_32, i32
+>;
+
+defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
+  "buffer_store_byte_d16_hi", VGPR_32, i32
+>;
+
+defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
+  "buffer_store_short_d16_hi", VGPR_32, i32
+>;
+
+} // End HasD16LoadStore
+
 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
                                        int_amdgcn_buffer_wbinvl1>;
 
 //===----------------------------------------------------------------------===//
 // MTBUF Instructions
 //===----------------------------------------------------------------------===//
 
 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;

 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_vi <0x11>;
 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_vi <0x12>;
 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_vi <0x13>;
 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_vi <0x14>;
 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
+defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
+defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
 
+defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
+defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
+defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
+defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
+defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
+defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
+
 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;

lib/Target/AMDGPU/DSInstructions.td

 def DS_MAX_F32        : DS_1A1D_NORET<"ds_max_f32">;
 
 let mayLoad = 0 in {
 def DS_WRITE_B8       : DS_1A1D_NORET<"ds_write_b8">;
 def DS_WRITE_B16      : DS_1A1D_NORET<"ds_write_b16">;
 def DS_WRITE_B32      : DS_1A1D_NORET<"ds_write_b32">;
 def DS_WRITE2_B32     : DS_1A2D_Off8_NORET<"ds_write2_b32">;
 def DS_WRITE2ST64_B32 : DS_1A2D_Off8_NORET<"ds_write2st64_b32">;
+
+let SubtargetPredicate = HasD16LoadStore in {
+def DS_WRITE_B8_D16_HI  : DS_1A1D_NORET<"ds_write_b8_d16_hi">;
+def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
+}
+
 }
 
 def DS_MSKOR_B32      : DS_1A2D_NORET<"ds_mskor_b32">;
 def DS_CMPST_B32      : DS_1A2D_NORET<"ds_cmpst_b32">;
 def DS_CMPST_F32      : DS_1A2D_NORET<"ds_cmpst_f32">;
 
 def DS_ADD_U64        : DS_1A1D_NORET<"ds_add_u64", VReg_64>;
 def DS_SUB_U64        : DS_1A1D_NORET<"ds_sub_u64", VReg_64>;

 def DS_READ_B32      : DS_1A_RET<"ds_read_b32">;
 def DS_READ_B64      : DS_1A_RET<"ds_read_b64", VReg_64>;
 
 def DS_READ2_B32     : DS_1A_Off8_RET<"ds_read2_b32", VReg_64>;
 def DS_READ2ST64_B32 : DS_1A_Off8_RET<"ds_read2st64_b32", VReg_64>;
 
 def DS_READ2_B64     : DS_1A_Off8_RET<"ds_read2_b64", VReg_128>;
 def DS_READ2ST64_B64 : DS_1A_Off8_RET<"ds_read2st64_b64", VReg_128>;
+
+let SubtargetPredicate = HasD16LoadStore in {
+def DS_READ_U8_D16     : DS_1A_RET<"ds_read_u8_d16">;
+def DS_READ_U8_D16_HI  : DS_1A_RET<"ds_read_u8_d16_hi">;
+def DS_READ_I8_D16     : DS_1A_RET<"ds_read_i8_d16">;
+def DS_READ_I8_D16_HI  : DS_1A_RET<"ds_read_i8_d16_hi">;
+def DS_READ_U16_D16    : DS_1A_RET<"ds_read_u16_d16">;
+def DS_READ_U16_D16_HI : DS_1A_RET<"ds_read_u16_d16_hi">;
+}
 }
 
 def DS_CONSUME       : DS_0A_RET<"ds_consume">;
 def DS_APPEND        : DS_0A_RET<"ds_append">;
 def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;
 
 //===----------------------------------------------------------------------===//
 // Instruction definitions for CI and newer.

 def DS_WRITE_B64_vi       : DS_Real_vi<0x4d, DS_WRITE_B64>;
 def DS_WRITE2_B64_vi      : DS_Real_vi<0x4E, DS_WRITE2_B64>;
 def DS_WRITE2ST64_B64_vi  : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;
 def DS_CMPST_B64_vi       : DS_Real_vi<0x50, DS_CMPST_B64>;
 def DS_CMPST_F64_vi       : DS_Real_vi<0x51, DS_CMPST_F64>;
 def DS_MIN_F64_vi         : DS_Real_vi<0x52, DS_MIN_F64>;
 def DS_MAX_F64_vi         : DS_Real_vi<0x53, DS_MAX_F64>;
 
+def DS_WRITE_B8_D16_HI_vi  : DS_Real_vi<0x54, DS_WRITE_B8_D16_HI>;
+def DS_WRITE_B16_D16_HI_vi : DS_Real_vi<0x55, DS_WRITE_B16_D16_HI>;
+
+def DS_READ_U8_D16_vi     : DS_Real_vi<0x56, DS_READ_U8_D16>;
+def DS_READ_U8_D16_HI_vi  : DS_Real_vi<0x57, DS_READ_U8_D16_HI>;
+def DS_READ_I8_D16_vi     : DS_Real_vi<0x58, DS_READ_I8_D16>;
+def DS_READ_I8_D16_HI_vi  : DS_Real_vi<0x59, DS_READ_I8_D16_HI>;
+def DS_READ_U16_D16_vi    : DS_Real_vi<0x5a, DS_READ_U16_D16>;
+def DS_READ_U16_D16_HI_vi : DS_Real_vi<0x5b, DS_READ_U16_D16_HI>;
+
 def DS_ADD_RTN_U64_vi     : DS_Real_vi<0x60, DS_ADD_RTN_U64>;
 def DS_SUB_RTN_U64_vi     : DS_Real_vi<0x61, DS_SUB_RTN_U64>;
 def DS_RSUB_RTN_U64_vi    : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;
 def DS_INC_RTN_U64_vi     : DS_Real_vi<0x63, DS_INC_RTN_U64>;
 def DS_DEC_RTN_U64_vi     : DS_Real_vi<0x64, DS_DEC_RTN_U64>;
 def DS_MIN_RTN_I64_vi     : DS_Real_vi<0x65, DS_MIN_RTN_I64>;
 def DS_MAX_RTN_I64_vi     : DS_Real_vi<0x66, DS_MAX_RTN_I64>;
 def DS_MIN_RTN_U64_vi     : DS_Real_vi<0x67, DS_MIN_RTN_U64>;

lib/Target/AMDGPU/FLATInstructions.td

 
 def FLAT_STORE_BYTE    : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>;
 def FLAT_STORE_SHORT   : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>;
 def FLAT_STORE_DWORD   : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>;
 def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>;
 def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>;
 def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>;
 
+let SubtargetPredicate = HasD16LoadStore in {
+def FLAT_LOAD_UBYTE_D16     : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32>;
+def FLAT_LOAD_UBYTE_D16_HI  : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32>;
+def FLAT_LOAD_SBYTE_D16     : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32>;
+def FLAT_LOAD_SBYTE_D16_HI  : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32>;
+def FLAT_LOAD_SHORT_D16     : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32>;
+def FLAT_LOAD_SHORT_D16_HI  : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32>;
+
+def FLAT_STORE_BYTE_D16_HI  : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>;
+def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>;
+}
+
 defm FLAT_ATOMIC_CMPSWAP    : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap",
                                 VGPR_32, i32, atomic_cmp_swap_flat,
                                 v2i32, VReg_64>;
 
 defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2",
                                 VReg_64, i64, atomic_cmp_swap_flat,
                                 v2i64, VReg_128>;
 

 defm GLOBAL_LOAD_SBYTE    : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>;
 defm GLOBAL_LOAD_USHORT   : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>;
 defm GLOBAL_LOAD_SSHORT   : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>;
 defm GLOBAL_LOAD_DWORD    : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>;
 defm GLOBAL_LOAD_DWORDX2  : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>;
 defm GLOBAL_LOAD_DWORDX3  : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>;
 defm GLOBAL_LOAD_DWORDX4  : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>;
 
+defm GLOBAL_LOAD_UBYTE_D16    : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32>;
+defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32>;
+defm GLOBAL_LOAD_SBYTE_D16    : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32>;
+defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32>;
+defm GLOBAL_LOAD_SHORT_D16    : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32>;
+defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32>;
+
 defm GLOBAL_STORE_BYTE    : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>;
 defm GLOBAL_STORE_SHORT   : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>;
 defm GLOBAL_STORE_DWORD   : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>;
 defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>;
 defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>;
 defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>;
 
+defm GLOBAL_STORE_BYTE_D16_HI  : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>;
+defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>;
 
 let is_flat_global = 1 in {
 defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap",
                                VGPR_32, i32, AMDGPUatomic_cmp_swap_global,
                                v2i32, VReg_64>;
 
 defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2",
                                   VReg_64, i64, AMDGPUatomic_cmp_swap_global,

 defm SCRATCH_LOAD_SBYTE    : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>;
 defm SCRATCH_LOAD_USHORT   : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>;
 defm SCRATCH_LOAD_SSHORT   : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>;
 defm SCRATCH_LOAD_DWORD    : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>;
 defm SCRATCH_LOAD_DWORDX2  : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>;
 defm SCRATCH_LOAD_DWORDX3  : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>;
 defm SCRATCH_LOAD_DWORDX4  : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>;
 
+defm SCRATCH_LOAD_UBYTE_D16    : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32>;
+defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>;
+defm SCRATCH_LOAD_SBYTE_D16    : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32>;
+defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>;
+defm SCRATCH_LOAD_SHORT_D16    : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32>;
+defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>;
+
 defm SCRATCH_STORE_BYTE    : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>;
 defm SCRATCH_STORE_SHORT   : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>;
 defm SCRATCH_STORE_DWORD   : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>;
 defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>;
 defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>;
 defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>;
 
+defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>;
+defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>;
+
 } // End SubtargetPredicate = HasFlatScratchInsts
 
 //===----------------------------------------------------------------------===//
 // Flat Patterns
 //===----------------------------------------------------------------------===//
 
 class flat_ld <SDPatternOperator ld> : PatFrag<(ops node:$ptr),
                                                (ld node:$ptr), [{

 def FLAT_LOAD_USHORT_vi        : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>;
 def FLAT_LOAD_SSHORT_vi        : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>;
 def FLAT_LOAD_DWORD_vi         : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>;
 def FLAT_LOAD_DWORDX2_vi       : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>;
 def FLAT_LOAD_DWORDX4_vi       : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>;
 def FLAT_LOAD_DWORDX3_vi       : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>;
 
 def FLAT_STORE_BYTE_vi         : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>;
+def FLAT_STORE_BYTE_D16_HI_vi  : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>;
 def FLAT_STORE_SHORT_vi        : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>;
+def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>;
 def FLAT_STORE_DWORD_vi        : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>;
 def FLAT_STORE_DWORDX2_vi      : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>;
 def FLAT_STORE_DWORDX4_vi      : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>;
 def FLAT_STORE_DWORDX3_vi      : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>;
 
+def FLAT_LOAD_UBYTE_D16_vi    : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>;
+def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>;
+def FLAT_LOAD_SBYTE_D16_vi    : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>;
+def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>;
+def FLAT_LOAD_SHORT_D16_vi    : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>;
+def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>;
+
 multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> {
   def _vi     : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>;
   def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>;
 }
 
 multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> :
   FLAT_Real_AllAddr_vi<op> {
   def _RTN_vi  : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>;

 defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>;
 defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>;
 defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>;
 defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>;
 defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>;
 defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>;
 defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>;
 
+defm GLOBAL_LOAD_UBYTE_D16    : FLAT_Real_AllAddr_vi <0x20>;
+defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>;
+defm GLOBAL_LOAD_SBYTE_D16    : FLAT_Real_AllAddr_vi <0x22>;
+defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>;
+defm GLOBAL_LOAD_SHORT_D16    : FLAT_Real_AllAddr_vi <0x24>;
+defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>;
+
 defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>;
+defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>;
 defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>;
+defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;
 defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>;
 defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>;
 defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>;
 defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>;
 
 
 defm GLOBAL_ATOMIC_SWAP       : FLAT_Global_Real_Atomics_vi <0x40>;
 defm GLOBAL_ATOMIC_CMPSWAP    : FLAT_Global_Real_Atomics_vi <0x41>;

 defm GLOBAL_ATOMIC_SMAX_X2    : FLAT_Global_Real_Atomics_vi <0x66>;
 defm GLOBAL_ATOMIC_UMAX_X2    : FLAT_Global_Real_Atomics_vi <0x67>;
 defm GLOBAL_ATOMIC_AND_X2     : FLAT_Global_Real_Atomics_vi <0x68>;
 defm GLOBAL_ATOMIC_OR_X2      : FLAT_Global_Real_Atomics_vi <0x69>;
 defm GLOBAL_ATOMIC_XOR_X2     : FLAT_Global_Real_Atomics_vi <0x6a>;
 defm GLOBAL_ATOMIC_INC_X2     : FLAT_Global_Real_Atomics_vi <0x6b>;
 defm GLOBAL_ATOMIC_DEC_X2     : FLAT_Global_Real_Atomics_vi <0x6c>;
 
-defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>;
-defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>;
-defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>;
-defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>;
-defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>;
-defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>;
-defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>;
-defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>;
-
-defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>;
-defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>;
-defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>;
-defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>;
-defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>;
-defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>;
+defm SCRATCH_LOAD_UBYTE         : FLAT_Real_AllAddr_vi <0x10>;
+defm SCRATCH_LOAD_SBYTE         : FLAT_Real_AllAddr_vi <0x11>;
+defm SCRATCH_LOAD_USHORT        : FLAT_Real_AllAddr_vi <0x12>;
+defm SCRATCH_LOAD_SSHORT        : FLAT_Real_AllAddr_vi <0x13>;
+defm SCRATCH_LOAD_DWORD         : FLAT_Real_AllAddr_vi <0x14>;
+defm SCRATCH_LOAD_DWORDX2       : FLAT_Real_AllAddr_vi <0x15>;
+defm SCRATCH_LOAD_DWORDX3       : FLAT_Real_AllAddr_vi <0x16>;
+defm SCRATCH_LOAD_DWORDX4       : FLAT_Real_AllAddr_vi <0x17>;
+defm SCRATCH_STORE_BYTE         : FLAT_Real_AllAddr_vi <0x18>;
+defm SCRATCH_STORE_BYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x19>;
+defm SCRATCH_LOAD_UBYTE_D16     : FLAT_Real_AllAddr_vi <0x20>;
+defm SCRATCH_LOAD_UBYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x21>;
+defm SCRATCH_LOAD_SBYTE_D16     : FLAT_Real_AllAddr_vi <0x22>;
+defm SCRATCH_LOAD_SBYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x23>;
+defm SCRATCH_LOAD_SHORT_D16     : FLAT_Real_AllAddr_vi <0x24>;
+defm SCRATCH_LOAD_SHORT_D16_HI  : FLAT_Real_AllAddr_vi <0x25>;
+defm SCRATCH_STORE_SHORT        : FLAT_Real_AllAddr_vi <0x1a>;
+defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;
+defm SCRATCH_STORE_DWORD        : FLAT_Real_AllAddr_vi <0x1c>;
+defm SCRATCH_STORE_DWORDX2      : FLAT_Real_AllAddr_vi <0x1d>;
+defm SCRATCH_STORE_DWORDX3      : FLAT_Real_AllAddr_vi <0x1e>;
+defm SCRATCH_STORE_DWORDX4      : FLAT_Real_AllAddr_vi <0x1f>;

test/MC/AMDGPU/ds-gfx9.s

+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding 2>&1 %s | FileCheck -check-prefix=VI-ERR -check-prefix=GCNERR %s
+
+ds_read_u8_d16 v8, v2
+// GFX9: ds_read_u8_d16 v8, v2           ; encoding: [0x00,0x00,0xac,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_read_u8_d16_hi v8, v2
+// GFX9: ds_read_u8_d16_hi v8, v2        ; encoding: [0x00,0x00,0xae,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_read_i8_d16 v8, v2
+// GFX9: ds_read_i8_d16 v8, v2           ; encoding: [0x00,0x00,0xb0,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_read_i8_d16_hi v8, v2
+// GFX9: ds_read_i8_d16_hi v8, v2        ; encoding: [0x00,0x00,0xb2,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_read_u16_d16 v8, v2
+// GFX9: ds_read_u16_d16 v8, v2          ; encoding: [0x00,0x00,0xb4,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_read_u16_d16_hi v8, v2
+// GFX9: ds_read_u16_d16_hi v8, v2       ; encoding: [0x00,0x00,0xb6,0xd8,0x02,0x00,0x00,0x08]
+// VI-ERR: error: instruction not supported on this GPU
+
+ds_write_b8_d16_hi v8, v2
+// VI-ERR: error: instruction not supported on this GPU
+// GFX9: ds_write_b8_d16_hi v8, v2       ; encoding: [0x00,0x00,0xa8,0xd8,0x08,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v8, v2
+// VI-ERR: error: instruction not supported on this GPU
+// GFX9: ds_write_b16_d16_hi v8, v2      ; encoding: [0x00,0x00,0xaa,0xd8,0x08,0x02,0x00,0x00]

test/MC/AMDGPU/flat-gfx9.s

 flat_load_dword v1, v[3:4], s0
 // GCNERR: :29: error: invalid operand for instruction
 
 flat_load_dword v1, v[3:4], exec_hi
 // GCNERR: :29: error: invalid operand for instruction
 
 flat_store_dword v[3:4], v1, exec_hi
 // GCNERR: :30: error: invalid operand for instruction
+
+flat_load_ubyte_d16 v1, v[3:4]
+// GFX9: flat_load_ubyte_d16 v1, v[3:4]  ; encoding: [0x00,0x00,0x80,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_load_ubyte_d16_hi v1, v[3:4]
+// GFX9: flat_load_ubyte_d16_hi v1, v[3:4] ; encoding: [0x00,0x00,0x84,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_load_sbyte_d16 v1, v[3:4]
+// GFX9: flat_load_sbyte_d16 v1, v[3:4]  ; encoding: [0x00,0x00,0x88,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_load_sbyte_d16_hi v1, v[3:4]
+// GFX9: flat_load_sbyte_d16_hi v1, v[3:4] ; encoding: [0x00,0x00,0x8c,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_load_short_d16 v1, v[3:4]
+// GFX9: flat_load_short_d16 v1, v[3:4]  ; encoding: [0x00,0x00,0x90,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_load_short_d16_hi v1, v[3:4]
+// GFX9: flat_load_short_d16_hi v1, v[3:4] ; encoding: [0x00,0x00,0x94,0xdc,0x03,0x00,0x00,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_store_byte_d16_hi v[3:4], v1
+// GFX9: flat_store_byte_d16_hi v[3:4], v1 ; encoding: [0x00,0x00,0x64,0xdc,0x03,0x01,0x00,0x00]
+// VI-ERR: error: instruction not supported on this GPU
+
+flat_store_short_d16_hi v[3:4], v1
+// GFX9: flat_store_short_d16_hi v[3:4], v1 ; encoding: [0x00,0x00,0x6c,0xdc,0x03,0x01,0x00,0x00
+// VI-ERR: error: instruction not supported on this GPU

test/MC/AMDGPU/flat-global.s

 
 global_atomic_inc_x2 v[3:4], v[5:6], off offset:-16
 // GFX9: global_atomic_inc_x2 v[3:4], v[5:6], off offset:-16 ; encoding: [0xf0,0x9f,0xac,0xdd,0x03,0x05,0x7f,0x00]
 // VI-ERR: :48: error: not a valid operand
 
 global_atomic_dec_x2 v[3:4], v[5:6], off offset:-16
 // GFX9: global_atomic_dec_x2 v[3:4], v[5:6], off offset:-16 ; encoding: [0xf0,0x9f,0xb0,0xdd,0x03,0x05,0x7f,0x00]
 // VI-ERR: :48: error: not a valid operand
+
+global_load_ubyte_d16 v1, v[3:4], off
+// GFX9: global_load_ubyte_d16 v1, v[3:4], off ; encoding: [0x00,0x80,0x80,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_load_ubyte_d16_hi v1, v[3:4], off
+// GFX9: global_load_ubyte_d16_hi v1, v[3:4], off ; encoding: [0x00,0x80,0x84,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_load_sbyte_d16 v1, v[3:4], off
+// GFX9: global_load_sbyte_d16 v1, v[3:4], off ; encoding: [0x00,0x80,0x88,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_load_sbyte_d16_hi v1, v[3:4], off
+// GFX9: global_load_sbyte_d16_hi v1, v[3:4], off ; encoding: [0x00,0x80,0x8c,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_load_short_d16 v1, v[3:4], off
+// GFX9: global_load_short_d16 v1, v[3:4], off ; encoding: [0x00,0x80,0x90,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_load_short_d16_hi v1, v[3:4], off
+// GFX9: global_load_short_d16_hi v1, v[3:4], off ; encoding: [0x00,0x80,0x94,0xdc,0x03,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+global_store_byte_d16_hi v[3:4], v1, off
+// GFX9: global_store_byte_d16_hi v[3:4], v1, off ; encoding: [0x00,0x80,0x64,0xdc,0x03,0x01,0x7f,0x00]
+// VI-ERR: instruction not supported on this GPU
+
+global_store_short_d16_hi v[3:4], v1, off
+// GFX9: global_store_short_d16_hi v[3:4], v1, off ; encoding: [0x00,0x80,0x6c,0xdc,0x03,0x01,0x7f,0x00]
+// VI-ERR: instruction not supported on this GPU

test/MC/AMDGPU/flat-scratch-instructions.s

 
 scratch_load_dword v1, off, m0
 // GFX9: scratch_load_dword v1, off, m0  ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x7c,0x01]
 // VI-ERR: instruction not supported on this GPU
 
 scratch_store_dword off, v2, m0
 // GFX9: scratch_store_dword off, v2, m0 ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x7c,0x00]
 // VI-ERR: instruction not supported on this GPU
+
+scratch_load_ubyte_d16 v1, v2, off
+// GFX9: scratch_load_ubyte_d16 v1, v2, off ; encoding: [0x00,0x40,0x80,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_load_ubyte_d16_hi v1, v2, off
+// GFX9: scratch_load_ubyte_d16_hi v1, v2, off ; encoding: [0x00,0x40,0x84,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_load_sbyte_d16 v1, v2, off
+// GFX9: scratch_load_sbyte_d16 v1, v2, off ; encoding: [0x00,0x40,0x88,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_load_sbyte_d16_hi v1, v2, off
+// GFX9: scratch_load_sbyte_d16_hi v1, v2, off ; encoding: [0x00,0x40,0x8c,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_load_short_d16 v1, v2, off
+// GFX9: scratch_load_short_d16 v1, v2, off ; encoding: [0x00,0x40,0x90,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_load_short_d16_hi v1, v2, off
+// GFX9: scratch_load_short_d16_hi v1, v2, off ; encoding: [0x00,0x40,0x94,0xdc,0x02,0x00,0x7f,0x01]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_store_byte_d16_hi off, v2, s1
+// GFX9: scratch_store_byte_d16_hi off, v2, s1 ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x01,0x00]
+// VI-ERR: instruction not supported on this GPU
+
+scratch_store_short_d16_hi off, v2, s1
+// GFX9: scratch_store_short_d16_hi off, v2, s1 ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x01,0x00]
+// VI-ERR: instruction not supported on this GPU

test/MC/AMDGPU/mubuf-gfx9.s

+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding 2>&1 %s | FileCheck -check-prefix=VI-ERR -check-prefix=GCNERR %s
+
+buffer_load_ubyte_d16 v1, off, s[4:7], s1
+// VI-ERR: error: instruction not supported on this GPU
+// GFX9: buffer_load_ubyte_d16 v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x80,0xe0,0x00,0x01,0x01,0x01]
+
+buffer_load_ubyte_d16_hi v1, off, s[4:7], s1
+// GFX9: buffer_load_ubyte_d16_hi v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x84,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_load_sbyte_d16 v1, off, s[4:7], s1
+// GFX9: buffer_load_sbyte_d16 v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x88,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_load_sbyte_d16_hi v1, off, s[4:7], s1
+// GFX9: buffer_load_sbyte_d16_hi v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x8c,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_load_short_d16 v1, off, s[4:7], s1
+// GFX9: buffer_load_short_d16 v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x90,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_load_short_d16_hi v1, off, s[4:7], s1
+// GFX9: buffer_load_short_d16_hi v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x94,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_store_byte_d16_hi v1, off, s[4:7], s1
+// GFX9: buffer_store_byte_d16_hi v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU
+
+buffer_store_short_d16_hi v1, off, s[4:7], s1
+// GFX9: buffer_store_short_d16_hi v1, off, s[4:7], s1 ; encoding: [0x00,0x00,0x6c,0xe0,0x00,0x01,0x01,0x01]
+// VI-ERR: error: instruction not supported on this GPU