AMDGPU: Fix not converting d16 load/stores to offset

[arsenm]

Summary

Fixes missed optimization with new MUBUF instructions.

[rampitec]

LGTM

[arsenm]

r318106

lib/Target/AMDGPU/SIRegisterInfo.cpp

static int getOffsetMUBUFStore(unsigned Opc) {
   case AMDGPU::BUFFER_STORE_BYTE_OFFEN:
     return AMDGPU::BUFFER_STORE_BYTE_OFFSET;
   case AMDGPU::BUFFER_STORE_SHORT_OFFEN:
     return AMDGPU::BUFFER_STORE_SHORT_OFFSET;
   case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN:
     return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET;
   case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN:
     return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET;
+  case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN:
+    return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET;
+  case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN:
+    return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET;
   default:
     return -1;
   }
 }
 
 static int getOffsetMUBUFLoad(unsigned Opc) {
   switch (Opc) {
   case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
     return AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
   case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN:
     return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET;
   case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN:
     return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET;
   case AMDGPU::BUFFER_LOAD_USHORT_OFFEN:
     return AMDGPU::BUFFER_LOAD_USHORT_OFFSET;
   case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN:
     return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET;
   case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN:
     return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET;
   case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN:
     return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET;
+  case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN:
+    return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET;
+  case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN:
+    return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET;
+  case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN:
+    return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET;
+  case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN:
+    return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET;
+  case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN:
+    return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET;
+  case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN:
+    return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET;
   default:
     return -1;
   }
 }
 
 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not
 // need to handle the case where an SGPR may need to be spilled while spilling.
 static bool buildMUBUFOffsetLoadStore(const SIInstrInfo *TII,
                                       MachineFrameInfo &MFI,
                                       MachineBasicBlock::iterator MI,
                                       int Index,
                                       int64_t Offset) {
   MachineBasicBlock *MBB = MI->getParent();
   const DebugLoc &DL = MI->getDebugLoc();
   bool IsStore = MI->mayStore();
 
   unsigned Opc = MI->getOpcode();
   int LoadStoreOp = IsStore ?
     getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc);
   if (LoadStoreOp == -1)
     return false;
 
   const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata);
-  BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp))
+  MachineInstrBuilder NewMI = BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp))
     .add(*Reg)
     .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc))
     .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset))
     .addImm(Offset)
     .addImm(0) // glc
     .addImm(0) // slc
     .addImm(0) // tfe
     .setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
+
+  const MachineOperand *VDataIn = TII->getNamedOperand(*MI,
+                                                       AMDGPU::OpName::vdata_in);
+  if (VDataIn)
+    NewMI.add(*VDataIn);
   return true;
 }
 
 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
                                          unsigned LoadStoreOp,
                                          int Index,
                                          unsigned ValueReg,
                                          bool IsKill,

test/CodeGen/AMDGPU/load-hi16.ll

-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
 
 ; GCN-LABEL: {{^}}load_local_hi_v2i16_undeflo:
 ; GCN: s_waitcnt
 ; GFX9-NEXT: ds_read_u16_d16_hi v0, v0
 ; GFX9-NEXT: s_waitcnt
 ; GFX9-NEXT: s_setpc_b64
 
 ; VI: ds_read_u16 v

test/CodeGen/AMDGPU/load-lo16.ll

-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
 
 ; GCN-LABEL: {{^}}load_local_lo_v2i16_undeflo:
 ; GCN: s_waitcnt
 ; GFX9-NEXT: ds_read_u16_d16 v0, v0
 ; GFX9-NEXT: s_waitcnt
 ; GFX9-NEXT: s_setpc_b64
 
 ; VI: ds_read_u16
entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %gep = getelementptr inbounds half, half addrspace(2)* %in, i64 -2047
   %load = load half, half addrspace(2)* %gep
   %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
   store <2 x half> %build1, <2 x half> addrspace(1)* undef
   ret void
 }
 
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_short_d16 v0, off, s[0:3], s5 offset:4094
+
+; VI: buffer_load_ushort v
+define void @load_private_lo_v2i16_reglo_vreg_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i16], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i16], [4096 x i16]* %obj1, i32 0, i32 2025
+  %load = load volatile i16, i16* %gep
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_sbyte_d16 v0, off, s[0:3], s5 offset:4095
+
+; VI: buffer_load_sbyte v
+define void @load_private_lo_v2i16_reglo_vreg_sexti8_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load volatile i8, i8* %gep
+  %load.ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load.ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_ubyte_d16 v0, off, s[0:3], s5 offset:4095
+
+; VI: buffer_load_ubyte v
+define void @load_private_lo_v2i16_reglo_vreg_zexti8_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load volatile i8, i8* %gep
+  %load.ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load.ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
 attributes #0 = { nounwind }

test/CodeGen/AMDGPU/store-hi16.ll

-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
 
 ; GCN-LABEL: {{^}}store_global_hi_v2i16:
 ; GCN: s_waitcnt
 
 ; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
 
 ; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
 ; VI-NEXT: flat_store_short v[0:1], v2
entry:
   ; FIXME: ABI for pre-gfx9
   %value = bitcast i32 %arg to <2 x i16>
   %hi = extractelement <2 x i16> %value, i32 1
   %gep = getelementptr inbounds i16, i16 addrspace(3)* %out, i64 32767
   store i16 %hi, i16 addrspace(3)* %gep
   ret void
 }
 
+; GCN-LABEL: {{^}}store_private_hi_v2i16_to_offset:
+; GCN: s_waitcnt
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094
+define void @store_private_hi_v2i16_to_offset(i32 %arg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i16], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds [4096 x i16], [4096 x i16]* %obj1, i32 0, i32 2025
+  store i16 %hi, i16* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_to_offset:
+; GCN: s_waitcnt
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095
+define void @store_private_hi_v2i16_i8_to_offset(i32 %arg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8* %gep
+  ret void
+}
+
 attributes #0 = { nounwind }