[AMDGPU]Increased vector length for global/constant loads.

[FarhanaAleen]

Summary

GCN ISA supports instructions that can read 16 consecutive dwords from memory through the scalar data cache; loadstoreVectorizer should take advantage of the wider vector length and pack 16/8 elements of dwords/quadwords.

[t-tye]

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

262 ↗

(On Diff #134126)

Does amdgpu only support gfx6 (si) and above? I thought northern islands was supported by the r600 backend.

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

121 ↗

(On Diff #134126)

I did not see where in this patch these new functions are being used.

[FarhanaAleen]

Does amdgpu only support gfx6 (si) and above? I thought northern islands was supported by the r600 backend.

AMDGPU contains the general target information for all gpus and specific target implementation is supported by the specific backend; such as r600 and SI backends.

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

121 ↗

(On Diff #134126)

These are not new functions; the default definitions can be found in TargetTransformInfoImpl.h. loadStoreVectorizer was using the default implementation of these functions for AMDGPU since there were no implementation of these functions for AMDGPU.
I have copy-pasted the default definitions here for convenience.

unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,

                             unsigned ChainSizeInBytes,
                             VectorType *VecTy) const {
  return VF;
}

unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                              unsigned ChainSizeInBytes,
                              VectorType *VecTy) const {
  return VF;
}

[rampitec]

Please fix the tests, otherwise looks good.

test/CodeGen/AMDGPU/load-constant-f32.ll

12 ↗

(On Diff #134126)

Run tests through opt -instnamer. There should be no numeric values.

[FarhanaAleen]

Renamed the instructions to get rid of the numeric values.

[rampitec]

LGTM

llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
     assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
     return TTI::PSK_FastHardware;
   }
 
   unsigned getHardwareNumberOfRegisters(bool Vector) const;
   unsigned getNumberOfRegisters(bool Vector) const;
   unsigned getRegisterBitWidth(bool Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
+  unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
+                               unsigned ChainSizeInBytes,
+                               VectorType *VecTy) const;
+  unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
+                                unsigned ChainSizeInBytes,
+                                VectorType *VecTy) const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
 
   bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
                                   unsigned Alignment,
                                   unsigned AddrSpace) const;
   bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
                                    unsigned Alignment,
                                    unsigned AddrSpace) const;

llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

 unsigned AMDGPUTTIImpl::getRegisterBitWidth(bool Vector) const {
   return 32;
 }
 
 unsigned AMDGPUTTIImpl::getMinVectorRegisterBitWidth() const {
   return 32;
 }
 
+unsigned AMDGPUTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
+                                            unsigned ChainSizeInBytes,
+                                            VectorType *VecTy) const {
+  unsigned VecRegBitWidth = VF * LoadSize;
+  if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
+    // TODO: Support element-size less than 32bit?
+    return 128 / LoadSize;
+
+  return VF;
+}
+
+unsigned AMDGPUTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
+                                             unsigned ChainSizeInBytes,
+                                             VectorType *VecTy) const {
+  unsigned VecRegBitWidth = VF * StoreSize;
+  if (VecRegBitWidth > 128)
+    return 128 / StoreSize;
+
+  return VF;
+}
+
 unsigned AMDGPUTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
   AMDGPUAS AS = ST->getAMDGPUAS();
   if (AddrSpace == AS.GLOBAL_ADDRESS ||
       AddrSpace == AS.CONSTANT_ADDRESS ||
-      AddrSpace == AS.CONSTANT_ADDRESS_32BIT ||
-      AddrSpace == AS.FLAT_ADDRESS)
+      AddrSpace == AS.CONSTANT_ADDRESS_32BIT) {
+    if (ST->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS)
+      return 128;
+    return 512;
+  }
+
+  if (AddrSpace == AS.FLAT_ADDRESS)
     return 128;
   if (AddrSpace == AS.LOCAL_ADDRESS ||
       AddrSpace == AS.REGION_ADDRESS)
     return 64;
   if (AddrSpace == AS.PRIVATE_ADDRESS)
     return 8 * ST->getMaxPrivateElementSize();
 
   if (ST->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS &&

llvm/trunk/test/CodeGen/AMDGPU/load-constant-f32.ll

+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+
+; Tests whether a load chain of 8 constants gets vectorized into a wider load.
+; FUNC-LABEL: {{^}}constant_load_v8f32:
+; GCN: s_load_dwordx8
+; EG: VTX_READ_128
+; EG: VTX_READ_128
+define amdgpu_kernel void @constant_load_v8f32(float addrspace(4)* noalias nocapture readonly %weights, float addrspace(1)* noalias nocapture %out_ptr) {
+entry:
+  %out_ptr.promoted = load float, float addrspace(1)* %out_ptr, align 4
+  %tmp = load float, float addrspace(4)* %weights, align 4
+  %add = fadd float %tmp, %out_ptr.promoted
+  %arrayidx.1 = getelementptr inbounds float, float addrspace(4)* %weights, i64 1
+  %tmp1 = load float, float addrspace(4)* %arrayidx.1, align 4
+  %add.1 = fadd float %tmp1, %add
+  %arrayidx.2 = getelementptr inbounds float, float addrspace(4)* %weights, i64 2
+  %tmp2 = load float, float addrspace(4)* %arrayidx.2, align 4
+  %add.2 = fadd float %tmp2, %add.1
+  %arrayidx.3 = getelementptr inbounds float, float addrspace(4)* %weights, i64 3
+  %tmp3 = load float, float addrspace(4)* %arrayidx.3, align 4
+  %add.3 = fadd float %tmp3, %add.2
+  %arrayidx.4 = getelementptr inbounds float, float addrspace(4)* %weights, i64 4
+  %tmp4 = load float, float addrspace(4)* %arrayidx.4, align 4
+  %add.4 = fadd float %tmp4, %add.3
+  %arrayidx.5 = getelementptr inbounds float, float addrspace(4)* %weights, i64 5
+  %tmp5 = load float, float addrspace(4)* %arrayidx.5, align 4
+  %add.5 = fadd float %tmp5, %add.4
+  %arrayidx.6 = getelementptr inbounds float, float addrspace(4)* %weights, i64 6
+  %tmp6 = load float, float addrspace(4)* %arrayidx.6, align 4
+  %add.6 = fadd float %tmp6, %add.5
+  %arrayidx.7 = getelementptr inbounds float, float addrspace(4)* %weights, i64 7
+  %tmp7 = load float, float addrspace(4)* %arrayidx.7, align 4
+  %add.7 = fadd float %tmp7, %add.6
+  store float %add.7, float addrspace(1)* %out_ptr, align 4
+  ret void
+}

llvm/trunk/test/CodeGen/AMDGPU/load-constant-f64.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}constant_load_f64:
 ; GCN: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}]
 ; GCN-NOHSA: buffer_store_dwordx2
 ; GCN-HSA: flat_store_dwordx2
 define amdgpu_kernel void @constant_load_f64(double addrspace(1)* %out, double addrspace(4)* %in) #0 {
   %ld = load double, double addrspace(4)* %in
   store double %ld, double addrspace(1)* %out
   ret void
 }
 
 attributes #0 = { nounwind }
+
+; Tests whether a load-chain of 8 constants of 64bit each gets vectorized into a wider load.
+; FUNC-LABEL: {{^}}constant_load_2v4f64:
+; GCN: s_load_dwordx16
+define amdgpu_kernel void @constant_load_2v4f64(double addrspace(4)* noalias nocapture readonly %weights, double addrspace(1)* noalias nocapture %out_ptr) {
+entry:
+  %out_ptr.promoted = load double, double addrspace(1)* %out_ptr, align 4
+  %tmp = load double, double addrspace(4)* %weights, align 4
+  %add = fadd double %tmp, %out_ptr.promoted
+  %arrayidx.1 = getelementptr inbounds double, double addrspace(4)* %weights, i64 1
+  %tmp1 = load double, double addrspace(4)* %arrayidx.1, align 4
+  %add.1 = fadd double %tmp1, %add
+  %arrayidx.2 = getelementptr inbounds double, double addrspace(4)* %weights, i64 2
+  %tmp2 = load double, double addrspace(4)* %arrayidx.2, align 4
+  %add.2 = fadd double %tmp2, %add.1
+  %arrayidx.3 = getelementptr inbounds double, double addrspace(4)* %weights, i64 3
+  %tmp3 = load double, double addrspace(4)* %arrayidx.3, align 4
+  %add.3 = fadd double %tmp3, %add.2
+  %arrayidx.4 = getelementptr inbounds double, double addrspace(4)* %weights, i64 4
+  %tmp4 = load double, double addrspace(4)* %arrayidx.4, align 4
+  %add.4 = fadd double %tmp4, %add.3
+  %arrayidx.5 = getelementptr inbounds double, double addrspace(4)* %weights, i64 5
+  %tmp5 = load double, double addrspace(4)* %arrayidx.5, align 4
+  %add.5 = fadd double %tmp5, %add.4
+  %arrayidx.6 = getelementptr inbounds double, double addrspace(4)* %weights, i64 6
+  %tmp6 = load double, double addrspace(4)* %arrayidx.6, align 4
+  %add.6 = fadd double %tmp6, %add.5
+  %arrayidx.7 = getelementptr inbounds double, double addrspace(4)* %weights, i64 7
+  %tmp7 = load double, double addrspace(4)* %arrayidx.7, align 4
+  %add.7 = fadd double %tmp7, %add.6
+  store double %add.7, double addrspace(1)* %out_ptr, align 4
+  ret void
+}

llvm/trunk/test/CodeGen/AMDGPU/waitcnt-looptest.ll

-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global | FileCheck --check-prefix=GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global -amdgpu-load-store-vectorizer=0 | FileCheck --check-prefix=GCN %s
 
 ; Check that the waitcnt insertion algorithm correctly propagates wait counts
 ; from before a loop to the loop header.
 
 ; GCN-LABEL: {{^}}testKernel
 ; GCN: BB0_1:
 ; GCN: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN-NEXT: v_cmp_eq_f32_e64