AMDGPU: Make some packed shuffles free

[arsenm]

Summary

VOP3P instructions can encode access to either
half of the register.

[arsenm]

ping

[kzhuravl]

LGTM.

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

120

override keyword?

[kzhuravl]

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

120

nevermind.

[arsenm]

r302730

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

unsigned getFlatAddressSpace() const {
     // don't use flat addressing.
     if (IsGraphicsShader)
       return -1;
     return ST->hasFlatAddressSpace() ?
       ST->getAMDGPUAS().FLAT_ADDRESS : ST->getAMDGPUAS().UNKNOWN_ADDRESS_SPACE;
   }
 
   unsigned getVectorSplitCost() { return 0; }
+
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
+                          Type *SubTp);
 };
 
 } // end namespace llvm
 
 #endif

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

default:
     return BaseT::getCFInstrCost(Opcode);
   }
 }
 
 int AMDGPUTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                       unsigned Index) {
   switch (Opcode) {
   case Instruction::ExtractElement:
-  case Instruction::InsertElement:
+  case Instruction::InsertElement: {
+    unsigned EltSize
+      = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
+    if (EltSize < 32) {
+      if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
+        return 0;
+      return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
+    }
+
     // Extracts are just reads of a subregister, so are free. Inserts are
     // considered free because we don't want to have any cost for scalarizing
     // operations, and we don't have to copy into a different register class.
 
     // Dynamic indexing isn't free and is best avoided.
     return Index == ~0u ? 2 : 0;
+  }
   default:
     return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
   }
 }
 
 static bool isIntrinsicSourceOfDivergence(const IntrinsicInst *I) {
   switch (I->getIntrinsicID()) {
   case Intrinsic::amdgcn_workitem_id_x:
if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
     return isIntrinsicSourceOfDivergence(Intrinsic);
 
   // Assume all function calls are a source of divergence.
   if (isa<CallInst>(V) || isa<InvokeInst>(V))
     return true;
 
   return false;
 }
+
+unsigned AMDGPUTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
+                                       Type *SubTp) {
+  if (ST->hasVOP3PInsts()) {
+    VectorType *VT = cast<VectorType>(Tp);
+    if (VT->getNumElements() == 2 &&
+        DL.getTypeSizeInBits(VT->getElementType()) == 16) {
+      // With op_sel VOP3P instructions freely can access the low half or high
+      // half of a register, so any swizzle is free.
+
+      switch (Kind) {
+      case TTI::SK_Broadcast:
+      case TTI::SK_Reverse:
+      case TTI::SK_PermuteSingleSrc:
+        return 0;
+      default:
+        break;
+      }
+    }
+  }
+
+  return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
+}

test/Analysis/CostModel/AMDGPU/extractelement.ll

-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
-; CHECK: 'extractelement_v2i32'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i32>
+; GCN: 'extractelement_v2i32'
+; GCN: estimated cost of 0 for {{.*}} extractelement <2 x i32>
 define amdgpu_kernel void @extractelement_v2i32(i32 addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr) {
   %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr
   %elt = extractelement <2 x i32> %vec, i32 1
   store i32 %elt, i32 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v2f32'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x float>
+; GCN: 'extractelement_v2f32'
+; GCN: estimated cost of 0 for {{.*}} extractelement <2 x float>
 define amdgpu_kernel void @extractelement_v2f32(float addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) {
   %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
   %elt = extractelement <2 x float> %vec, i32 1
   store float %elt, float addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v3i32'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <3 x i32>
+; GCN: 'extractelement_v3i32'
+; GCN: estimated cost of 0 for {{.*}} extractelement <3 x i32>
 define amdgpu_kernel void @extractelement_v3i32(i32 addrspace(1)* %out, <3 x i32> addrspace(1)* %vaddr) {
   %vec = load <3 x i32>, <3 x i32> addrspace(1)* %vaddr
   %elt = extractelement <3 x i32> %vec, i32 1
   store i32 %elt, i32 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v4i32'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i32>
+; GCN: 'extractelement_v4i32'
+; GCN: estimated cost of 0 for {{.*}} extractelement <4 x i32>
 define amdgpu_kernel void @extractelement_v4i32(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %vaddr) {
   %vec = load <4 x i32>, <4 x i32> addrspace(1)* %vaddr
   %elt = extractelement <4 x i32> %vec, i32 1
   store i32 %elt, i32 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v8i32'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <8 x i32>
+; GCN: 'extractelement_v8i32'
+; GCN: estimated cost of 0 for {{.*}} extractelement <8 x i32>
 define amdgpu_kernel void @extractelement_v8i32(i32 addrspace(1)* %out, <8 x i32> addrspace(1)* %vaddr) {
   %vec = load <8 x i32>, <8 x i32> addrspace(1)* %vaddr
   %elt = extractelement <8 x i32> %vec, i32 1
   store i32 %elt, i32 addrspace(1)* %out
   ret void
 }
 
 ; FIXME: Should be non-0
-; CHECK: 'extractelement_v8i32_dynindex'
-; CHECK: estimated cost of 2 for {{.*}} extractelement <8 x i32>
+; GCN: 'extractelement_v8i32_dynindex'
+; GCN: estimated cost of 2 for {{.*}} extractelement <8 x i32>
 define amdgpu_kernel void @extractelement_v8i32_dynindex(i32 addrspace(1)* %out, <8 x i32> addrspace(1)* %vaddr, i32 %idx) {
   %vec = load <8 x i32>, <8 x i32> addrspace(1)* %vaddr
   %elt = extractelement <8 x i32> %vec, i32 %idx
   store i32 %elt, i32 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v2i64'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i64>
+; GCN: 'extractelement_v2i64'
+; GCN: estimated cost of 0 for {{.*}} extractelement <2 x i64>
 define amdgpu_kernel void @extractelement_v2i64(i64 addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr) {
   %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr
   %elt = extractelement <2 x i64> %vec, i64 1
   store i64 %elt, i64 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v3i64'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <3 x i64>
+; GCN: 'extractelement_v3i64'
+; GCN: estimated cost of 0 for {{.*}} extractelement <3 x i64>
 define amdgpu_kernel void @extractelement_v3i64(i64 addrspace(1)* %out, <3 x i64> addrspace(1)* %vaddr) {
   %vec = load <3 x i64>, <3 x i64> addrspace(1)* %vaddr
   %elt = extractelement <3 x i64> %vec, i64 1
   store i64 %elt, i64 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v4i64'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i64>
+; GCN: 'extractelement_v4i64'
+; GCN: estimated cost of 0 for {{.*}} extractelement <4 x i64>
 define amdgpu_kernel void @extractelement_v4i64(i64 addrspace(1)* %out, <4 x i64> addrspace(1)* %vaddr) {
   %vec = load <4 x i64>, <4 x i64> addrspace(1)* %vaddr
   %elt = extractelement <4 x i64> %vec, i64 1
   store i64 %elt, i64 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v8i64'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <8 x i64>
+; GCN: 'extractelement_v8i64'
+; GCN: estimated cost of 0 for {{.*}} extractelement <8 x i64>
 define amdgpu_kernel void @extractelement_v8i64(i64 addrspace(1)* %out, <8 x i64> addrspace(1)* %vaddr) {
   %vec = load <8 x i64>, <8 x i64> addrspace(1)* %vaddr
   %elt = extractelement <8 x i64> %vec, i64 1
   store i64 %elt, i64 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v4i8'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <4 x i8>
+; GCN: 'extractelement_v4i8'
+; GCN: estimated cost of 1 for {{.*}} extractelement <4 x i8>
 define amdgpu_kernel void @extractelement_v4i8(i8 addrspace(1)* %out, <4 x i8> addrspace(1)* %vaddr) {
   %vec = load <4 x i8>, <4 x i8> addrspace(1)* %vaddr
   %elt = extractelement <4 x i8> %vec, i8 1
   store i8 %elt, i8 addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'extractelement_v2i16'
-; CHECK: estimated cost of 0 for {{.*}} extractelement <2 x i16>
-define amdgpu_kernel void @extractelement_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+; GCN: 'extractelement_0_v2i16':
+; CI: estimated cost of 1 for {{.*}} extractelement <2 x i16> %vec, i16 0
+; VI: estimated cost of 0 for {{.*}} extractelement <2 x i16>
+; GFX9: estimated cost of 0 for {{.*}} extractelement <2 x i16>
+define amdgpu_kernel void @extractelement_0_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %elt = extractelement <2 x i16> %vec, i16 0
+  store i16 %elt, i16 addrspace(1)* %out
+  ret void
+}
+
+; GCN: 'extractelement_1_v2i16':
+; GCN: estimated cost of 1 for {{.*}} extractelement <2 x i16>
+define amdgpu_kernel void @extractelement_1_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
   %elt = extractelement <2 x i16> %vec, i16 1
   store i16 %elt, i16 addrspace(1)* %out
   ret void
 }
+
+; GCN: 'extractelement_var_v2i16'
+; GCN: estimated cost of 1 for {{.*}} extractelement <2 x i16>
+define amdgpu_kernel void @extractelement_var_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr, i32 %idx) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %elt = extractelement <2 x i16> %vec, i32 %idx
+  store i16 %elt, i16 addrspace(1)* %out
+  ret void
+}

test/Analysis/CostModel/AMDGPU/insertelement.ll

-; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
-; CHECK: 'insertelement_v2i32'
-; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i32>
+; GCN-LABEL: 'insertelement_v2i32'
+; GCN: estimated cost of 0 for {{.*}} insertelement <2 x i32>
 define amdgpu_kernel void @insertelement_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr) {
   %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr
-  %insert = insertelement <2 x i32> %vec, i32 1, i32 123
+  %insert = insertelement <2 x i32> %vec, i32 123, i32 1
   store <2 x i32> %insert, <2 x i32> addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'insertelement_v2i64'
-; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i64>
+; GCN-LABEL: 'insertelement_v2i64'
+; GCN: estimated cost of 0 for {{.*}} insertelement <2 x i64>
 define amdgpu_kernel void @insertelement_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr) {
   %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr
-  %insert = insertelement <2 x i64> %vec, i64 1, i64 123
+  %insert = insertelement <2 x i64> %vec, i64 123, i64 1
   store <2 x i64> %insert, <2 x i64> addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'insertelement_v2i16'
-; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i16>
-define amdgpu_kernel void @insertelement_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+; GCN-LABEL: 'insertelement_0_v2i16'
+; CI: estimated cost of 1 for {{.*}} insertelement <2 x i16>
+; VI: estimated cost of 0 for {{.*}} insertelement <2 x i16>
+; GFX9: estimated cost of 0 for {{.*}} insertelement <2 x i16>
+define amdgpu_kernel void @insertelement_0_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
   %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
-  %insert = insertelement <2 x i16> %vec, i16 1, i16 123
+  %insert = insertelement <2 x i16> %vec, i16 123, i16 0
   store <2 x i16> %insert, <2 x i16> addrspace(1)* %out
   ret void
 }
 
-; CHECK: 'insertelement_v2i8'
-; CHECK: estimated cost of 0 for {{.*}} insertelement <2 x i8>
-define amdgpu_kernel void @insertelement_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %vaddr) {
+; GCN-LABEL: 'insertelement_1_v2i16'
+; GCN: estimated cost of 1 for {{.*}} insertelement <2 x i16>
+define amdgpu_kernel void @insertelement_1_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %insert = insertelement <2 x i16> %vec, i16 123, i16 1
+  store <2 x i16> %insert, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: 'insertelement_1_v2i8'
+; GCN: estimated cost of 1 for {{.*}} insertelement <2 x i8>
+define amdgpu_kernel void @insertelement_1_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %vaddr) {
   %vec = load <2 x i8>, <2 x i8> addrspace(1)* %vaddr
-  %insert = insertelement <2 x i8> %vec, i8 1, i8 123
+  %insert = insertelement <2 x i8> %vec, i8 123, i8 1
   store <2 x i8> %insert, <2 x i8> addrspace(1)* %out
   ret void
 }

test/Analysis/CostModel/AMDGPU/shufflevector.ll

+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX9,GCN %s
+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji %s | FileCheck -check-prefixes=VI,GCN %s
+
+; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer
+define amdgpu_kernel void @shufflevector_00_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> zeroinitializer
+  store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 0, i32 1>
+define amdgpu_kernel void @shufflevector_01_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 0, i32 1>
+  store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+define amdgpu_kernel void @shufflevector_10_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; GFX9: estimated cost of 0 for {{.*}} shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+define amdgpu_kernel void @shufflevector_11_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr) {
+  %vec = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr
+  %shuf = shufflevector <2 x i16> %vec, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
+  ret void
+}
+
+; GCN: estimated cost of 2 for {{.*}} shufflevector <2 x i16> %vec0, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
+define amdgpu_kernel void @shufflevector_02_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %vaddr0, <2 x i16> addrspace(1)* %vaddr1) {
+  %vec0 = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr0
+  %vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %vaddr1
+  %shuf = shufflevector <2 x i16> %vec0, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
+  store <2 x i16> %shuf, <2 x i16> addrspace(1)* %out
+  ret void
+}