TTI: Improve default costs for addrspacecast

[arsenm]

Summary

For some reason multiple places need to do this, and the variant the
loop unroller and inliner use was not handling it.

Also, introduce a new wrapper to be slightly more precise, since on
AMDGPU some addrspacecasts are free, but not no-ops.

[arsenm]

ping

[hfinkel]

include/llvm/CodeGen/TargetLowering.h

1591

Do you indent for targets to override this separately from isCheapAddrSpaceCast? You didn't make it virtual. I'd be happy renaming Cheap to Free and just saying that, in CGP, we only sink free addrspacecasts.

[arsenm]

include/llvm/CodeGen/TargetLowering.h

1591

Renaming cheap to free would be fine for me

[hfinkel]

include/llvm/CodeGen/TargetLowering.h

1591

Great. With that done, this LGTM.

[arsenm]

r362436

include/llvm/CodeGen/BasicTTIImpl.h

unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) {
     case Instruction::Trunc:
       if (TLI->isTruncateFree(OpTy, Ty))
         return TargetTransformInfo::TCC_Free;
       return TargetTransformInfo::TCC_Basic;
     case Instruction::ZExt:
       if (TLI->isZExtFree(OpTy, Ty))
         return TargetTransformInfo::TCC_Free;
       return TargetTransformInfo::TCC_Basic;
+
+    case Instruction::AddrSpaceCast:
+      if (TLI->isFreeAddrSpaceCast(OpTy->getPointerAddressSpace(),
+                                   Ty->getPointerAddressSpace()))
+        return TargetTransformInfo::TCC_Free;
+      return TargetTransformInfo::TCC_Basic;
     }
 
     return BaseT::getOperationCost(Opcode, Ty, OpTy);
   }
 
   unsigned getInliningThresholdMultiplier() { return 1; }
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
if (Opcode == Instruction::Trunc &&
         TLI->isTruncateFree(SrcLT.second, DstLT.second))
       return 0;
 
     if (Opcode == Instruction::ZExt &&
         TLI->isZExtFree(SrcLT.second, DstLT.second))
       return 0;
 
     if (Opcode == Instruction::AddrSpaceCast &&
-        TLI->isNoopAddrSpaceCast(Src->getPointerAddressSpace(),
+        TLI->isFreeAddrSpaceCast(Src->getPointerAddressSpace(),
                                  Dst->getPointerAddressSpace()))
       return 0;
 
     // If this is a zext/sext of a load, return 0 if the corresponding
     // extending load exists on target.
     if ((Opcode == Instruction::ZExt || Opcode == Instruction::SExt) &&
         I && isa<LoadInst>(I->getOperand(0))) {
         EVT ExtVT = EVT::getEVT(Dst);

include/llvm/CodeGen/TargetLowering.h

public:
   }
 
   /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
   /// are happy to sink it into basic blocks.
   virtual bool isCheapAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
     return isNoopAddrSpaceCast(SrcAS, DestAS);
   }
 
+  /// Returns true if a cast from SrcAS to DestAS is free, but not necessarily a
+  /// no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
+  bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
+    return isCheapAddrSpaceCast(SrcAS, DestAS);
+  }
+
   /// Return true if the pointer arguments to CI should be aligned by aligning
   /// the object whose address is being passed. If so then MinSize is set to the
   /// minimum size the object must be to be aligned and PrefAlign is set to the
   /// preferred alignment.
   virtual bool shouldAlignPointerArgs(CallInst * /*CI*/, unsigned & /*MinSize*/,
                                       unsigned & /*PrefAlign*/) const {
     return false;
   }

test/Analysis/CostModel/AMDGPU/addrspacecast.ll

 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri < %s | FileCheck %s
 
-; CHECK: 'addrspacecast_global_to_flat'
+; CHECK-LABEL: 'addrspacecast_global_to_flat'
 ; CHECK: estimated cost of 0 for {{.*}} addrspacecast i8 addrspace(1)* %ptr to i8*
 define i8* @addrspacecast_global_to_flat(i8 addrspace(1)* %ptr) #0 {
   %cast = addrspacecast i8 addrspace(1)* %ptr to i8*
   ret i8* %cast
 }
 
-; CHECK: 'addrspacecast_global_to_flat_v2'
+; CHECK-LABEL: 'addrspacecast_global_to_flat_v2'
 ; CHECK: estimated cost of 0 for {{.*}} addrspacecast <2 x i8 addrspace(1)*> %ptr to <2 x i8*>
 define <2 x i8*> @addrspacecast_global_to_flat_v2(<2 x i8 addrspace(1)*> %ptr) #0 {
   %cast = addrspacecast <2 x i8 addrspace(1)*> %ptr to <2 x i8*>
   ret <2 x i8*> %cast
 }
 
-; CHECK: 'addrspacecast_global_to_flat_v32'
+; CHECK-LABEL: 'addrspacecast_global_to_flat_v32'
 ; CHECK: estimated cost of 0 for {{.*}} addrspacecast <32 x i8 addrspace(1)*> %ptr to <32 x i8*>
 define <32 x i8*> @addrspacecast_global_to_flat_v32(<32 x i8 addrspace(1)*> %ptr) #0 {
   %cast = addrspacecast <32 x i8 addrspace(1)*> %ptr to <32 x i8*>
   ret <32 x i8*> %cast
 }
 
-; CHECK: 'addrspacecast_local_to_flat'
+; CHECK-LABEL: 'addrspacecast_local_to_flat'
 ; CHECK: estimated cost of 1 for {{.*}} addrspacecast i8 addrspace(3)* %ptr to i8*
 define i8* @addrspacecast_local_to_flat(i8 addrspace(3)* %ptr) #0 {
   %cast = addrspacecast i8 addrspace(3)* %ptr to i8*
   ret i8* %cast
 }
 
-; CHECK: 'addrspacecast_local_to_flat_v2'
+; CHECK-LABEL: 'addrspacecast_local_to_flat_v2'
 ; CHECK: estimated cost of 2 for {{.*}} addrspacecast <2 x i8 addrspace(3)*> %ptr to <2 x i8*>
 define <2 x i8*> @addrspacecast_local_to_flat_v2(<2 x i8 addrspace(3)*> %ptr) #0 {
   %cast = addrspacecast <2 x i8 addrspace(3)*> %ptr to <2 x i8*>
   ret <2 x i8*> %cast
 }
 
-; CHECK: 'addrspacecast_local_to_flat_v32'
+; CHECK-LABEL: 'addrspacecast_local_to_flat_v32'
 ; CHECK: estimated cost of 32 for {{.*}} addrspacecast <32 x i8 addrspace(3)*> %ptr to <32 x i8*>
 define <32 x i8*> @addrspacecast_local_to_flat_v32(<32 x i8 addrspace(3)*> %ptr) #0 {
   %cast = addrspacecast <32 x i8 addrspace(3)*> %ptr to <32 x i8*>
   ret <32 x i8*> %cast
 }
 
+; CHECK-LABEL: 'addrspacecast_flat_to_local'
+; CHECK: estimated cost of 0 for {{.*}} addrspacecast i8* %ptr to i8 addrspace(3)*
+define i8 addrspace(3)* @addrspacecast_flat_to_local(i8* %ptr) #0 {
+  %cast = addrspacecast i8* %ptr to i8 addrspace(3)*
+  ret i8 addrspace(3)* %cast
+}
+
+; CHECK-LABEL: 'addrspacecast_flat_to_local_v2'
+; CHECK: estimated cost of 0 for {{.*}} addrspacecast <2 x i8*> %ptr to <2 x i8 addrspace(3)*>
+define <2 x i8 addrspace(3)*> @addrspacecast_flat_to_local_v2(<2 x i8*> %ptr) #0 {
+  %cast = addrspacecast <2 x i8*> %ptr to <2 x i8 addrspace(3)*>
+  ret <2 x i8 addrspace(3)*> %cast
+}
+
+; CHECK-LABEL: 'addrspacecast_flat_to_local_v32'
+; CHECK: estimated cost of 0 for {{.*}} addrspacecast <32 x i8*> %ptr to <32 x i8 addrspace(3)*>
+define <32 x i8 addrspace(3)*> @addrspacecast_flat_to_local_v32(<32 x i8*> %ptr) #0 {
+  %cast = addrspacecast <32 x i8*> %ptr to <32 x i8 addrspace(3)*>
+  ret <32 x i8 addrspace(3)*> %cast
+}
+
 attributes #0 = { nounwind readnone }

test/Transforms/LoopUnroll/AMDGPU/unroll-cost-addrspacecast.ll

+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii -loop-unroll -unroll-threshold=75 -unroll-peel-count=0 -unroll-allow-partial=false -unroll-max-iteration-count-to-analyze=16 < %s | FileCheck %s
+
+; CHECK-LABEL: @test_func_addrspacecast_cost_noop(
+; CHECK-NOT: br i1
+define amdgpu_kernel void @test_func_addrspacecast_cost_noop(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float addrspace(1)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float addrspace(1)* %out, i32 %indvars.iv
+  %cast.in = addrspacecast float addrspace(1)* %arrayidx.in to float*
+  %cast.out = addrspacecast float addrspace(1)* %arrayidx.out to float*
+  %load = load float, float* %cast.in
+  %fmul = fmul float %load, %sum.02
+  store float %fmul, float* %cast.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Free, but not a no-op
+; CHECK-LABEL: @test_func_addrspacecast_cost_free(
+; CHECK-NOT: br i1
+define amdgpu_kernel void @test_func_addrspacecast_cost_free(float* noalias nocapture %out, float* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float* %out, i32 %indvars.iv
+  %cast.in = addrspacecast float* %arrayidx.in to float addrspace(3)*
+  %cast.out = addrspacecast float* %arrayidx.out to float addrspace(3)*
+  %load = load float, float addrspace(3)* %cast.in
+  %fmul = fmul float %load, %sum.02
+  store float %fmul, float addrspace(3)* %cast.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @test_func_addrspacecast_cost_nonfree(
+; CHECK: br i1 %exitcond
+define amdgpu_kernel void @test_func_addrspacecast_cost_nonfree(float addrspace(3)* noalias nocapture %out, float addrspace(3)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float addrspace(3)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float addrspace(3)* %out, i32 %indvars.iv
+  %cast.in = addrspacecast float addrspace(3)* %arrayidx.in to float*
+  %cast.out = addrspacecast float addrspace(3)* %arrayidx.out to float*
+  %load = load float, float* %cast.in
+  %fmul = fmul float %load, %sum.02
+  store float %fmul, float* %cast.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }