TTI: Allow targets to override getScalarizationOverhead

[arsenm]

Summary

AMDGPU wants to mostly pretend that vectors don't exist and
there's no cost to scalarizing vectors.

[arsenm]

ping

[arsenm]

ping

[arsenm]

ping

[arsenm]

ping

[arsenm]

ping

[hfinkel]

If the vectors don't exist, shouldn't the insert/extract costs be zero?

[arsenm]

In D15552#375044, @hfinkel wrote:

If the vectors don't exist, shouldn't the insert/extract costs be zero?

The extract cost is zero. I'm not sure a vector insert should be considered as 0, because the only time this would really happen in the ISA is if packing a vector for a store

[hfinkel]

In D15552#379215, @arsenm wrote:

In D15552#375044, @hfinkel wrote:

If the vectors don't exist, shouldn't the insert/extract costs be zero?

The extract cost is zero. I'm not sure a vector insert should be considered as 0, because the only time this would really happen in the ISA is if packing a vector for a store

I don't understand what this means. What are you packing into? Is this packing in any way associated with the vector insertion IR instruction? But if you don't actually have vectors, then why is this cost not just part of the store cost?

[arsenm]

In D15552#381685, @hfinkel wrote:

In D15552#379215, @arsenm wrote:

In D15552#375044, @hfinkel wrote:

If the vectors don't exist, shouldn't the insert/extract costs be zero?

The extract cost is zero. I'm not sure a vector insert should be considered as 0, because the only time this would really happen in the ISA is if packing a vector for a store

I don't understand what this means. What are you packing into? Is this packing in any way associated with the vector insertion IR instruction? But if you don't actually have vectors, then why is this cost not just part of the store cost?

That would make more sense. I'll go with the other plan of making insertelement free

include/llvm/Analysis/TargetTransformInfoImpl.h

public:
   unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty,
                                   TTI::OperandValueKind Opd1Info,
                                   TTI::OperandValueKind Opd2Info,
                                   TTI::OperandValueProperties Opd1PropInfo,
                                   TTI::OperandValueProperties Opd2PropInfo) {
     return 1;
   }
 
+  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
+    return 1;
+  }
+
   unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Ty, int Index,
                           Type *SubTp) {
     return 1;
   }
 
   unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) { return 1; }
 
   unsigned getCFInstrCost(unsigned Opcode) { return 1; }
if (const CastInst *CI = dyn_cast<CastInst>(U)) {
       if (isa<CmpInst>(CI->getOperand(0)))
         return TTI::TCC_Free;
     }
 
     return static_cast<T *>(this)->getOperationCost(
         Operator::getOpcode(U), U->getType(),
         U->getNumOperands() == 1 ? U->getOperand(0)->getType() : nullptr);
   }
+
+  /// Estimate the overhead of scalarizing an instruction. Insert and Extract
+  /// are set if the result needs to be inserted and/or extracted from vectors.
+  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
+    assert(Ty->isVectorTy() && "Can only scalarize vectors");
+    unsigned Cost = 0;
+
+    for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
+      if (Insert)
+        Cost += static_cast<T *>(this)
+          ->getVectorInstrCost(Instruction::InsertElement, Ty, i);
+      if (Extract)
+        Cost += static_cast<T *>(this)
+          ->getVectorInstrCost(Instruction::ExtractElement, Ty, i);
+    }
+
+    return Cost;
+  }
 };
 }
 
 #endif

include/llvm/CodeGen/BasicTTIImpl.h

 /// We need these methods implemented in the derived class so that this class
 /// doesn't have to duplicate storage for them.
 template <typename T>
 class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 private:
   typedef TargetTransformInfoImplCRTPBase<T> BaseT;
   typedef TargetTransformInfo TTI;
 
-  /// Estimate the overhead of scalarizing an instruction. Insert and Extract
-  /// are set if the result needs to be inserted and/or extracted from vectors.
-  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
-    assert(Ty->isVectorTy() && "Can only scalarize vectors");
-    unsigned Cost = 0;
-
-    for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
-      if (Insert)
-        Cost += static_cast<T *>(this)
-                    ->getVectorInstrCost(Instruction::InsertElement, Ty, i);
-      if (Extract)
-        Cost += static_cast<T *>(this)
-                    ->getVectorInstrCost(Instruction::ExtractElement, Ty, i);
-    }
-
-    return Cost;
-  }
-
   /// Estimate the cost overhead of SK_Alternate shuffle.
   unsigned getAltShuffleOverhead(Type *Ty) {
     assert(Ty->isVectorTy() && "Can only shuffle vectors");
     unsigned Cost = 0;
     // Shuffle cost is equal to the cost of extracting element from its argument
     // plus the cost of inserting them onto the result vector.
 
     // e.g. <4 x float> has a mask of <0,5,2,7> i.e we need to extract from
unsigned getArithmeticInstrCost(
     // Else, assume that we need to scalarize this op.
     if (Ty->isVectorTy()) {
       unsigned Num = Ty->getVectorNumElements();
       unsigned Cost = static_cast<T *>(this)
                           ->getArithmeticInstrCost(Opcode, Ty->getScalarType());
       // return the cost of multiple scalar invocation plus the cost of
       // inserting
       // and extracting the values.
-      return getScalarizationOverhead(Ty, true, true) + Num * Cost;
+      unsigned ScalarizeCost
+        = static_cast<T *>(this)->getScalarizationOverhead(Ty, true, true);
+      return ScalarizeCost + Num * Cost;
     }
 
     // We don't know anything about this scalar instruction.
     return OpCost;
   }
 
   unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
                           Type *SubTp) {
if (Dst->isVectorTy() && Src->isVectorTy()) {
       // if the vectors are legalized to different types, estimate the
       // scalarization costs.
       unsigned Num = Dst->getVectorNumElements();
       unsigned Cost = static_cast<T *>(this)->getCastInstrCost(
           Opcode, Dst->getScalarType(), Src->getScalarType());
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
-      return getScalarizationOverhead(Dst, true, true) + Num * Cost;
+      unsigned ScalarizeCost
+        = static_cast<T *>(this)->getScalarizationOverhead(Dst, true, true);
+      return ScalarizeCost + Num * Cost;
     }
 
     // We already handled vector-to-vector and scalar-to-scalar conversions.
     // This
     // is where we handle bitcast between vectors and scalars. We need to assume
     //  that the conversion is scalarized in one way or another.
-    if (Opcode == Instruction::BitCast)
+    if (Opcode == Instruction::BitCast) {
+      T *This = static_cast<T *>(this);
       // Illegal bitcasts are done by storing and loading from a stack slot.
-      return (Src->isVectorTy() ? getScalarizationOverhead(Src, false, true)
-                                : 0) +
-             (Dst->isVectorTy() ? getScalarizationOverhead(Dst, true, false)
-                                : 0);
+      return (Src->isVectorTy() ?
+              This->getScalarizationOverhead(Src, false, true) : 0) +
+        (Dst->isVectorTy() ?
+         This->getScalarizationOverhead(Dst, true, false) : 0);
+    }
 
     llvm_unreachable("Unhandled cast");
   }
 
   unsigned getCFInstrCost(unsigned Opcode) {
     // Branches are assumed to be predicted.
     return 0;
   }
if (ValTy->isVectorTy()) {
       if (CondTy)
         CondTy = CondTy->getScalarType();
       unsigned Cost = static_cast<T *>(this)->getCmpSelInstrCost(
           Opcode, ValTy->getScalarType(), CondTy);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting
       // and extracting the values.
-      return getScalarizationOverhead(ValTy, true, false) + Num * Cost;
+      unsigned ScalarizeCost
+        = static_cast<T *>(this)->getScalarizationOverhead(ValTy, true, false);
+      return ScalarizeCost + Num * Cost;
     }
 
     // Unknown scalar opcode.
     return 1;
   }
 
   unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
     std::pair<unsigned, MVT> LT =
if (Src->isVectorTy() &&
       if (Opcode == Instruction::Store)
         LA = getTLI()->getTruncStoreAction(LT.second, MemVT);
       else
         LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT);
 
       if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) {
         // This is a vector load/store for some illegal type that is scalarized.
         // We must account for the cost of building or decomposing the vector.
-        Cost += getScalarizationOverhead(Src, Opcode != Instruction::Store,
-                                         Opcode == Instruction::Store);
+
+        unsigned ScalarizeCost = static_cast<T *>(this)->
+          getScalarizationOverhead(Src, Opcode != Instruction::Store,
+        Cost += ScalarizeCost;
       }
     }
 
     return Cost;
   }
 
   unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
                                       unsigned Factor,
unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     unsigned ISD = 0;
     switch (IID) {
     default: {
       // Assume that we need to scalarize this intrinsic.
       unsigned ScalarizationCost = 0;
       unsigned ScalarCalls = 1;
       Type *ScalarRetTy = RetTy;
       if (RetTy->isVectorTy()) {
-        ScalarizationCost = getScalarizationOverhead(RetTy, true, false);
+        ScalarizationCost
+          = static_cast<T *>(this)->getScalarizationOverhead(RetTy, true, false);
         ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements());
         ScalarRetTy = RetTy->getScalarType();
       }
       SmallVector<Type *, 4> ScalarTys;
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         Type *Ty = Tys[i];
         if (Ty->isVectorTy()) {
-          ScalarizationCost += getScalarizationOverhead(Ty, false, true);
+          ScalarizationCost +=
+            static_cast<T *>(this)->getScalarizationOverhead(Ty, false, true);
           ScalarCalls = std::max(ScalarCalls, Ty->getVectorNumElements());
           Ty = Ty->getScalarType();
         }
         ScalarTys.push_back(Ty);
       }
       if (ScalarCalls == 1)
         return 1; // Return cost of a scalar intrinsic. Assume it to be cheap.
 
if (IID == Intrinsic::fmuladd)
                  ->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) +
              static_cast<T *>(this)
                  ->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy);
 
     // Else, assume that we need to scalarize this intrinsic. For math builtins
     // this will emit a costly libcall, adding call overhead and spills. Make it
     // very expensive.
     if (RetTy->isVectorTy()) {
-      unsigned ScalarizationCost = getScalarizationOverhead(RetTy, true, false);
+      unsigned ScalarizationCost
+        = static_cast<T *>(this)->getScalarizationOverhead(RetTy, true, false);
       unsigned ScalarCalls = RetTy->getVectorNumElements();
       SmallVector<Type *, 4> ScalarTys;
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         Type *Ty = Tys[i];
         if (Ty->isVectorTy())
           Ty = Ty->getScalarType();
         ScalarTys.push_back(Ty);
       }
       unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost(
           IID, RetTy->getScalarType(), ScalarTys);
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         if (Tys[i]->isVectorTy()) {
-          ScalarizationCost += getScalarizationOverhead(Tys[i], false, true);
+          ScalarizationCost +=
+            static_cast<T *>(this)->getScalarizationOverhead(Tys[i], false,
+                                                             true);
           ScalarCalls = std::max(ScalarCalls, Tys[i]->getVectorNumElements());
         }
       }
 
       return ScalarCalls * ScalarCost + ScalarizationCost;
     }
 
     // This is going to be turned into a library call, make it expensive.
unsigned getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise) {
     unsigned ArithCost =
         NumReduxLevels *
         static_cast<T *>(this)->getArithmeticInstrCost(Opcode, Ty);
     // Assume the pairwise shuffles add a cost.
     unsigned ShuffleCost =
         NumReduxLevels * (IsPairwise + 1) *
         static_cast<T *>(this)
             ->getShuffleCost(TTI::SK_ExtractSubvector, Ty, NumVecElts / 2, Ty);
-    return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true);
+    unsigned ScalarizeCost
+      = static_cast<T *>(this)->getScalarizationOverhead(Ty, false, true);
+    return ShuffleCost + ArithCost + ScalarizeCost;
   }
 
   /// @}
 };
 
 /// \brief Concrete BasicTTIImpl that can be used if no further customization
 /// is needed.
 class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> {

lib/Target/AArch64/AArch64TargetTransformInfo.h

 class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   typedef BasicTTIImplBase<AArch64TTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
   const AArch64Subtarget *ST;
   const AArch64TargetLowering *TLI;
 
-  /// Estimate the overhead of scalarizing an instruction. Insert and Extract
-  /// are set if the result needs to be inserted and/or extracted from vectors.
-  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract);
-
   const AArch64Subtarget *getST() const { return ST; }
   const AArch64TargetLowering *getTLI() const { return TLI; }
 
   enum MemIntrinsicType {
     VECTOR_LDST_TWO_ELEMENTS,
     VECTOR_LDST_THREE_ELEMENTS,
     VECTOR_LDST_FOUR_ELEMENTS
   };

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
     assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
     return ST->hasBCNT(TyWidth) ? TTI::PSK_FastHardware : TTI::PSK_Software;
   }
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector);
   unsigned getMaxInterleaveFactor(unsigned VF);
 
+  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
+    return 0;
+  }
+
   int getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index);
   bool isSourceOfDivergence(const Value *V) const;
 };
 
 } // end namespace llvm
 
 #endif

lib/Target/ARM/ARMTargetTransformInfo.h

 class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   typedef BasicTTIImplBase<ARMTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
   const ARMSubtarget *ST;
   const ARMTargetLowering *TLI;
 
-  /// Estimate the overhead of scalarizing an instruction. Insert and Extract
-  /// are set if the result needs to be inserted and/or extracted from vectors.
-  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract);
-
   const ARMSubtarget *getST() const { return ST; }
   const ARMTargetLowering *getTLI() const { return TLI; }
 
 public:
   explicit ARMTTIImpl(const ARMBaseTargetMachine *TM, const Function &F)
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 

lib/Target/X86/X86TargetTransformInfo.h

 class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   typedef BasicTTIImplBase<X86TTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
   const X86Subtarget *ST;
   const X86TargetLowering *TLI;
 
-  int getScalarizationOverhead(Type *Ty, bool Insert, bool Extract);
-
   const X86Subtarget *getST() const { return ST; }
   const X86TargetLowering *getTLI() const { return TLI; }
 
 public:
   explicit X86TTIImpl(const X86TargetMachine *TM, const Function &F)
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 

lib/Target/X86/X86TargetTransformInfo.cpp

if (Index != -1U) {
     // Floating point scalars are already located in index #0.
     if (Val->getScalarType()->isFloatingPointTy() && Index == 0)
       return 0;
   }
 
   return BaseT::getVectorInstrCost(Opcode, Val, Index);
 }
 
-int X86TTIImpl::getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
-  assert (Ty->isVectorTy() && "Can only scalarize vectors");
-  int Cost = 0;
-
-  for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
-    if (Insert)
-      Cost += getVectorInstrCost(Instruction::InsertElement, Ty, i);
-    if (Extract)
-      Cost += getVectorInstrCost(Instruction::ExtractElement, Ty, i);
-  }
-
-  return Cost;
-}
-
 int X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                                 unsigned AddressSpace) {
   // Handle non-power-of-two vectors such as <3 x float>
   if (VectorType *VTy = dyn_cast<VectorType>(Src)) {
     unsigned NumElem = VTy->getVectorNumElements();
 
     // Handle a few common cases:
     // <3 x float>

test/Analysis/CostModel/AMDGPU/add.ll

+; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+
+; CHECK: 'add_i32'
+; CHECK: estimated cost of 1 for {{.*}} add i32
+define void @add_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %vaddr, i32 %b) #0 {
+  %vec = load i32, i32 addrspace(1)* %vaddr
+  %add = add i32 %vec, %b
+  store i32 %add, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK: 'add_v2i32'
+; CHECK: estimated cost of 2 for {{.*}} add <2 x i32>
+define void @add_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %vaddr, <2 x i32> %b) #0 {
+  %vec = load <2 x i32>, <2 x i32> addrspace(1)* %vaddr
+  %add = add <2 x i32> %vec, %b
+  store <2 x i32> %add, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+; CHECK: 'add_v3i32'
+; CHECK: estimated cost of 3 for {{.*}} add <3 x i32>
+define void @add_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %vaddr, <3 x i32> %b) #0 {
+  %vec = load <3 x i32>, <3 x i32> addrspace(1)* %vaddr
+  %add = add <3 x i32> %vec, %b
+  store <3 x i32> %add, <3 x i32> addrspace(1)* %out
+  ret void
+}
+
+; CHECK: 'add_i64'
+; CHECK: estimated cost of 1 for {{.*}} add i64
+define void @add_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %vaddr, i64 %b) #0 {
+  %vec = load i64, i64 addrspace(1)* %vaddr
+  %add = add i64 %vec, %b
+  store i64 %add, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK: 'add_v2i64'
+; CHECK: estimated cost of 2 for {{.*}} add <2 x i64>
+define void @add_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %vaddr, <2 x i64> %b) #0 {
+  %vec = load <2 x i64>, <2 x i64> addrspace(1)* %vaddr
+  %add = add <2 x i64> %vec, %b
+  store <2 x i64> %add, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+; CHECK: 'add_v3i64'
+; CHECK: estimated cost of 3 for {{.*}} add <3 x i64>
+define void @add_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> addrspace(1)* %vaddr, <3 x i64> %b) #0 {
+  %vec = load <3 x i64>, <3 x i64> addrspace(1)* %vaddr
+  %add = add <3 x i64> %vec, %b
+  store <3 x i64> %add, <3 x i64> addrspace(1)* %out
+  ret void
+}
+

test/Analysis/CostModel/AMDGPU/fabs.ll

 ; RUN: opt -cost-model -analyze -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
 
 ; CHECK: 'fabs_f32'
 ; CHECK: estimated cost of 0 for {{.*}} call float @llvm.fabs.f32
 define void @fabs_f32(float addrspace(1)* %out, float addrspace(1)* %vaddr) #0 {
   %vec = load float, float addrspace(1)* %vaddr
   %fabs = call float @llvm.fabs.f32(float %vec) #1
   store float %fabs, float addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_v2f32'
-; CHECK: estimated cost of 2 for {{.*}} call <2 x float> @llvm.fabs.v2f32
+; CHECK: estimated cost of 0 for {{.*}} call <2 x float> @llvm.fabs.v2f32
 define void @fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x float>, <2 x float> addrspace(1)* %vaddr
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %vec) #1
   store <2 x float> %fabs, <2 x float> addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_v3f32'
-; CHECK: estimated cost of 3 for {{.*}} call <3 x float> @llvm.fabs.v3f32
+; CHECK: estimated cost of 0 for {{.*}} call <3 x float> @llvm.fabs.v3f32
 define void @fabs_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %vaddr) #0 {
   %vec = load <3 x float>, <3 x float> addrspace(1)* %vaddr
   %fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %vec) #1
   store <3 x float> %fabs, <3 x float> addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_f64'
 ; CHECK: estimated cost of 0 for {{.*}} call double @llvm.fabs.f64
 define void @fabs_f64(double addrspace(1)* %out, double addrspace(1)* %vaddr) #0 {
   %vec = load double, double addrspace(1)* %vaddr
   %fabs = call double @llvm.fabs.f64(double %vec) #1
   store double %fabs, double addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_v2f64'
-; CHECK: estimated cost of 2 for {{.*}} call <2 x double> @llvm.fabs.v2f64
+; CHECK: estimated cost of 0 for {{.*}} call <2 x double> @llvm.fabs.v2f64
 define void @fabs_v2f64(<2 x double> addrspace(1)* %out, <2 x double> addrspace(1)* %vaddr) #0 {
   %vec = load <2 x double>, <2 x double> addrspace(1)* %vaddr
   %fabs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %vec) #1
   store <2 x double> %fabs, <2 x double> addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_v3f64'
-; CHECK: estimated cost of 3 for {{.*}} call <3 x double> @llvm.fabs.v3f64
+; CHECK: estimated cost of 0 for {{.*}} call <3 x double> @llvm.fabs.v3f64
 define void @fabs_v3f64(<3 x double> addrspace(1)* %out, <3 x double> addrspace(1)* %vaddr) #0 {
   %vec = load <3 x double>, <3 x double> addrspace(1)* %vaddr
   %fabs = call <3 x double> @llvm.fabs.v3f64(<3 x double> %vec) #1
   store <3 x double> %fabs, <3 x double> addrspace(1)* %out
   ret void
 }
 
 ; CHECK: 'fabs_f16'