diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -117,7 +117,7 @@ SmallVector ParamTys; SmallVector Arguments; FastMathFlags FMF; - unsigned VF = 1; + ElementCount VF = ElementCount::getFixed(1); // If ScalarizationCost is UINT_MAX, the cost of scalarizing the // arguments and the return value will be computed based on types. unsigned ScalarizationCost = std::numeric_limits::max(); @@ -128,15 +128,10 @@ IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI); IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, - unsigned Factor); - IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, - ElementCount Factor) - : IntrinsicCostAttributes(Id, CI, Factor.getKnownMinValue()) { - assert(!Factor.isScalable()); - } + ElementCount Factor); IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, - unsigned Factor, unsigned ScalarCost); + ElementCount Factor, unsigned ScalarCost); IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, ArrayRef Tys, FastMathFlags Flags); @@ -159,7 +154,7 @@ Intrinsic::ID getID() const { return IID; } const IntrinsicInst *getInst() const { return II; } Type *getReturnType() const { return RetTy; } - unsigned getVectorFactor() const { return VF; } + ElementCount getVectorFactor() const { return VF; } FastMathFlags getFlags() const { return FMF; } unsigned getScalarizationCost() const { return ScalarizationCost; } const SmallVectorImpl &getArgs() const { return Arguments; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1207,11 +1207,12 @@ if (isa(RetTy)) return BaseT::getIntrinsicInstrCost(ICA, CostKind); - unsigned VF = ICA.getVectorFactor(); - unsigned RetVF = - (RetTy->isVectorTy() ? cast(RetTy)->getNumElements() - : 1); - assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type"); + ElementCount VF = ICA.getVectorFactor(); + ElementCount RetVF = + (RetTy->isVectorTy() ? cast(RetTy)->getElementCount() + : ElementCount::getFixed(1)); + assert((RetVF.isScalar() || VF.isScalar()) && + "VF > 1 and RetVF is a vector type"); const IntrinsicInst *I = ICA.getInst(); const SmallVectorImpl &Args = ICA.getArgs(); FastMathFlags FMF = ICA.getFlags(); @@ -1221,13 +1222,15 @@ case Intrinsic::cttz: // FIXME: If necessary, this should go in target-specific overrides. - if (VF == 1 && RetVF == 1 && getTLI()->isCheapToSpeculateCttz()) + if (VF.isScalar() && RetVF.isScalar() && + getTLI()->isCheapToSpeculateCttz()) return TargetTransformInfo::TCC_Basic; break; case Intrinsic::ctlz: // FIXME: If necessary, this should go in target-specific overrides. - if (VF == 1 && RetVF == 1 && getTLI()->isCheapToSpeculateCtlz()) + if (VF.isScalar() && RetVF.isScalar() && + getTLI()->isCheapToSpeculateCtlz()) return TargetTransformInfo::TCC_Basic; break; @@ -1235,7 +1238,7 @@ return thisT()->getMemcpyCost(ICA.getInst()); case Intrinsic::masked_scatter: { - assert(VF == 1 && "Can't vectorize types here."); + assert(VF.isScalar() && "Can't vectorize types here."); const Value *Mask = Args[3]; bool VarMask = !isa(Mask); Align Alignment = cast(Args[2])->getAlignValue(); @@ -1244,7 +1247,7 @@ VarMask, Alignment, CostKind, I); } case Intrinsic::masked_gather: { - assert(VF == 1 && "Can't vectorize types here."); + assert(VF.isScalar() && "Can't vectorize types here."); const Value *Mask = Args[2]; bool VarMask = !isa(Mask); Align Alignment = cast(Args[1])->getAlignValue(); @@ -1318,23 +1321,26 @@ SmallVector Types; for (const Value *Op : Args) { Type *OpTy = Op->getType(); - assert(VF == 1 || !OpTy->isVectorTy()); - Types.push_back(VF == 1 ? OpTy : FixedVectorType::get(OpTy, VF)); + assert(VF.isScalar() || !OpTy->isVectorTy()); + Types.push_back(VF.isScalar() + ? OpTy + : FixedVectorType::get(OpTy, VF.getKnownMinValue())); } - if (VF > 1 && !RetTy->isVoidTy()) - RetTy = FixedVectorType::get(RetTy, VF); + if (VF.isVector() && !RetTy->isVoidTy()) + RetTy = FixedVectorType::get(RetTy, VF.getKnownMinValue()); // Compute the scalarization overhead based on Args for a vector // intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while // CostModel will pass a vector RetTy and VF is 1. unsigned ScalarizationCost = std::numeric_limits::max(); - if (RetVF > 1 || VF > 1) { + if (RetVF.isVector() || VF.isVector()) { ScalarizationCost = 0; if (!RetTy->isVoidTy()) ScalarizationCost += getScalarizationOverhead(cast(RetTy), true, false); - ScalarizationCost += getOperandsScalarizationOverhead(Args, VF); + ScalarizationCost += + getOperandsScalarizationOverhead(Args, VF.getKnownMinValue()); } IntrinsicCostAttributes Attrs(IID, RetTy, Types, FMF, ScalarizationCost, I); diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -79,9 +79,10 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, - unsigned Factor) : - RetTy(CI.getType()), IID(Id), VF(Factor) { + ElementCount Factor) + : RetTy(CI.getType()), IID(Id), VF(Factor) { + assert(!Factor.isScalable() && "Scalable vectors are not yet supported"); if (auto *FPMO = dyn_cast(&CI)) FMF = FPMO->getFastMathFlags(); @@ -93,9 +94,9 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI, - unsigned Factor, - unsigned ScalarCost) : - RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) { + ElementCount Factor, + unsigned ScalarCost) + : RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) { if (const auto *FPMO = dyn_cast(&CI)) FMF = FPMO->getFastMathFlags(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -696,7 +696,7 @@ return getTypeBasedIntrinsicInstrCost(ICA, CostKind); Type *RetTy = ICA.getReturnType(); - unsigned VF = ICA.getVectorFactor(); + unsigned VF = ICA.getVectorFactor().getFixedValue(); unsigned RetVF = (RetTy->isVectorTy() ? cast(RetTy)->getNumElements() : 1); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3390,7 +3390,7 @@ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); // Calculate the cost of the scalar and vector calls. - IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getNumElements()); + IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getElementCount()); int IntrinsicCost = TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput); @@ -3773,7 +3773,7 @@ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); // Calculate the cost of the scalar and vector calls. - IntrinsicCostAttributes CostAttrs(ID, *CI, 1, 1); + IntrinsicCostAttributes CostAttrs(ID, *CI, ElementCount::getFixed(1), 1); int ScalarEltCost = TTI->getIntrinsicInstrCost(CostAttrs, CostKind); if (NeedToShuffleReuses) { ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;