Index: llvm/include/llvm/Analysis/TargetTransformInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfo.h +++ llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -660,6 +660,10 @@ /// Return true if the target supports masked expand load. bool isLegalMaskedExpandLoad(Type *DataType) const; + /// Return true if the target supports vectorization of the intrinsic IID for + /// a given ElementCount VF. + bool isLegalVectorIntrinsic(Intrinsic::ID IID, ElementCount VF) const; + /// Return true if the target has a unified operation to calculate division /// and remainder. If so, the additional implicit multiplication and /// subtraction required to calculate a remainder from division are free. This @@ -1516,6 +1520,7 @@ virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; + virtual bool isLegalVectorIntrinsic(Intrinsic::ID IID, ElementCount VF) = 0; virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; virtual bool prefersVectorizedAddressing() = 0; @@ -1896,6 +1901,10 @@ bool isLegalMaskedExpandLoad(Type *DataType) override { return Impl.isLegalMaskedExpandLoad(DataType); } + bool isLegalVectorIntrinsic(Intrinsic::ID IID, ElementCount VF) override { + return Impl.isLegalVectorIntrinsic(IID, VF); + } + bool hasDivRemOp(Type *DataType, bool IsSigned) override { return Impl.hasDivRemOp(DataType, IsSigned); } Index: llvm/include/llvm/Analysis/TargetTransformInfoImpl.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -258,6 +258,10 @@ return false; } + bool isLegalVectorIntrinsic(Intrinsic::ID IID, ElementCount VF) const { + return isTriviallyVectorizable(IID); + } + bool isLegalMaskedCompressStore(Type *DataType) const { return false; } bool isLegalMaskedExpandLoad(Type *DataType) const { return false; } Index: llvm/lib/Analysis/TargetTransformInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetTransformInfo.cpp +++ llvm/lib/Analysis/TargetTransformInfo.cpp @@ -409,6 +409,11 @@ return TTIImpl->isLegalMaskedExpandLoad(DataType); } +bool TargetTransformInfo::isLegalVectorIntrinsic(Intrinsic::ID IID, + ElementCount VF) const { + return TTIImpl->isLegalVectorIntrinsic(IID, VF); +} + bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { return TTIImpl->hasDivRemOp(DataType, IsSigned); } Index: llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -252,6 +252,8 @@ return isLegalMaskedGatherScatter(DataType); } + bool isLegalVectorIntrinsic(Intrinsic::ID IID, ElementCount VF) const; + bool isLegalNTStore(Type *DataType, Align Alignment) { // NOTE: The logic below is mostly geared towards LV, which calls it with // vectors with 2 elements. We might want to improve that, if other Index: llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1703,3 +1703,23 @@ return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp); } + +bool AArch64TTIImpl::isLegalVectorIntrinsic(Intrinsic::ID IID, + ElementCount VF) const { + switch (IID) { + case Intrinsic::sin: + case Intrinsic::cos: + case Intrinsic::exp: + case Intrinsic::exp2: + case Intrinsic::log: + case Intrinsic::log10: + case Intrinsic::log2: + case Intrinsic::pow: + case Intrinsic::powi: + case Intrinsic::canonicalize: + // We can fall back on scalarization for fixed width vectors, but not for + // scalable vectors. + return !VF.isScalable(); + } + return BaseT::isLegalVectorIntrinsic(IID, VF); +} Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1505,13 +1505,18 @@ /// Returns true if the target machine supports all of the reduction /// variables found for the given VF. - bool canVectorizeReductions(ElementCount VF) { + bool canVectorizeReductions(ElementCount VF) const { return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool { RecurrenceDescriptor RdxDesc = Reduction.second; return TTI.isLegalToVectorizeReduction(RdxDesc, VF); })); } + /// Returns true if all instructions in the loop are legal to vectorize using + /// a scalable vectorization factor VF. If the loop is illegal the function + /// returns an appropriate error remark in Msg. + bool isScalableLoopLegal(ElementCount VF, StringRef &Msg) const; + /// Returns true if \p I is an instruction that will be scalarized with /// predication. Such instructions include conditional stores and /// instructions that may divide by zero. @@ -5644,6 +5649,38 @@ return false; } +bool LoopVectorizationCostModel::isScalableLoopLegal(ElementCount VF, + StringRef &Msg) const { + // Disable scalable vectorization if the loop contains unsupported reductions. + // Test that the loop-vectorizer can legalize all operations for this VF. + // FIXME: While for scalable vectors this is currently sufficient, this should + // be replaced by a more detailed mechanism that filters out specific VFs, + // instead of invalidating vectorization for a whole set of VFs based on the + // MaxVF. + if (!canVectorizeReductions(VF)) { + Msg = "Scalable vectorization not supported for the reduction " + "operations found in this loop."; + return false; + } + + // Iterate through all instructions in the loop ensuring that is legal to + // vectorize with a scalable VF. + for (BasicBlock *BB : TheLoop->blocks()) { + for (Instruction &I : *BB) { + if (auto *CI = dyn_cast(&I)) { + Intrinsic::ID VecID = getVectorIntrinsicIDForCall(CI, TLI); + if (VecID && VFDatabase::getMappings(*CI).empty() && + !TTI.isLegalVectorIntrinsic(VecID, VF)) { + Msg = "Scalable vectorization not supported for the call " + "instructions found in this loop"; + return false; + } + } + } + } + return true; +} + ElementCount LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) { if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors) { @@ -5657,17 +5694,9 @@ auto MaxScalableVF = ElementCount::getScalable( std::numeric_limits::max()); - // Disable scalable vectorization if the loop contains unsupported reductions. - // Test that the loop-vectorizer can legalize all operations for this MaxVF. - // FIXME: While for scalable vectors this is currently sufficient, this should - // be replaced by a more detailed mechanism that filters out specific VFs, - // instead of invalidating vectorization for a whole set of VFs based on the - // MaxVF. - if (!canVectorizeReductions(MaxScalableVF)) { - reportVectorizationInfo( - "Scalable vectorization not supported for the reduction " - "operations found in this loop.", - "ScalableVFUnfeasible", ORE, TheLoop); + StringRef Msg; + if (!isScalableLoopLegal(MaxScalableVF, Msg)) { + reportVectorizationInfo(Msg, "ScalableVFUnfeasible", ORE, TheLoop); return ElementCount::getScalable(0); } Index: llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll @@ -1,4 +1,6 @@ -; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -instcombine -mattr=+sve -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s +; RUN: opt -pass-remarks-missed=loop-vectorize -S -loop-vectorize -force-vector-interleave=1 \ +; RUN: -instcombine -mattr=+sve -mtriple aarch64-unknown-linux-gnu < %s 2>%t | FileCheck %s +; RUN: cat %t | FileCheck %s --check-prefix=CHECK-REMARKS define void @vec_load(i64 %N, double* nocapture %a, double* nocapture readonly %b) { ; CHECK-LABEL: @vec_load @@ -95,9 +97,57 @@ ret void } +; CHECK-REMARKS: Scalable vectorization not supported for the call instructions found in this loop +define void @vec_sin_no_mapping(float* noalias nocapture %dst, float* noalias nocapture readonly %src, i64 %n) { +; CHECK: @vec_sin_no_mapping +; CHECK: call fast <2 x float> @llvm.sin.v2f32 +; CHECK-NOT: @llvm.sqrt.nxv2f32 +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %i.07 = phi i64 [ %inc, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float, float* %src, i64 %i.07 + %0 = load float, float* %arrayidx, align 4 + %1 = tail call fast float @llvm.sqrt.f32(float %0) + %arrayidx1 = getelementptr inbounds float, float* %dst, i64 %i.07 + store float %1, float* %arrayidx1, align 4 + %inc = add nuw nsw i64 %i.07, 1 + %exitcond.not = icmp eq i64 %inc, %n + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1 + +for.cond.cleanup: ; preds = %for.body + ret void +} + + declare double @foo(double) declare i64 @bar(i64*) declare double @llvm.sin.f64(double) +declare float @llvm.sin.f32(float) +declare float @llvm.sqrt.f32(float) declare @foo_vec() declare @bar_vec()