diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -390,12 +390,16 @@ FunctionAnalysisManager::Invalidator &) { return false; } - /// Returns the largest vectorization factor used in the list of /// vector functions. unsigned getWidestVF(StringRef ScalarF) const { return Impl->getWidestVF(ScalarF); } + + /// Check if the function "F" is listed in a library known to LLVM. + bool isKnownVectorFunctionInLibrary(StringRef F) const { + return this->isFunctionVectorizable(F); + } }; /// Analysis pass providing the \c TargetLibraryInfo. diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -16,6 +16,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -116,10 +117,10 @@ /// Holds the VFShape for a specific scalar to vector function mapping. struct VFInfo { - VFShape Shape; // Classification of the vector function. - StringRef ScalarName; // Scalar Function Name. - StringRef VectorName; // Vector Function Name associated to this VFInfo. - VFISAKind ISA; // Instruction Set Architecture. + VFShape Shape; /// Classification of the vector function. + std::string ScalarName; /// Scalar Function Name. + std::string VectorName; /// Vector Function Name associated to this VFInfo. + VFISAKind ISA; /// Instruction Set Architecture. // Comparison operator. bool operator==(const VFInfo &Other) const { @@ -131,6 +132,13 @@ namespace VFABI { /// LLVM Internal VFABI ISA token for vector functions. static constexpr char const *_LLVM_ = "_LLVM_"; +/// Prefix for internal name redirection for vector function that +/// tells the compiler to scalarize the call using the scalar name +/// of the function. For example, a mangled name like +/// `_ZGV_LLVM_N2v_foo(_LLVM_Scalarize_foo)` would tell the +/// vectorizer to vectorize the scalar call `foo`, and to scalarize +/// it once vectorization is done. +static constexpr char const *_LLVM_Scalarize_ = "_LLVM_Scalarize_"; /// Function to contruct a VFInfo out of a mangled names in the /// following format: @@ -167,6 +175,75 @@ SmallVectorImpl &VariantMappings); } // end namespace VFABI +/// The Vector Function Database. +/// +/// Helper class used to find the vector functions associated to a +/// scalar CallInst. +class VFDatabase { + /// The Module of the CallInst CI. + const Module *M; + /// List of vector functions descritors associated to the call + /// instruction. + const SmallVector ScalarToVectorMappings; + + /// Retreive the scalar-to-vector mappings associated to the rule of + /// a vector Function ABI. + static void getVFABIMappings(const CallInst &CI, + SmallVectorImpl &Mappings) { + const StringRef ScalarName = CI.getCalledFunction()->getName(); + const StringRef S = + CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) + .getValueAsString(); + if (S.empty()) + return; + + SmallVector ListOfStrings; + VFABI::getVectorVariantNames(CI, ListOfStrings); + for (const auto &MangledName : ListOfStrings) { + const Optional Shape = VFABI::tryDemangleForVFABI(MangledName); + // A match is found via scalar and vector names, and also by + // ensuring that the variant described in the attribute has a + // corresponding definition or declaration of the vector + // function in the Module M. + if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) { + assert(CI.getModule()->getFunction(Shape.getValue().VectorName) && + "Vector function is missing."); + Mappings.push_back(Shape.getValue()); + } + } + } + +public: + /// Retrieve all the VFInfo instances associated to the CallInst CI. + static SmallVector getMappings(const CallInst &CI) { + SmallVector Ret; + + // Get mappings from the Vector Function ABI variants. + getVFABIMappings(CI, Ret); + + // Other non-VFABI variants should be retrieved here. + + return Ret; + } + + /// Constructor, requires a CallInst instance. + VFDatabase(CallInst &CI) + : M(CI.getModule()), ScalarToVectorMappings(VFDatabase::getMappings(CI)) { + } + /// \defgroup VFDatabase query interface. + /// + /// @{ + /// Retrieve the Function with VFShape \p Shape. + Function *getVectorizedFunction(const VFShape &Shape) const { + for (const auto &Info : ScalarToVectorMappings) + if (Info.Shape == Shape) + return M->getFunction(Info.VectorName); + + return nullptr; + } + /// @} +}; + template class ArrayRef; class DemandedBits; class GetElementPtrInst; diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp --- a/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/llvm/lib/Analysis/LazyCallGraph.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" @@ -146,8 +147,11 @@ static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) { LibFunc LF; - // Either this is a normal library function or a "vectorizable" function. - return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName()); + // Either this is a normal library function or a "vectorizable" + // function. Not using the VFDatabase here because this query + // is related only to libraries handled via the TLI. + return TLI.getLibFunc(F, LF) || + TLI.isKnownVectorFunctionInLibrary(F.getName()); } LazyCallGraph::LazyCallGraph( diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1845,7 +1845,7 @@ // If the function has an explicit vectorized counterpart, we can safely // assume that it can be vectorized. if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && - TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) + !VFDatabase::getMappings(*Call).empty()) continue; auto *Ld = dyn_cast(&I); diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1174,6 +1174,7 @@ for (auto &S : SetVector(ListAttr.begin(), ListAttr.end())) { #ifndef NDEBUG + LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n"); Optional Info = VFABI::tryDemangleForVFABI(S); assert(Info.hasValue() && "Invalid name for a VFABI variant."); assert(CI.getModule()->getFunction(Info.getValue().VectorName) && diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp --- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -13,8 +13,11 @@ #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/DemandedBits.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/InstIterator.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" @@ -168,6 +171,11 @@ AU.setPreservesCFG(); AU.addRequired(); AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp --- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp +++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp @@ -11,15 +11,17 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" - using namespace llvm; +#define DEBUG_TYPE "moduleutils" + static void appendToGlobalArray(const char *Array, Module &M, Function *F, int Priority, Constant *Data) { IRBuilder<> IRB(M.getContext()); @@ -298,8 +300,9 @@ Module *M = CI->getModule(); #ifndef NDEBUG for (const std::string &VariantMapping : VariantMappings) { + LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n"); Optional VI = VFABI::tryDemangleForVFABI(VariantMapping); - assert(VI.hasValue() && "Canno add an invalid VFABI name."); + assert(VI.hasValue() && "Cannot add an invalid VFABI name."); assert(M->getNamedValue(VI.getValue().VectorName) && "Cannot add variant to attribute: " "vector function declaration is missing."); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -568,6 +568,28 @@ return false; } +/// Checks if a function is scalarizable according to the TLI, in +/// the sense that it should be vectorized and then expanded in +/// multiple scalarcalls. This is represented in the +/// TLI via mappings that do not specify a vector name, as in the +/// following example: +/// +/// const VecDesc VecIntrinsics[] = { +/// {"llvm.phx.abs.i32", "", 4} +/// }; +static bool isTLIScalarize(const TargetLibraryInfo &TLI, const CallInst &CI) { + const StringRef ScalarName = CI.getCalledFunction()->getName(); + bool Scalarize = TLI.isFunctionVectorizable(ScalarName); + // Check that all known VFs are not associated to a vector + // function, i.e. the vector name is emty. + if (Scalarize) + for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); + VF <= WidestVF; VF *= 2) { + Scalarize &= !TLI.isFunctionVectorizable(ScalarName, VF); + } + return Scalarize; +} + bool LoopVectorizationLegality::canVectorizeInstrs() { BasicBlock *Header = TheLoop->getHeader(); @@ -669,10 +691,12 @@ // * Have a mapping to an IR intrinsic. // * Have a vector version available. auto *CI = dyn_cast(&I); + if (CI && !getVectorIntrinsicIDForCall(CI, TLI) && !isa(CI) && !(CI->getCalledFunction() && TLI && - TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) { + (!VFDatabase::getMappings(*CI).empty() || + isTLIScalarize(*TLI, *CI)))) { // If the call is a recognized math libary call, it is likely that // we can vectorize it given loosened floating-point constraints. LibFunc Func; @@ -687,7 +711,8 @@ // but it's hard to provide meaningful yet generic advice. // Also, should this be guarded by allowExtraAnalysis() and/or be part // of the returned info from isFunctionVectorizable()? - reportVectorizationFailure("Found a non-intrinsic callsite", + reportVectorizationFailure( + "Found a non-intrinsic callsite", "library call cannot be vectorized. " "Try compiling with -fno-math-errno, -ffast-math, " "or similar flags", diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -134,6 +134,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/LoopVersioning.h" @@ -1640,6 +1641,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); // We currently do not preserve loopinfo/dominator analyses with outer loop // vectorization. Until this is addressed, mark these analyses as preserved @@ -3258,7 +3260,6 @@ unsigned VF, bool &NeedToScalarize) { Function *F = CI->getCalledFunction(); - StringRef FnName = CI->getCalledFunction()->getName(); Type *ScalarRetTy = CI->getType(); SmallVector Tys, ScalarTys; for (auto &ArgOp : CI->arg_operands()) @@ -3286,7 +3287,7 @@ // If we can't emit a vector call for this function, then the currently found // cost is the cost we need to return. NeedToScalarize = true; - if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin()) + if (!TLI || CI->isNoBuiltin() || VFDatabase::getMappings(*CI).empty()) return Cost; // If the corresponding vector cost is cheaper, return its cost. @@ -4339,9 +4340,6 @@ Module *M = I.getParent()->getParent()->getParent(); auto *CI = cast(&I); - StringRef FnName = CI->getCalledFunction()->getName(); - Function *F = CI->getCalledFunction(); - Type *RetTy = ToVectorTy(CI->getType(), VF); SmallVector Tys; for (Value *ArgOperand : CI->arg_operands()) Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); @@ -4377,17 +4375,18 @@ TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF); VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); } else { - // Use vector version of the library call. - StringRef VFnName = TLI->getVectorizedFunction(FnName, VF); - assert(!VFnName.empty() && "Vector function name is empty."); - VectorF = M->getFunction(VFnName); - if (!VectorF) { - // Generate a declaration - FunctionType *FTy = FunctionType::get(RetTy, Tys, false); - VectorF = - Function::Create(FTy, Function::ExternalLinkage, VFnName, M); - VectorF->copyAttributesFrom(F); - } + // Use vector version of the function call. + const VFShape Shape = + VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/); +#ifndef NDEBUG + const SmallVector Infos = VFDatabase::getMappings(*CI); + assert(std::find_if(Infos.begin(), Infos.end(), + [&Shape](const VFInfo &Info) { + return Info.Shape == Shape; + }) != Infos.end() && + "Vector function shape is missing from the database."); +#endif + VectorF = VFDatabase(*CI).getVectorizedFunction(Shape); } assert(VectorF && "Can't create vector function."); @@ -6397,6 +6396,7 @@ INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy) INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false) namespace llvm { diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -225,9 +225,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -230,9 +230,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -212,9 +212,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll @@ -107,3 +107,6 @@ exit: ret void } + + + diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -13,7 +13,7 @@ using namespace llvm; -// This test makes sure that the getFromVFABI method succeeds only on +// This test makes sure that the demangling method succeeds only on // valid values of the string. TEST(VectorFunctionABITests, OnlyValidNames) { // Incomplete string. @@ -91,8 +91,8 @@ unsigned &VF = Info.Shape.VF; VFISAKind &ISA = Info.ISA; SmallVector &Parameters = Info.Shape.Parameters; - StringRef &ScalarName = Info.ScalarName; - StringRef &VectorName = Info.VectorName; + std::string &ScalarName = Info.ScalarName; + std::string &VectorName = Info.VectorName; bool &IsScalable = Info.Shape.IsScalable; // Invoke the parser. bool invokeParser(const StringRef MangledName) { @@ -243,6 +243,12 @@ EXPECT_EQ(ISA, VFISAKind::AVX512); } +TEST_F(VFABIParserTest, LLVM_ISA) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); +} + TEST_F(VFABIParserTest, InvalidMask) { EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); } @@ -525,3 +531,15 @@ EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); EXPECT_EQ(ISA, VFISAKind::LLVM); } + +TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)")); + EXPECT_EQ(VF, (unsigned)4); + EXPECT_FALSE(IsMasked()); + EXPECT_FALSE(IsScalable); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); +}