diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -341,12 +341,16 @@ FunctionAnalysisManager::Invalidator &) { return false; } - /// Returns the largest vectorization factor used in the list of /// vector functions. unsigned getWidestVF(StringRef ScalarF) const { return Impl->getWidestVF(ScalarF); } + + /// Check if the function "F" is listed in a library known to LLVM. + bool isKnownVectorFunctionInLibrary(StringRef F) const { + return this->isFunctionVectorizable(F); + } }; /// Analysis pass providing the \c TargetLibraryInfo. diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -16,6 +16,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -116,10 +117,10 @@ /// Holds the VFShape for a specific scalar to vector function mapping. struct VFInfo { - VFShape Shape; // Classification of the vector function. - StringRef ScalarName; // Scalar Function Name. - StringRef VectorName; // Vector Function Name associated to this VFInfo. - VFISAKind ISA; // Instruction Set Architecture. + VFShape Shape; /// Classification of the vector function. + std::string ScalarName; /// Scalar Function Name. + std::string VectorName; /// Vector Function Name associated to this VFInfo. + VFISAKind ISA; /// Instruction Set Architecture. // Comparison operator. bool operator==(const VFInfo &Other) const { @@ -167,6 +168,81 @@ SmallVectorImpl &VariantMappings); } // end namespace VFABI +/// The Vector Function Database. +/// +/// Helper class used to find the vector functions associated to a +/// scalar CallInst. +class VFDatabase { + /// The CallInst for which we are looking for vector functions. + const CallInst &CI; + /// The Module of the CallInst CI. + const Module *M; + /// List of vector functions descritors associated to the call + /// instruction. + const SmallVector ScalarToVectorMappings; + + /// Retreive the scalar-to-vector mappings associated to the rule of + /// a vector Function ABI. + static void getVFABIMappings(const CallInst &CI, + SmallVectorImpl &Mappings) { + const StringRef ScalarName = CI.getCalledFunction()->getName(); + const StringRef S = + CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) + .getValueAsString(); + if (S.empty()) + return; + + SmallVector ListOfStrings; + VFABI::getVectorVariantNames(CI, ListOfStrings); + for (const auto &MangledName : ListOfStrings) { + const Optional Shape = VFABI::tryDemangleForVFABI(MangledName); + // A match is found via scalar and vector names, and also by + // ensuring that the variant described in the attribute has a + // corresponding definition or declaration of the vector + // function in the Module M. + if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) { + assert(CI.getModule()->getFunction(Shape.getValue().VectorName) && + "Vector function is missing."); + Mappings.push_back(Shape.getValue()); + } + } + } + +public: + /// Retrieve all the VFInfo instances associated to the CallInst CI. + static SmallVector getMappings(const CallInst &CI) { + SmallVector Ret; + + // Get mappings from the Vector Function ABI variants. + getVFABIMappings(CI, Ret); + + // Other non-VFABI variants should be retrieved here. + + return Ret; + } + + /// Constructor, requires a CallInst instance. + VFDatabase(CallInst &CI) + : CI(CI), M(CI.getModule()), + ScalarToVectorMappings(VFDatabase::getMappings(CI)) {} + /// \defgroup VFDatabase query interface. + /// + /// @{ + /// Retrieve the Function with VFShape \p Shape. + Function *getVectorizedFunction(const VFShape &Shape) const { + for (const auto &Info : ScalarToVectorMappings) + if (Info.Shape == Shape) + return M->getFunction(Info.VectorName); + + return nullptr; + } + /// Checks if a function is vectorizable with VFShape \p Shape. + bool isFunctionVectorizable(const VFShape &Shape) const { + return getVectorizedFunction(Shape) != nullptr; + } + /// @} +}; + template class ArrayRef; class DemandedBits; class GetElementPtrInst; diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp --- a/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/llvm/lib/Analysis/LazyCallGraph.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" @@ -146,8 +147,11 @@ static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) { LibFunc LF; - // Either this is a normal library function or a "vectorizable" function. - return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName()); + // Either this is a normal library function or a "vectorizable" + // function. Not using the VFDatabase here because this query + // is related only to libraries handled via the TLI. + return TLI.getLibFunc(F, LF) || + TLI.isKnownVectorFunctionInLibrary(F.getName()); } LazyCallGraph::LazyCallGraph( diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1845,7 +1845,7 @@ // If the function has an explicit vectorized counterpart, we can safely // assume that it can be vectorized. if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && - TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) + !VFDatabase::getMappings(*Call).empty()) continue; auto *Ld = dyn_cast(&I); diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1174,6 +1174,7 @@ for (auto &S : SetVector(ListAttr.begin(), ListAttr.end())) { #ifndef NDEBUG + LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n"); Optional Info = VFABI::tryDemangleForVFABI(S); assert(Info.hasValue() && "Invalid name for a VFABI variant."); assert(CI.getModule()->getFunction(Info.getValue().VectorName) && diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp --- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -13,6 +13,8 @@ #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/DemandedBits.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/InstIterator.h" #include "llvm/Transforms/Utils.h" @@ -168,6 +170,11 @@ AU.setPreservesCFG(); AU.addRequired(); AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp --- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp +++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp @@ -11,15 +11,17 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" - using namespace llvm; +#define DEBUG_TYPE "moduleutils" + static void appendToGlobalArray(const char *Array, Module &M, Function *F, int Priority, Constant *Data) { IRBuilder<> IRB(M.getContext()); @@ -298,8 +300,9 @@ Module *M = CI->getModule(); #ifndef NDEBUG for (const std::string &VariantMapping : VariantMappings) { + LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n"); Optional VI = VFABI::tryDemangleForVFABI(VariantMapping); - assert(VI.hasValue() && "Canno add an invalid VFABI name."); + assert(VI.hasValue() && "Cannot add an invalid VFABI name."); assert(M->getNamedValue(VI.getValue().VectorName) && "Cannot add variant to attribute: " "vector function declaration is missing."); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -670,7 +670,7 @@ if (CI && !getVectorIntrinsicIDForCall(CI, TLI) && !isa(CI) && !(CI->getCalledFunction() && TLI && - TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) { + !VFDatabase::getMappings(*CI).empty())) { // If the call is a recognized math libary call, it is likely that // we can vectorize it given loosened floating-point constraints. LibFunc Func; @@ -685,7 +685,8 @@ // but it's hard to provide meaningful yet generic advice. // Also, should this be guarded by allowExtraAnalysis() and/or be part // of the returned info from isFunctionVectorizable()? - reportVectorizationFailure("Found a non-intrinsic callsite", + reportVectorizationFailure( + "Found a non-intrinsic callsite", "library call cannot be vectorized. " "Try compiling with -fno-math-errno, -ffast-math, " "or similar flags", diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -134,6 +134,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/LoopVersioning.h" @@ -1634,6 +1635,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); // We currently do not preserve loopinfo/dominator analyses with outer loop // vectorization. Until this is addressed, mark these analyses as preserved @@ -3226,7 +3228,6 @@ unsigned VF, bool &NeedToScalarize) { Function *F = CI->getCalledFunction(); - StringRef FnName = CI->getCalledFunction()->getName(); Type *ScalarRetTy = CI->getType(); SmallVector Tys, ScalarTys; for (auto &ArgOp : CI->arg_operands()) @@ -3254,7 +3255,9 @@ // If we can't emit a vector call for this function, then the currently found // cost is the cost we need to return. NeedToScalarize = true; - if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin()) + if (!TLI || CI->isNoBuiltin() || + !VFDatabase(*CI).isFunctionVectorizable( + VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/))) return Cost; // If the corresponding vector cost is cheaper, return its cost. @@ -4279,9 +4282,6 @@ Module *M = I.getParent()->getParent()->getParent(); auto *CI = cast(&I); - StringRef FnName = CI->getCalledFunction()->getName(); - Function *F = CI->getCalledFunction(); - Type *RetTy = ToVectorTy(CI->getType(), VF); SmallVector Tys; for (Value *ArgOperand : CI->arg_operands()) Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); @@ -4317,17 +4317,18 @@ TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF); VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); } else { - // Use vector version of the library call. - StringRef VFnName = TLI->getVectorizedFunction(FnName, VF); - assert(!VFnName.empty() && "Vector function name is empty."); - VectorF = M->getFunction(VFnName); - if (!VectorF) { - // Generate a declaration - FunctionType *FTy = FunctionType::get(RetTy, Tys, false); - VectorF = - Function::Create(FTy, Function::ExternalLinkage, VFnName, M); - VectorF->copyAttributesFrom(F); - } + // Use vector version of the function call. + const VFShape Shape = + VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/); +#ifndef NDEBUG + const SmallVector Infos = VFDatabase::getMappings(*CI); + assert(std::find_if(Infos.begin(), Infos.end(), + [&Shape](const VFInfo &Info) { + return Info.Shape == Shape; + }) != Infos.end() && + "Vector function shape is missing from the database."); +#endif + VectorF = VFDatabase(*CI).getVectorizedFunction(Shape); } assert(VectorF && "Can't create vector function."); @@ -6356,6 +6357,7 @@ INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy) INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false) namespace llvm { diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -223,9 +223,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -228,9 +228,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -210,9 +210,11 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter +; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -13,7 +13,7 @@ using namespace llvm; -// This test makes sure that the getFromVFABI method succeeds only on +// This test makes sure that the demangling method succeeds only on // valid values of the string. TEST(VectorFunctionABITests, OnlyValidNames) { // Incomplete string. @@ -91,8 +91,8 @@ unsigned &VF = Info.Shape.VF; VFISAKind &ISA = Info.ISA; SmallVector &Parameters = Info.Shape.Parameters; - StringRef &ScalarName = Info.ScalarName; - StringRef &VectorName = Info.VectorName; + std::string &ScalarName = Info.ScalarName; + std::string &VectorName = Info.VectorName; bool &IsScalable = Info.Shape.IsScalable; // Invoke the parser. bool invokeParser(const StringRef MangledName) { @@ -243,6 +243,12 @@ EXPECT_EQ(ISA, VFISAKind::AVX512); } +TEST_F(VFABIParserTest, LLVM_ISA) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); +} + TEST_F(VFABIParserTest, InvalidMask) { EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); } @@ -525,3 +531,15 @@ EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); EXPECT_EQ(ISA, VFISAKind::LLVM); } + +TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)")); + EXPECT_EQ(VF, (unsigned)4); + EXPECT_FALSE(IsMasked()); + EXPECT_FALSE(IsScalable); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); +}