Index: llvm/include/llvm/Analysis/TargetLibraryInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -197,8 +197,13 @@ /// Returns the size of the wchar_t type in bytes or 0 if the size is unknown. /// This queries the 'wchar_size' metadata. unsigned getWCharSize(const Module &M) const; + + void fixUpVFABINames(CallInst *CI) const; }; +// Forward declaration for friend use. +class SearchVFSystem; + /// Provides information about what library functions are available for /// the current target. /// @@ -207,7 +212,7 @@ class TargetLibraryInfo { friend class TargetLibraryAnalysis; friend class TargetLibraryInfoWrapperPass; - + friend class SearchVFSystem; const TargetLibraryInfoImpl *Impl; public: @@ -248,6 +253,8 @@ bool has(LibFunc F) const { return Impl->getState(F) != TargetLibraryInfoImpl::Unavailable; } + +private: bool isFunctionVectorizable(StringRef F, unsigned VF) const { return Impl->isFunctionVectorizable(F, VF); } @@ -258,6 +265,7 @@ return Impl->getVectorizedFunction(F, VF); } +public: /// Tests if the function is both available and a candidate for optimized code /// generation. bool hasOptimizedCodeGen(LibFunc F) const { @@ -337,6 +345,14 @@ FunctionAnalysisManager::Invalidator &) { return false; } + /// Check if the function "F" is listed in a library known to LLVM. + bool isKnownVectorFunctionInLibrary(StringRef F) const { + return this->isFunctionVectorizable(F); + } + /// Add to the Vector Function ABI mapping attribute of CI all the + /// functions that are know to the TLI of being a vectorized version + /// of CI. + void fixUpVFABINames(CallInst *CI) const { Impl->fixUpVFABINames(CI); } }; /// Analysis pass providing the \c TargetLibraryInfo. Index: llvm/include/llvm/Analysis/VectorUtils.h =================================================================== --- llvm/include/llvm/Analysis/VectorUtils.h +++ llvm/include/llvm/Analysis/VectorUtils.h @@ -14,7 +14,9 @@ #define LLVM_ANALYSIS_VECTORUTILS_H #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -41,15 +43,19 @@ /// Describes the type of Instruction Set Architecture enum class VFISAKind { - AdvancedSIMD, // AArch64 Advanced SIMD (NEON) - SVE, // AArch64 Scalable Vector Extension - SSE, // x86 SSE - AVX, // x86 AVX - AVX2, // x86 AVX2 - AVX512, // x86 AVX512 - Unknown // Unknown ISA + AdvancedSIMD, // AArch64 Advanced SIMD (NEON) + SVE, // AArch64 Scalable Vector Extension + SSE, // x86 SSE + AVX, // x86 AVX + AVX2, // x86 AVX2 + AVX512, // x86 AVX512 + LLVM_INTERNAL_TLI, // Internal ABI for the function handled via + // TargetLibraryInfo + Unknown // Unknown ISA }; +#define _LLVM_INTERNAL_TLI_ "_LLVM_INTERNAL_TLI_" + /// Encapsulates information needed to describe a parameter. /// /// The description of the parameter is not linked directly to @@ -91,8 +97,8 @@ /// Holds the VFShape for a specific scalar to vector function mapping. struct VFInfo { VFShape Shape; // Classification of the vector function. - StringRef ScalarName; // Scalar Function Name. - StringRef VectorName; // Vector Function Name associated to this VFInfo. + std::string ScalarName; // Scalar Function Name. + std::string VectorName; // Vector Function Name associated to this VFInfo. // Comparison operator. bool operator==(const VFInfo &Other) const { @@ -121,23 +127,125 @@ /// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and /// https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt /// -/// -/// /// \param MangledName -> input string in the format /// _ZGV_[()]. -Optional tryDemangleForVFABI(StringRef MangledName); +Optional tryDemangleForVFABI(const std::string Input); /// Retrieve the `VFParamKind` from a string token. VFParamKind getVFParamKindFromString(const StringRef Token); +/// Returns a set of strings representing the Vector Function ABI variants +/// associated to the CallInst CI. +SmallSet getVectorVariantNames(CallInst *CI); +/// Overwrite the Vector Function ABI variants attribute with the names provide +/// in VariantMappings +void setVectorVariantNames(CallInst *CI, + const SmallSet VariantMappings); + } // end namespace VFABI +class SearchVFSystem { +private: + CallInst *CI; /// The CallInst for which we are looking for vector + /// functions. + const TargetLibraryInfo *TLI; /// Optional hook to the TLI to check + /// for vector calls in external + /// libraries. + const Module *M; /// The Module of the CallInst CI. + // Extract all names listed in the Vector Function ABI variants + // attribute associated to the CallInst CI. + SmallVector extractMangledNames() const { + SmallVector ListOfStrings; + + AttributeList Attrs = CI->getAttributes(); + AttributeSet FnAttrs = Attrs.getFnAttributes(); + const StringRef AttrString = + FnAttrs.getAttribute("vector-function-abi-variant").getValueAsString(); + SmallVector List; + AttrString.split(List, ","); + + for (auto &S : List) { + ListOfStrings.push_back(S.str()); + } + return ListOfStrings; + } + + /// Retrieve all the VFInfo instances associated to the CallInst CI, + /// by looking into the Vector Function ABI Variant attribute. + SmallVector getVFMappings() const { + SmallVector Ret; + + // Get mappings from the IR attribute + const std::string ScalarName = CI->getCalledFunction()->getName(); + const SmallVector ListOfStrings = extractMangledNames(); + + // Module *M = CI->getParent()->getParent()->getParent(); + for (auto MangledName : ListOfStrings) { + auto Shape = VFABI::tryDemangleForVFABI(MangledName); + // A match is found via scalar and vector names, and also by + // ensuring that the variant described in the attribute has a + // corresponding definition or declaration in the Module M. + if (Shape.hasValue()) + if ((Shape.getValue().ScalarName == ScalarName) && + M->getFunction(Shape.getValue().VectorName)) + Ret.push_back(Shape.getValue()); + } + + return Ret; + } + +public: + SearchVFSystem() = delete; /// No default constructor, as we + /// require this class to be associated + /// to a CallInst CI. + explicit SearchVFSystem(CallInst *CI, const TargetLibraryInfo *TLI = nullptr) + : CI(CI), TLI(TLI), M(CI->getParent()->getParent()->getParent()) { + // If TLI is present, add to the list also the symbols that the + // TLI provides. + const StringRef ScalarName = CI->getCalledFunction()->getName(); + if (TLI && TLI->isFunctionVectorizable(ScalarName)) { + TLI->fixUpVFABINames(CI); + } + } + /// \defgroup TLI legacy interface + /// + /// These functions are here for compatibility with the the equivalent + /// methods provided by the TLI (the TLI ones have been made + /// private). + /// + /// @{ + bool isFunctionVectorizable() const { + auto Mappings = getVFMappings(); + if (!Mappings.empty()) + return true; + + return false; + } + std::string getVectorizedFunction(unsigned VF) const { + const auto Mappings = getVFMappings(); + SmallVector Parameters; + for (unsigned I = 0; I < CI->arg_size(); ++I) { + Parameters.push_back(VFParameter({I, VFParamKind::Vector})); + } + const VFShape TLIShape = {VF, false /*isScalable*/, + VFISAKind::LLVM_INTERNAL_TLI, Parameters}; + for (const auto &Info : Mappings) + if (Info.Shape == TLIShape) + return Info.VectorName; + + return ""; + } + bool isFunctionVectorizable(unsigned VF) const { + return !getVectorizedFunction(VF).empty(); + } + /// @} +}; + template class ArrayRef; class DemandedBits; class GetElementPtrInst; template class InterleaveGroup; class Loop; class ScalarEvolution; -class TargetLibraryInfo; class TargetTransformInfo; class Type; class Value; Index: llvm/lib/Analysis/LazyCallGraph.cpp =================================================================== --- llvm/lib/Analysis/LazyCallGraph.cpp +++ llvm/lib/Analysis/LazyCallGraph.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" @@ -146,8 +147,11 @@ static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) { LibFunc LF; - // Either this is a normal library function or a "vectorizable" function. - return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName()); + // Either this is a normal library function or a "vectorizable" + // function. Not using the SearchVFSystem here because this query + // is related only to libraries handled via the TLI. + return TLI.getLibFunc(F, LF) || + TLI.isKnownVectorFunctionInLibrary(F.getName()); } LazyCallGraph::LazyCallGraph( Index: llvm/lib/Analysis/LoopAccessAnalysis.cpp =================================================================== --- llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1844,7 +1844,7 @@ // If the function has an explicit vectorized counterpart, we can safely // assume that it can be vectorized. if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && - TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) + SearchVFSystem(Call, TLI).isFunctionVectorizable()) continue; auto *Ld = dyn_cast(&I); Index: llvm/lib/Analysis/TargetLibraryInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetLibraryInfo.cpp +++ llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -11,9 +11,13 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Triple.h" +#include "llvm/Analysis/VectorUtils.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; static cl::opt ClVectorLibrary( @@ -1636,3 +1640,77 @@ char TargetLibraryInfoWrapperPass::ID = 0; void TargetLibraryInfoWrapperPass::anchor() {} + +/// Helper function to map the TLI name to a strings that holds +/// scalar-to-vector mapping. +/// +/// _ZGV_() +/// +/// where: +/// +/// = "_LLVM_INTERNAL_TLI_" +/// = "N". Note: TLI does not support masked interfaces. +/// = Number of concurrent lanes, stored in the `VectorizationFactor` +/// field of the `VecDesc` struct. +/// = "v", as many as are the number of parameters of CI. +/// = the name of the scalar function called by CI. +/// = the name of the vector function mapped by the TLI. +static std::string mangleTLIName(StringRef VectorName, CallInst *CI, + unsigned VF) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << _LLVM_INTERNAL_TLI_ << "N" << VF; + for (unsigned I = 0; I < CI->getNumArgOperands(); ++I) { + Out << "v"; + } + Out << "_" << CI->getCalledFunction()->getName() << "(" << VectorName << ")"; + return Out.str(); +} + +/// A helper function for converting Scalar types to vector types. +/// If the incoming type is void, we return void. If the VF is 1, we return +/// the scalar type. +static Type *ToVectorTy(Type *Scalar, unsigned VF) { + if (Scalar->isVoidTy() || VF == 1) + return Scalar; + return VectorType::get(Scalar, VF); +} + +/// A helper function that adds the vector function declaration that +/// vectorizes the CallInst CI with a vectorization factor of VF +/// lanes. The TLI assumes that all parameters and the return type of +/// CI (other than void) need to be widened to a VectorType of VF +/// lanes. +static void addVariantDeclaration(CallInst *CI, const unsigned VF, + const StringRef VFName) { + Module *M = CI->getParent()->getParent()->getParent(); + Type *RetTy = ToVectorTy(CI->getType(), VF); + SmallVector Tys; + for (Value *ArgOperand : CI->arg_operands()) + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); + FunctionType *FTy = FunctionType::get(RetTy, Tys, false); + + Function *VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFName, M); + VectorF->copyAttributesFrom(CI->getCalledFunction()); +} + +void TargetLibraryInfoImpl::fixUpVFABINames(CallInst *CI) const { + SmallSet SetOfMangledNames = VFABI::getVectorVariantNames(CI); + Module *M = CI->getParent()->getParent()->getParent(); + + const std::string ScalarName = CI->getCalledFunction()->getName(); + for (unsigned VF = 2; VF <= 16; VF *= 2) { + const std::string TLIName = this->getVectorizedFunction(ScalarName, VF); + if (TLIName != "") { + std::string MangledName = mangleTLIName(TLIName, CI, VF); + // List.push_back(MangledName); + SetOfMangledNames.insert(MangledName); + Function *VariantF = M->getFunction(TLIName); + if (!VariantF) + addVariantDeclaration(CI, VF, TLIName); + } + } + + VFABI::setVectorVariantNames(CI, SetOfMangledNames); +} Index: llvm/lib/Analysis/VFABIDemangling.cpp =================================================================== --- llvm/lib/Analysis/VFABIDemangling.cpp +++ llvm/lib/Analysis/VFABIDemangling.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallString.h" #include "llvm/Analysis/VectorUtils.h" using namespace llvm; @@ -26,16 +28,20 @@ if (MangledName.empty()) return ParseRet::Error; - ISA = StringSwitch(MangledName.take_front(1)) - .Case("n", VFISAKind::AdvancedSIMD) - .Case("s", VFISAKind::SVE) - .Case("b", VFISAKind::SSE) - .Case("c", VFISAKind::AVX) - .Case("d", VFISAKind::AVX2) - .Case("e", VFISAKind::AVX512) - .Default(VFISAKind::Unknown); - - MangledName = MangledName.drop_front(1); + if (MangledName.startswith(_LLVM_INTERNAL_TLI_)) { + MangledName = MangledName.drop_front(strlen(_LLVM_INTERNAL_TLI_)); + ISA = VFISAKind::LLVM_INTERNAL_TLI; + } else { + ISA = StringSwitch(MangledName.take_front(1)) + .Case("n", VFISAKind::AdvancedSIMD) + .Case("s", VFISAKind::SVE) + .Case("b", VFISAKind::SSE) + .Case("c", VFISAKind::AVX) + .Case("d", VFISAKind::AVX2) + .Case("e", VFISAKind::AVX512) + .Default(VFISAKind::Unknown); + MangledName = MangledName.drop_front(1); + } return ParseRet::OK; } @@ -285,7 +291,9 @@ // Format of the ABI name: // _ZGV_[()] -Optional VFABI::tryDemangleForVFABI(StringRef MangledName) { +Optional VFABI::tryDemangleForVFABI(const std::string Input) { + StringRef MangledName = Input; + const StringRef OriginalName = MangledName; // Assume there is no custom name , and therefore the // vector name consists of // _ZGV_. @@ -338,7 +346,7 @@ } } while (ParamFound == ParseRet::OK); - // A valid MangledName mus have at least one valid entry in the + // A valid MangledName must have at least one valid entry in the // . if (Parameters.empty()) return None; @@ -369,6 +377,11 @@ return None; } + // LLVM internal mapping via the TargetLibraryInfo (TLI) must be + // redirected to an existing name. + if (ISA == VFISAKind::LLVM_INTERNAL_TLI && VectorName == OriginalName) + return None; + // When is "M", we need to add a parameter that is used as // global predicate for the function. if (IsMasked) { @@ -416,3 +429,48 @@ " that have a textual representation in the mangled name" " of the Vector Function ABI"); } + +SmallSet VFABI::getVectorVariantNames(CallInst *CI) { + AttributeList Attrs = CI->getAttributes(); + AttributeSet FnAttrs = Attrs.getFnAttributes(); + const StringRef S = + FnAttrs.getAttribute("vector-function-abi-variant").getValueAsString(); + SmallVector ListAttr; + S.split(ListAttr, ","); + + SmallSet VariantMappings; + for (auto &S : ListAttr) { + // const std::string S = Tmp.str(); + if (S.size() > 0) { + assert(VFABI::tryDemangleForVFABI(S).hasValue() && + "Invalid name for a VFABI variant."); + VariantMappings.insert(S); + } + } + return VariantMappings; +} + +void VFABI::setVectorVariantNames( + CallInst *CI, const SmallSet VariantMappings) { + + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + auto I = VariantMappings.begin(); + auto E = VariantMappings.end(); + if (VariantMappings.size() >= 1) { + Out << *I; + ++I; + while (I != E) { + Out << "," << *I; + ++I; + } + } + std::string Value = Out.str(); + + Module *M = CI->getParent()->getParent()->getParent(); + auto &C = M->getContext(); + AttributeList Attrs = CI->getAttributes(); + Attrs = Attrs.addAttribute(C, AttributeList::FunctionIndex, + "vector-function-abi-variant", Value); + CI->setAttributes(Attrs); +} Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -670,7 +670,7 @@ if (CI && !getVectorIntrinsicIDForCall(CI, TLI) && !isa(CI) && !(CI->getCalledFunction() && TLI && - TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) { + SearchVFSystem(CI, TLI).isFunctionVectorizable())) { // If the call is a recognized math libary call, it is likely that // we can vectorize it given loosened floating-point constraints. LibFunc Func; @@ -685,7 +685,8 @@ // but it's hard to provide meaningful yet generic advice. // Also, should this be guarded by allowExtraAnalysis() and/or be part // of the returned info from isFunctionVectorizable()? - reportVectorizationFailure("Found a non-intrinsic callsite", + reportVectorizationFailure( + "Found a non-intrinsic callsite", "library call cannot be vectorized. " "Try compiling with -fno-math-errno, -ffast-math, " "or similar flags", Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3221,7 +3221,6 @@ unsigned VF, bool &NeedToScalarize) { Function *F = CI->getCalledFunction(); - StringRef FnName = CI->getCalledFunction()->getName(); Type *ScalarRetTy = CI->getType(); SmallVector Tys, ScalarTys; for (auto &ArgOp : CI->arg_operands()) @@ -3249,7 +3248,8 @@ // If we can't emit a vector call for this function, then the currently found // cost is the cost we need to return. NeedToScalarize = true; - if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin()) + if (!TLI || !SearchVFSystem(CI, TLI).isFunctionVectorizable(VF) || + CI->isNoBuiltin()) return Cost; // If the corresponding vector cost is cheaper, return its cost. @@ -4263,7 +4263,6 @@ Module *M = I.getParent()->getParent()->getParent(); auto *CI = cast(&I); - StringRef FnName = CI->getCalledFunction()->getName(); Function *F = CI->getCalledFunction(); Type *RetTy = ToVectorTy(CI->getType(), VF); SmallVector Tys; @@ -4302,7 +4301,7 @@ VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); } else { // Use vector version of the library call. - StringRef VFnName = TLI->getVectorizedFunction(FnName, VF); + StringRef VFnName = SearchVFSystem(CI, TLI).getVectorizedFunction(VF); assert(!VFnName.empty() && "Vector function name is empty."); VectorF = M->getFunction(VFnName); if (!VectorF) { Index: llvm/unittests/Analysis/VectorFunctionABITest.cpp =================================================================== --- llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -11,7 +11,7 @@ using namespace llvm; -// This test makes sure that the getFromVFABI method succeeds only on +// This test makes sure that the demangling method succeeds only on // valid values of the string. TEST(VectorFunctionABITests, OnlyValidNames) { // Incomplete string. @@ -89,8 +89,8 @@ unsigned &VF = Info.Shape.VF; VFISAKind &ISA = Info.Shape.ISA; SmallVector &Parameters = Info.Shape.Parameters; - StringRef &ScalarName = Info.ScalarName; - StringRef &VectorName = Info.VectorName; + std::string &ScalarName = Info.ScalarName; + std::string &VectorName = Info.VectorName; bool &IsScalable = Info.Shape.IsScalable; // Invoke the parser. bool invokeParser(const StringRef MangledName) { @@ -241,6 +241,12 @@ EXPECT_EQ(ISA, VFISAKind::AVX512); } +TEST_F(VFABIParserTest, LLVM_INTERNAL_TLI) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_INTERNAL_TLI_N2v_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_INTERNAL_TLI_N2v_sin_(vector_name)")); + EXPECT_EQ(ISA, VFISAKind::LLVM_INTERNAL_TLI); +} + TEST_F(VFABIParserTest, InvalidMask) { EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); } @@ -344,6 +350,13 @@ __COMMON_CHECKS; EXPECT_EQ(VectorName, "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"); + // LLVM_INTERNAL_TLI: = "_LLVM_INTERNAL_TLI_" + EXPECT_TRUE(invokeParser( + "_ZGV_LLVM_INTERNAL_TLI_N2vls2Ls27Us4Rs5l1L10U100R1000u2_sin(vectorf)")); + EXPECT_EQ(ISA, VFISAKind::LLVM_INTERNAL_TLI); + __COMMON_CHECKS; + EXPECT_EQ(VectorName, "vectorf"); + // Unknown ISA (randomly using "q"). This test will need update if // some targets decide to use "q" as their ISA token. EXPECT_TRUE(invokeParser("_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin")); @@ -437,3 +450,16 @@ EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(ScalarName, "sin"); } + +TEST_F(VFABIParserTest, Intrinsics) { + EXPECT_TRUE( + invokeParser("_ZGV_LLVM_INTERNAL_TLI_N4vv_llvm.pow.f32(__svml_powf4)")); + EXPECT_EQ(VF, (unsigned)4); + EXPECT_FALSE(IsMasked()); + EXPECT_FALSE(IsScalable); + EXPECT_EQ(ISA, VFISAKind::LLVM_INTERNAL_TLI); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); +}