Index: llvm/include/llvm/Analysis/TargetLibraryInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -199,6 +199,13 @@ unsigned getWCharSize(const Module &M) const; }; +// Forward declarations needed for the `friend` declaration of +// `VFABI::addMappingsFromTLI` in `TargetLibraryInfo`. +class TargetLibraryInfo; +namespace VFABI { +void addMappingsFromTLI(const TargetLibraryInfo *TLI, CallInst *CI); +} // End VFABI namespace + /// Provides information about what library functions are available for /// the current target. /// @@ -207,6 +214,10 @@ class TargetLibraryInfo { friend class TargetLibraryAnalysis; friend class TargetLibraryInfoWrapperPass; + // `friend` is needed for the method to be able to call the methods + // `isFunctionVectorizable` and `getVectorizedFunction`. + friend void VFABI::addMappingsFromTLI(const TargetLibraryInfo *TLI, + CallInst *CI); const TargetLibraryInfoImpl *Impl; @@ -248,6 +259,8 @@ bool has(LibFunc F) const { return Impl->getState(F) != TargetLibraryInfoImpl::Unavailable; } + +private: bool isFunctionVectorizable(StringRef F, unsigned VF) const { return Impl->isFunctionVectorizable(F, VF); } @@ -258,6 +271,7 @@ return Impl->getVectorizedFunction(F, VF); } +public: /// Tests if the function is both available and a candidate for optimized code /// generation. bool hasOptimizedCodeGen(LibFunc F) const { @@ -337,6 +351,10 @@ FunctionAnalysisManager::Invalidator &) { return false; } + /// Check if the function "F" is listed in a library known to LLVM. + bool isKnownVectorFunctionInLibrary(StringRef F) const { + return this->isFunctionVectorizable(F); + } }; /// Analysis pass providing the \c TargetLibraryInfo. Index: llvm/include/llvm/Analysis/VectorUtils.h =================================================================== --- llvm/include/llvm/Analysis/VectorUtils.h +++ llvm/include/llvm/Analysis/VectorUtils.h @@ -14,7 +14,9 @@ #define LLVM_ANALYSIS_VECTORUTILS_H #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -47,9 +49,13 @@ AVX, // x86 AVX AVX2, // x86 AVX2 AVX512, // x86 AVX512 + LLVM_TLI, // Internal ABI for the function handled via + // TargetLibraryInfo Unknown // Unknown ISA }; +static constexpr char const *_LLVM_TLI_ = "_LLVM_TLI_"; + /// Encapsulates information needed to describe a parameter. /// /// The description of the parameter is not linked directly to @@ -91,8 +97,8 @@ /// Holds the VFShape for a specific scalar to vector function mapping. struct VFInfo { VFShape Shape; // Classification of the vector function. - StringRef ScalarName; // Scalar Function Name. - StringRef VectorName; // Vector Function Name associated to this VFInfo. + std::string ScalarName; // Scalar Function Name. + std::string VectorName; // Vector Function Name associated to this VFInfo. // Comparison operator. bool operator==(const VFInfo &Other) const { @@ -121,23 +127,123 @@ /// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and /// https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt /// -/// -/// /// \param MangledName -> input string in the format /// _ZGV_[()]. Optional tryDemangleForVFABI(StringRef MangledName); /// Retrieve the `VFParamKind` from a string token. VFParamKind getVFParamKindFromString(const StringRef Token); +/// Populates a set of strings representing the Vector Function ABI variants +/// associated to the CallInst CI. +void getVectorVariantNames(CallInst *CI, + SmallSet &VariantMappings); } // end namespace VFABI +class SearchVFSystem { +private: + CallInst *CI; /// The CallInst for which we are looking for vector + /// functions. + const TargetLibraryInfo *TLI; /// Optional hook to the TLI to check + /// for vector calls in external + /// libraries. + const Module *M; /// The Module of the CallInst CI. + // Extract all names listed in the Vector Function ABI variants + // attribute associated to the CallInst CI. + SmallVector extractMangledNames() const { + SmallVector ListOfStrings; + + AttributeList Attrs = CI->getAttributes(); + AttributeSet FnAttrs = Attrs.getFnAttributes(); + const StringRef AttrString = + FnAttrs.getAttribute("vector-function-abi-variant").getValueAsString(); + SmallVector List; + AttrString.split(List, ","); + + for (auto &S : List) { + ListOfStrings.push_back(S.str()); + } + return ListOfStrings; + } + + /// Retrieve all the VFInfo instances associated to the CallInst CI, + /// by looking into the Vector Function ABI Variant attribute. + SmallVector getVFMappings() const { + SmallVector Ret; + + // Get mappings from the IR attribute + const std::string ScalarName = CI->getCalledFunction()->getName(); + const SmallVector ListOfStrings = extractMangledNames(); + + // Module *M = CI->getParent()->getParent()->getParent(); + for (auto MangledName : ListOfStrings) { + auto Shape = VFABI::tryDemangleForVFABI(MangledName); + // A match is found via scalar and vector names, and also by + // ensuring that the variant described in the attribute has a + // corresponding definition or declaration in the Module M. + if (Shape.hasValue()) + if ((Shape.getValue().ScalarName == ScalarName) && + M->getFunction(Shape.getValue().VectorName)) + Ret.push_back(Shape.getValue()); + } + + return Ret; + } + +public: + SearchVFSystem() = delete; /// No default constructor, as we + /// require this class to be associated + /// to a CallInst CI. + // Constructor. + explicit SearchVFSystem(CallInst *CI) + : CI(CI), M(CI->getParent()->getParent()->getParent()) {} + /// \defgroup TLI legacy interface + /// + /// These functions are here for compatibility with the the equivalent + /// methods provided by the TLI (the TLI ones have been made + /// private). + /// + /// @{ + bool isFunctionVectorizable() const { + auto Mappings = getVFMappings(); + return !Mappings.empty(); + } + std::string getVectorizedFunction(unsigned VF) const { + const auto Mappings = getVFMappings(); + SmallVector Parameters; + for (unsigned I = 0; I < CI->arg_size(); ++I) { + Parameters.push_back(VFParameter({I, VFParamKind::Vector})); + } + // TODO: at the moment the shape is constructed by forcing the ISA + // to be onlye the one handled via the TLI. What we really shoudl + // do it to move the ISAAKind field of VFShape into VFInfo, and + // accept any of the available ISA that present the shape expected + // by the query from the vectorizer. For example, AVX and AVX2 + // have both 256-bit registers. The IR signature generated for + // those two ISA will be teh same. The `getVectorizedFunction` + // method should just return the first available match. We will + // then need to add more heuristics to the search system to decide + // which of the available version (TLI, AVX, AVX2) the system + // should choose, when some with the same VFShape are present. + const VFShape TLIShape = {VF, false /*isScalable*/, VFISAKind::LLVM_TLI, + Parameters}; + for (const auto &Info : Mappings) + if (Info.Shape == TLIShape) + return Info.VectorName; + + return ""; + } + bool isFunctionVectorizable(unsigned VF) const { + return !getVectorizedFunction(VF).empty(); + } + /// @} +}; + template class ArrayRef; class DemandedBits; class GetElementPtrInst; template class InterleaveGroup; class Loop; class ScalarEvolution; -class TargetLibraryInfo; class TargetTransformInfo; class Type; class Value; Index: llvm/include/llvm/Transforms/Utils/ModuleUtils.h =================================================================== --- llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -13,6 +13,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" #include // for std::pair @@ -108,6 +109,28 @@ /// unique identifier for this module, so we return the empty string. std::string getUniqueModuleId(Module *M); +class TargetLibraryInfo; +class CallInst; +namespace VFABI { + +/// \defgroup Vector Function ABI (VABI) Module functions. +/// +/// Utility functions for VFABI data that can modify the module. +/// +/// @{ +/// Add to the Vector Function ABI mapping attribute of CI all the +/// functions that are know to \p TLI of being some vectorized +/// version of \p CI. +void addMappingsFromTLI(const TargetLibraryInfo *TLI, CallInst *CI); + +/// Overwrite the Vector Function ABI variants attribute with the names provide +/// in \p VariantMappings. +void setVectorVariantNames(CallInst *CI, + const SmallSet &VariantMappings); + +/// @} +} // End VFABI namespace + } // End llvm namespace #endif // LLVM_TRANSFORMS_UTILS_MODULEUTILS_H Index: llvm/lib/Analysis/LazyCallGraph.cpp =================================================================== --- llvm/lib/Analysis/LazyCallGraph.cpp +++ llvm/lib/Analysis/LazyCallGraph.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" @@ -146,8 +147,11 @@ static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) { LibFunc LF; - // Either this is a normal library function or a "vectorizable" function. - return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName()); + // Either this is a normal library function or a "vectorizable" + // function. Not using the SearchVFSystem here because this query + // is related only to libraries handled via the TLI. + return TLI.getLibFunc(F, LF) || + TLI.isKnownVectorFunctionInLibrary(F.getName()); } LazyCallGraph::LazyCallGraph( Index: llvm/lib/Analysis/LoopAccessAnalysis.cpp =================================================================== --- llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1844,7 +1844,7 @@ // If the function has an explicit vectorized counterpart, we can safely // assume that it can be vectorized. if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && - TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) + SearchVFSystem(Call).isFunctionVectorizable()) continue; auto *Ld = dyn_cast(&I); @@ -2425,6 +2425,13 @@ DT = &getAnalysis().getDomTree(); LI = &getAnalysis().getLoopInfo(); + // // Update IR attribute on Vector Function variants if the TLI is present. + // if (TLI) + // for (auto &B : F) + // for (auto &I : B) + // if (auto *CI = dyn_cast(&I)) + // VFABI::addMappingsFromTLI(TLI, CI); + return false; } Index: llvm/lib/Analysis/TargetLibraryInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetLibraryInfo.cpp +++ llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -11,9 +11,13 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Triple.h" +#include "llvm/Analysis/VectorUtils.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; static cl::opt ClVectorLibrary( Index: llvm/lib/Analysis/VFABIDemangling.cpp =================================================================== --- llvm/lib/Analysis/VFABIDemangling.cpp +++ llvm/lib/Analysis/VFABIDemangling.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallString.h" #include "llvm/Analysis/VectorUtils.h" using namespace llvm; @@ -26,16 +28,20 @@ if (MangledName.empty()) return ParseRet::Error; - ISA = StringSwitch(MangledName.take_front(1)) - .Case("n", VFISAKind::AdvancedSIMD) - .Case("s", VFISAKind::SVE) - .Case("b", VFISAKind::SSE) - .Case("c", VFISAKind::AVX) - .Case("d", VFISAKind::AVX2) - .Case("e", VFISAKind::AVX512) - .Default(VFISAKind::Unknown); - - MangledName = MangledName.drop_front(1); + if (MangledName.startswith(_LLVM_TLI_)) { + MangledName = MangledName.drop_front(strlen(_LLVM_TLI_)); + ISA = VFISAKind::LLVM_TLI; + } else { + ISA = StringSwitch(MangledName.take_front(1)) + .Case("n", VFISAKind::AdvancedSIMD) + .Case("s", VFISAKind::SVE) + .Case("b", VFISAKind::SSE) + .Case("c", VFISAKind::AVX) + .Case("d", VFISAKind::AVX2) + .Case("e", VFISAKind::AVX512) + .Default(VFISAKind::Unknown); + MangledName = MangledName.drop_front(1); + } return ParseRet::OK; } @@ -286,6 +292,7 @@ // Format of the ABI name: // _ZGV_[()] Optional VFABI::tryDemangleForVFABI(StringRef MangledName) { + const StringRef OriginalName = MangledName; // Assume there is no custom name , and therefore the // vector name consists of // _ZGV_. @@ -338,7 +345,7 @@ } } while (ParamFound == ParseRet::OK); - // A valid MangledName mus have at least one valid entry in the + // A valid MangledName must have at least one valid entry in the // . if (Parameters.empty()) return None; @@ -369,6 +376,11 @@ return None; } + // LLVM internal mapping via the TargetLibraryInfo (TLI) must be + // redirected to an existing name. + if (ISA == VFISAKind::LLVM_TLI && VectorName == OriginalName) + return None; + // When is "M", we need to add a parameter that is used as // global predicate for the function. if (IsMasked) { @@ -416,3 +428,21 @@ " that have a textual representation in the mangled name" " of the Vector Function ABI"); } + +void VFABI::getVectorVariantNames(CallInst *CI, + SmallSet &VariantMappings) { + AttributeList Attrs = CI->getAttributes(); + AttributeSet FnAttrs = Attrs.getFnAttributes(); + const StringRef S = + FnAttrs.getAttribute("vector-function-abi-variant").getValueAsString(); + SmallVector ListAttr; + S.split(ListAttr, ","); + + for (auto &S : ListAttr) { + if (S.size() > 0) { + assert(VFABI::tryDemangleForVFABI(S).hasValue() && + "Invalid name for a VFABI variant."); + VariantMappings.insert(S); + } + } +} Index: llvm/lib/Transforms/Utils/ModuleUtils.cpp =================================================================== --- llvm/lib/Transforms/Utils/ModuleUtils.cpp +++ llvm/lib/Transforms/Utils/ModuleUtils.cpp @@ -11,12 +11,13 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" - using namespace llvm; static void appendToGlobalArray(const char *Array, Module &M, Function *F, @@ -280,3 +281,141 @@ MD5::stringifyResult(R, Str); return ("$" + Str).str(); } + +/// Helper function to map the TLI name to a strings that holds +/// scalar-to-vector mapping. +/// +/// _ZGV_() +/// +/// where: +/// +/// = "_LLVM_TLI_" +/// = "N". Note: TLI does not support masked interfaces. +/// = Number of concurrent lanes, stored in the `VectorizationFactor` +/// field of the `VecDesc` struct. +/// = "v", as many as are the number of parameters of CI. +/// = the name of the scalar function called by CI. +/// = the name of the vector function mapped by the TLI. +static std::string mangleTLIName(StringRef VectorName, CallInst *CI, + unsigned VF) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << _LLVM_TLI_ << "N" << VF; + for (unsigned I = 0; I < CI->getNumArgOperands(); ++I) { + Out << "v"; + } + Out << "_" << CI->getCalledFunction()->getName() << "(" << VectorName << ")"; + return Out.str(); +} + +/// A helper function for converting Scalar types to vector types. +/// If the incoming type is void, we return void. If the VF is 1, we return +/// the scalar type. +static Type *ToVectorTy(Type *Scalar, unsigned VF) { + if (Scalar->isVoidTy() || VF == 1) + return Scalar; + return VectorType::get(Scalar, VF); +} + +/// A helper function that adds the vector function declaration that +/// vectorizes the CallInst CI with a vectorization factor of VF +/// lanes. The TLI assumes that all parameters and the return type of +/// CI (other than void) need to be widened to a VectorType of VF +/// lanes. +static void addVariantDeclaration(CallInst *CI, const unsigned VF, + const StringRef VFName) { + assert(CI && "Invalid CallInst."); + Module *M = CI->getParent()->getParent()->getParent(); + llvm::GlobalValue *Global = M->getNamedValue(VFName); + // Nothing to do if the function already exists in the module. + if (Global) + return; + + Type *RetTy = ToVectorTy(CI->getType(), VF); + SmallVector Tys; + for (Value *ArgOperand : CI->arg_operands()) + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); + FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); + Function *VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFName, M); + VectorF->copyAttributesFrom(CI->getCalledFunction()); + + // Make function declaration (without a body) "sticky" in the IR by + // listing them in the @llvm.compiler.used attribute + if (VectorF->size() == 0) { + Global = M->getNamedValue(VFName); + assert(Global && "Missing function declaration."); + appendToCompilerUsed(*M, {Global}); + } +} + +void VFABI::addMappingsFromTLI(const TargetLibraryInfo *TLI, CallInst *CI) { + assert(TLI && "Invalid TLI."); + assert(CI && "Invalid CallInst."); + + // This is needed to make sure we don't query the TLI for calls to + // bitcast of function pointers, like `%call = call i32 (i32*, ...) + // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, + // as such calls make the `isFunctionVectorizable` raise an + // exception. + if (CI->isNoBuiltin() || !CI->getCalledFunction()) + return; + + const std::string ScalarName = CI->getCalledFunction()->getName(); + // Nothing to be done if the TLI things the function is not + // vectorizable. + if (!TLI->isFunctionVectorizable(ScalarName)) + return; + + SmallSet SetOfMangledNames; + VFABI::getVectorVariantNames(CI, SetOfMangledNames); + Module *M = CI->getParent()->getParent()->getParent(); + + for (unsigned VF = 2; VF <= 16; VF *= 2) { + const std::string TLIName = TLI->getVectorizedFunction(ScalarName, VF); + if (TLIName != "") { + std::string MangledName = mangleTLIName(TLIName, CI, VF); + // List.push_back(MangledName); + SetOfMangledNames.insert(MangledName); + Function *VariantF = M->getFunction(TLIName); + if (!VariantF) + addVariantDeclaration(CI, VF, TLIName); + } + } + + VFABI::setVectorVariantNames(CI, SetOfMangledNames); +} + +void VFABI::setVectorVariantNames( + CallInst *CI, const SmallSet &VariantMappings) { + assert(CI && "Invalid CallInst"); + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + if (VariantMappings.size() >= 1) { + auto I = VariantMappings.begin(); + auto E = VariantMappings.end(); + Out << *I; + ++I; + while (I != E) { + Out << "," << *I; + ++I; + } + } + Module *M = CI->getParent()->getParent()->getParent(); +#ifndef NDEBUG + for (auto I = VariantMappings.begin(), E = VariantMappings.end(); I != E; + ++I) { + Optional VI = VFABI::tryDemangleForVFABI(*I); + assert(VI.hasValue() && "Canno add an invalid VFABI name."); + assert(M->getNamedValue(VI.getValue().VectorName) && + "Cannot add variant to attribute: " + "vector function declaration is missing."); + } +#endif + std::string Value = Out.str(); + auto &C = M->getContext(); + AttributeList Attrs = CI->getAttributes(); + Attrs = Attrs.addAttribute(C, AttributeList::FunctionIndex, + "vector-function-abi-variant", Value); + CI->setAttributes(Attrs); +} Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -670,7 +670,7 @@ if (CI && !getVectorIntrinsicIDForCall(CI, TLI) && !isa(CI) && !(CI->getCalledFunction() && TLI && - TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) { + SearchVFSystem(CI).isFunctionVectorizable())) { // If the call is a recognized math libary call, it is likely that // we can vectorize it given loosened floating-point constraints. LibFunc Func; @@ -685,7 +685,8 @@ // but it's hard to provide meaningful yet generic advice. // Also, should this be guarded by allowExtraAnalysis() and/or be part // of the returned info from isFunctionVectorizable()? - reportVectorizationFailure("Found a non-intrinsic callsite", + reportVectorizationFailure( + "Found a non-intrinsic callsite", "library call cannot be vectorized. " "Try compiling with -fno-math-errno, -ffast-math, " "or similar flags", Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1611,6 +1611,15 @@ auto *ORE = &getAnalysis().getORE(); auto *PSI = &getAnalysis().getPSI(); + // Update IR attribute on Vector Function variants if the TLI is + // present. This should probably be invoked as an early stage + // pass. + if (TLI) + for (auto &B : F) + for (auto &I : B) + if (auto *CI = dyn_cast(&I)) + VFABI::addMappingsFromTLI(TLI, CI); + std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); }; @@ -3221,7 +3230,6 @@ unsigned VF, bool &NeedToScalarize) { Function *F = CI->getCalledFunction(); - StringRef FnName = CI->getCalledFunction()->getName(); Type *ScalarRetTy = CI->getType(); SmallVector Tys, ScalarTys; for (auto &ArgOp : CI->arg_operands()) @@ -3249,7 +3257,8 @@ // If we can't emit a vector call for this function, then the currently found // cost is the cost we need to return. NeedToScalarize = true; - if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin()) + if (!TLI || !SearchVFSystem(CI).isFunctionVectorizable(VF) || + CI->isNoBuiltin()) return Cost; // If the corresponding vector cost is cheaper, return its cost. @@ -4263,7 +4272,6 @@ Module *M = I.getParent()->getParent()->getParent(); auto *CI = cast(&I); - StringRef FnName = CI->getCalledFunction()->getName(); Function *F = CI->getCalledFunction(); Type *RetTy = ToVectorTy(CI->getType(), VF); SmallVector Tys; @@ -4302,7 +4310,7 @@ VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); } else { // Use vector version of the library call. - StringRef VFnName = TLI->getVectorizedFunction(FnName, VF); + StringRef VFnName = SearchVFSystem(CI).getVectorizedFunction(VF); assert(!VFnName.empty() && "Vector function name is empty."); VectorF = M->getFunction(VFnName); if (!VectorF) { Index: llvm/test/Transforms/LoopVectorize/X86/TLI-to-vfabi-attribute.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/X86/TLI-to-vfabi-attribute.ll @@ -0,0 +1,35 @@ +; RUN: opt -vector-library=SVML -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -mattr=avx -S < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @sin_f64(double* nocapture %varray) { +; CHECK-LABEL: @sin_f64( +; CHECK: [[TMP5:%.*]] = call <4 x double> @__svml_sin4(<4 x double> [[TMP4:%.*]]) +; CHECK: call double @sin(double %{{.*}}) #[[N:[0-9]+]] +; CHECK: ret void +; CHECK: attributes #[[N]] = { "vector-function-abi-variant"= +; CHECK-SAME: "_ZGV_LLVM_TLI_N2v_sin(__svml_sin2), +; CHECK-SAME: _ZGV_LLVM_TLI_N4v_sin(__svml_sin4), +; CHECK-SAME: _ZGV_LLVM_TLI_N8v_sin(__svml_sin8)" } +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @sin(double %conv) + %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv + store double %call, double* %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare double @sin(double) #0 + +attributes #0 = { nounwind readnone } Index: llvm/unittests/Analysis/VectorFunctionABITest.cpp =================================================================== --- llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -11,7 +11,7 @@ using namespace llvm; -// This test makes sure that the getFromVFABI method succeeds only on +// This test makes sure that the demangling method succeeds only on // valid values of the string. TEST(VectorFunctionABITests, OnlyValidNames) { // Incomplete string. @@ -89,8 +89,8 @@ unsigned &VF = Info.Shape.VF; VFISAKind &ISA = Info.Shape.ISA; SmallVector &Parameters = Info.Shape.Parameters; - StringRef &ScalarName = Info.ScalarName; - StringRef &VectorName = Info.VectorName; + std::string &ScalarName = Info.ScalarName; + std::string &VectorName = Info.VectorName; bool &IsScalable = Info.Shape.IsScalable; // Invoke the parser. bool invokeParser(const StringRef MangledName) { @@ -241,6 +241,12 @@ EXPECT_EQ(ISA, VFISAKind::AVX512); } +TEST_F(VFABIParserTest, LLVM_TLI) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_TLI_N2v_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_TLI_N2v_sin_(vector_name)")); + EXPECT_EQ(ISA, VFISAKind::LLVM_TLI); +} + TEST_F(VFABIParserTest, InvalidMask) { EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); } @@ -344,6 +350,13 @@ __COMMON_CHECKS; EXPECT_EQ(VectorName, "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"); + // LLVM_TLI: = "_LLVM_TLI_" + EXPECT_TRUE(invokeParser( + "_ZGV_LLVM_TLI_N2vls2Ls27Us4Rs5l1L10U100R1000u2_sin(vectorf)")); + EXPECT_EQ(ISA, VFISAKind::LLVM_TLI); + __COMMON_CHECKS; + EXPECT_EQ(VectorName, "vectorf"); + // Unknown ISA (randomly using "q"). This test will need update if // some targets decide to use "q" as their ISA token. EXPECT_TRUE(invokeParser("_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin")); @@ -437,3 +450,15 @@ EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(ScalarName, "sin"); } + +TEST_F(VFABIParserTest, Intrinsics) { + EXPECT_TRUE(invokeParser("_ZGV_LLVM_TLI_N4vv_llvm.pow.f32(__svml_powf4)")); + EXPECT_EQ(VF, (unsigned)4); + EXPECT_FALSE(IsMasked()); + EXPECT_FALSE(IsScalable); + EXPECT_EQ(ISA, VFISAKind::LLVM_TLI); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); +}