diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -180,6 +180,7 @@ void initializeIndirectBrExpandPassPass(PassRegistry&); void initializeInferAddressSpacesPass(PassRegistry&); void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); +void initializeInjectTLIMappingsLegacyPass(PassRegistry &); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstCountPass(PassRegistry&); void initializeInstNamerPass(PassRegistry&); diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h @@ -0,0 +1,40 @@ +//===- InjectTLIMAppings.h - TODO brief description ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H +#define LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { +class InjectTLIMappings : public PassInfoMixin { + +public: + static StringRef name() { return "InjectTLIMappings"; } + explicit InjectTLIMappings() {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +// Legacy pass +class InjectTLIMappingsLegacy : public FunctionPass { +public: + static char ID; + + InjectTLIMappingsLegacy() : FunctionPass(ID) { + initializeInjectTLIMappingsLegacyPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &F) override; +}; + +} // End namespace llvm +#endif // LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H diff --git a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h --- a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -109,10 +109,8 @@ /// unique identifier for this module, so we return the empty string. std::string getUniqueModuleId(Module *M); -class TargetLibraryInfo; class CallInst; namespace VFABI { - /// \defgroup Vector Function ABI (VABI) Module functions. /// /// Utility functions for VFABI data that can modify the module. diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -23,6 +23,7 @@ GuardUtils.cpp InlineFunction.cpp ImportedFunctionsInliningStatistics.cpp + InjectTLIMappings.cpp InstructionNamer.cpp IntegerDivision.cpp LCSSA.cpp diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -0,0 +1,163 @@ +//===- InjectTLIMappings.cpp - TODO description --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// TODO +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/InjectTLIMappings.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/VectorUtils.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "inject-TLI-mappings" + +STATISTIC(NumCallInjected, + "Number of calls in which the mappings have been injected."); + +STATISTIC(NumVFDefAdded, + "Number of function definitions/declarations that have been added."); +STATISTIC(NumCompUsedAdded, + "Number of `@llvm.compiler.used` operands that have been added."); + +/// Helper function to map the TLI name to a strings that holds +/// scalar-to-vector mapping. +/// +/// _ZGV_() +/// +/// where: +/// +/// = "_LLVM_" +/// = "N". Note: TLI does not support masked interfaces. +/// = Number of concurrent lanes, stored in the `VectorizationFactor` +/// field of the `VecDesc` struct. +/// = "v", as many as are the number of parameters of CI. +/// = the name of the scalar function called by CI. +/// = the name of the vector function mapped by the TLI. +static std::string mangleTLIName(StringRef VectorName, const CallInst &CI, + unsigned VF) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF; + for (unsigned I = 0; I < CI.getNumArgOperands(); ++I) + Out << "v"; + Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")"; + return Out.str(); +} + +/// A helper function for converting Scalar types to vector types. +/// If the incoming type is void, we return void. If the VF is 1, we return +/// the scalar type. +static Type *ToVectorTy(Type *Scalar, unsigned VF) { + if (Scalar->isVoidTy() || VF == 1) + return Scalar; + return VectorType::get(Scalar, VF); +} + +/// A helper function that adds the vector function declaration that +/// vectorizes the CallInst CI with a vectorization factor of VF +/// lanes. The TLI assumes that all parameters and the return type of +/// CI (other than void) need to be widened to a VectorType of VF +/// lanes. +static void addVariantDeclaration(CallInst &CI, const unsigned VF, + const StringRef VFName) { + Module *M = CI.getModule(); + llvm::GlobalValue *Global = M->getNamedValue(VFName); + // Nothing to do if the function already exists in the module. + if (Global) + return; + + Type *RetTy = ToVectorTy(CI.getType(), VF); + SmallVector Tys; + for (Value *ArgOperand : CI.arg_operands()) + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); + FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); + Function *VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFName, M); + VectorF->copyAttributesFrom(CI.getCalledFunction()); + ++NumVFDefAdded; + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName + << "` of type " << *(VectorF->getType())); + + // Make function declaration (without a body) "sticky" in the IR by + // listing them in the @llvm.compiler.used intrinsic. + if (VectorF->size() == 0) { + Global = M->getNamedValue(VFName); + assert(Global && "Missing function declaration."); + appendToCompilerUsed(*M, {Global}); + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName + << "` to `@llvm.compiler.used`."); + ++NumCompUsedAdded; + } +} + +static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { + // This is needed to make sure we don't query the TLI for calls to + // bitcast of function pointers, like `%call = call i32 (i32*, ...) + // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, + // as such calls make the `isFunctionVectorizable` raise an + // exception. + if (CI.isNoBuiltin() || !CI.getCalledFunction()) + return; + + const std::string ScalarName = CI.getCalledFunction()->getName(); + // Nothing to be done if the TLI things the function is not + // vectorizable. + if (!TLI.isFunctionVectorizable(ScalarName)) + return; + SmallVector Mappings; + VFABI::getVectorVariantNames(CI, Mappings); + Module *M = CI.getModule(); + const auto OriginalSetOfMappings = + SetVector(Mappings.begin(), Mappings.end()); + // 16 is the max number of lanes the TLI has in its VecDesc + // listings. All VFs are powers of 2. + for (unsigned VF = 2; VF <= 16; VF *= 2) { + const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); + if (!TLIName.empty()) { + std::string MangledName = mangleTLIName(TLIName, CI, VF); + if (OriginalSetOfMappings.count(MangledName) == 0) { + Mappings.push_back(MangledName); + ++NumCallInjected; + } + Function *VariantF = M->getFunction(TLIName); + if (!VariantF) + addVariantDeclaration(CI, VF, TLIName); + } + } + + VFABI::setVectorVariantNames(&CI, Mappings); +} + +namespace llvm { +PreservedAnalyses InjectTLIMappings::run(Function &F, + FunctionAnalysisManager &AM) { + const TargetLibraryInfo &TLI = AM.getResult(F); + + for (auto &BB : F) + for (auto &I : BB) { + if (auto CI = dyn_cast(&I)) + addMappingsFromTLI(TLI, *CI); + } + + // Even if the pass adds IR attributes, the analyses are preserved. + return PreservedAnalyses::all(); +} +} // End namespace llvm + +//////////////////////////////////////////////////////////////////////////////// +// Legacy Pass manager initialization +//////////////////////////////////////////////////////////////////////////////// +char InjectTLIMappingsLegacy::ID = 0; + +INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, + "Inject TLI Mappings", false, false) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", + false, false) diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll @@ -0,0 +1,20 @@ +; RUN: opt -vector-library=SVML -inject-TLI-mappings -S < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define double @sin_f64(double %in) { +; CHECK-LABEL: @sin_f64( +; CHECK: call double @sin(double %{{.*}}) #[[N:[0-9]+]] + %call = tail call double @sin(double %in) + ret double %call +} + +declare double @sin(double) #0 + +attributes #0 = { nounwind readnone } + +; CHECK: attributes #[[N]] = { "vector-function-abi-variant"= +; CHECK-SAME: "_ZGV_LLVM_TLI_N2v_sin(__svml_sin2), +; CHECK-SAME: _ZGV_LLVM_TLI_N4v_sin(__svml_sin4), +; CHECK-SAME: _ZGV_LLVM_TLI_N8v_sin(__svml_sin8)" }