diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h @@ -0,0 +1,27 @@ +//===- InjectTLIMAppings.h - TODO brief description ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H +#define LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H + +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { +class InjectTLIMappingsPass : public PassInfoMixin { + const TargetLibraryInfo &TLI; + +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} + +#endif // LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H diff --git a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h --- a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -109,10 +109,8 @@ /// unique identifier for this module, so we return the empty string. std::string getUniqueModuleId(Module *M); -class TargetLibraryInfo; class CallInst; namespace VFABI { - /// \defgroup Vector Function ABI (VABI) Module functions. /// /// Utility functions for VFABI data that can modify the module. diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -23,6 +23,7 @@ GuardUtils.cpp InlineFunction.cpp ImportedFunctionsInliningStatistics.cpp + InjectTLIMappings.cpp InstructionNamer.cpp IntegerDivision.cpp LCSSA.cpp diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -0,0 +1,145 @@ +//===- InjectTLIMappings.cpp - TODO description --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// TODO +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/InjectTLIMappings.h" +#include "llvm/Analysis/VectorUtils.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +#include "llvm/ADT/Statistic.h" + +using namespace llvm; + +#define DEBUG_TYPE "inject-TLI-mappings" + +STATISTIC(NumInjections, + "Number of calls in which the mappings have been injected."); + +/// Helper function to map the TLI name to a strings that holds +/// scalar-to-vector mapping. +/// +/// _ZGV_() +/// +/// where: +/// +/// = "_LLVM_TLI_" +/// = "N". Note: TLI does not support masked interfaces. +/// = Number of concurrent lanes, stored in the `VectorizationFactor` +/// field of the `VecDesc` struct. +/// = "v", as many as are the number of parameters of CI. +/// = the name of the scalar function called by CI. +/// = the name of the vector function mapped by the TLI. +static std::string mangleTLIName(StringRef VectorName, CallInst *CI, + unsigned VF) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << _LLVM_VF_ << "N" << VF; + for (unsigned I = 0; I < CI->getNumArgOperands(); ++I) + Out << "v"; + Out << "_" << CI->getCalledFunction()->getName() << "(" << VectorName << ")"; + return Out.str(); +} + +/// A helper function for converting Scalar types to vector types. +/// If the incoming type is void, we return void. If the VF is 1, we return +/// the scalar type. +static Type *ToVectorTy(Type *Scalar, unsigned VF) { + if (Scalar->isVoidTy() || VF == 1) + return Scalar; + return VectorType::get(Scalar, VF); +} + +/// A helper function that adds the vector function declaration that +/// vectorizes the CallInst CI with a vectorization factor of VF +/// lanes. The TLI assumes that all parameters and the return type of +/// CI (other than void) need to be widened to a VectorType of VF +/// lanes. +static void addVariantDeclaration(CallInst *CI, const unsigned VF, + const StringRef VFName) { + assert(CI && "Invalid CallInst."); + Module *M = CI->getParent()->getParent()->getParent(); + llvm::GlobalValue *Global = M->getNamedValue(VFName); + // Nothing to do if the function already exists in the module. + if (Global) + return; + + Type *RetTy = ToVectorTy(CI->getType(), VF); + SmallVector Tys; + for (Value *ArgOperand : CI->arg_operands()) + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); + FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); + Function *VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFName, M); + VectorF->copyAttributesFrom(CI->getCalledFunction()); + + // Make function declaration (without a body) "sticky" in the IR by + // listing them in the @llvm.compiler.used attribute + if (VectorF->size() == 0) { + Global = M->getNamedValue(VFName); + assert(Global && "Missing function declaration."); + appendToCompilerUsed(*M, {Global}); + } +} + +static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst *CI, + bool &Changed) { + assert(CI && "Invalid CallInst."); + + // This is needed to make sure we don't query the TLI for calls to + // bitcast of function pointers, like `%call = call i32 (i32*, ...) + // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, + // as such calls make the `isFunctionVectorizable` raise an + // exception. + if (CI->isNoBuiltin() || !CI->getCalledFunction()) + return; + + const std::string ScalarName = CI->getCalledFunction()->getName(); + // Nothing to be done if the TLI things the function is not + // vectorizable. + if (!TLI.isFunctionVectorizable(ScalarName)) + return; + SmallVector SetOfMangledNames; + VFABI::getVectorVariantNames(*CI, SetOfMangledNames); + Module *M = CI->getParent()->getParent()->getParent(); + + for (unsigned VF = 2; VF <= 16; VF *= 2) { + const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); + if (!TLIName.empty()) { + std::string MangledName = mangleTLIName(TLIName, CI, VF); + SetOfMangledNames.push_back(MangledName); + Function *VariantF = M->getFunction(TLIName); + if (!VariantF) + addVariantDeclaration(CI, VF, TLIName); + } + } + + VFABI::setVectorVariantNames(CI, SetOfMangledNames); +} + +//////////////////////////////////////////////////////////////////////////////// +// Pass declaration and initialization +//////////////////////////////////////////////////////////////////////////////// + +namespace llvm { +PreservedAnalyses InjectTLIMappingsPass::run(Function &F, + FunctionAnalysisManager &AM) { + bool Changed = false; + for (auto &BB : F) + for (auto &I : BB) { + if (auto CI = dyn_cast(&I)) + addMappingsFromTLI(TLI, CI, Changed); + } + + if (!Changed) + return PreservedAnalyses::all(); + + return PreservedAnalyses::none(); +} +} diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll @@ -0,0 +1,33 @@ +; RUN: opt -vector-library=SVML -inject-TLI-mappings -S < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @sin_f64(double* nocapture %varray) { +; CHECK-LABEL: @sin_f64( +; CHECK: call double @sin(double %{{.*}}) #[[N:[0-9]+]] +; CHECK: attributes #[[N]] = { "vector-function-abi-variant"= +; CHECK-SAME: "_ZGV_LLVM_TLI_N2v_sin(__svml_sin2), +; CHECK-SAME: _ZGV_LLVM_TLI_N4v_sin(__svml_sin4), +; CHECK-SAME: _ZGV_LLVM_TLI_N8v_sin(__svml_sin8)" } +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @sin(double %conv) + %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv + store double %call, double* %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare double @sin(double) #0 + +attributes #0 = { nounwind readnone }