Index: polly/trunk/lib/CMakeLists.txt =================================================================== --- polly/trunk/lib/CMakeLists.txt +++ polly/trunk/lib/CMakeLists.txt @@ -107,6 +107,8 @@ LLVMipo LLVMMC LLVMPasses + LLVMLinker + LLVMIRReader ${nvptx_libs} # The libraries below are required for darwin: http://PR26392 LLVMBitReader Index: polly/trunk/lib/CodeGen/PPCGCodeGeneration.cpp =================================================================== --- polly/trunk/lib/CodeGen/PPCGCodeGeneration.cpp +++ polly/trunk/lib/CodeGen/PPCGCodeGeneration.cpp @@ -31,6 +31,8 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Verifier.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/Linker/Linker.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Target/TargetMachine.h" @@ -102,6 +104,11 @@ cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory)); +static cl::opt CUDALibDevice( + "polly-acc-libdevice", cl::desc("Path to CUDA libdevice"), cl::Hidden, + cl::init("/usr/local/cuda/nvvm/libdevice/libdevice.compute_20.10.ll"), + cl::ZeroOrMore, cl::cat(PollyCategory)); + static cl::opt CudaVersion("polly-acc-cuda-version", cl::desc("The CUDA version to compile for"), cl::Hidden, @@ -605,6 +612,12 @@ /// @param F The function to remove references to. void clearLoops(Function *F); + /// Check if the scop requires to be linked with CUDA's libdevice. + bool requiresCUDALibDevice(); + + /// Link with the NVIDIA libdevice library (if needed and available). + void addCUDALibDevice(); + /// Finalize the generation of the kernel function. /// /// Free the LLVM-IR module corresponding to the kernel and -- if requested -- @@ -1324,13 +1337,32 @@ return isl_bool_true; } +/// A list of functions that are available in NVIDIA's libdevice. +const std::set CUDALibDeviceFunctions = { + "exp", "expf", "expl", "cos", "cosf", + "sqrt", "sqrtf", "copysign", "copysignf", "copysignl"}; + +/// Return the corresponding CUDA libdevice function name for @p F. +/// +/// Return "" if we are not compiling for CUDA. +std::string getCUDALibDeviceFuntion(Function *F) { + if (CUDALibDeviceFunctions.count(F->getName())) + return std::string("__nv_") + std::string(F->getName()); + + return ""; +} + /// Check if F is a function that we can code-generate in a GPU kernel. -static bool isValidFunctionInKernel(llvm::Function *F) { +static bool isValidFunctionInKernel(llvm::Function *F, bool AllowLibDevice) { assert(F && "F is an invalid pointer"); // We string compare against the name of the function to allow // all variants of the intrinsic "llvm.sqrt.*", "llvm.fabs", and // "llvm.copysign". const StringRef Name = F->getName(); + + if (AllowLibDevice && getCUDALibDeviceFuntion(F).length() > 0) + return true; + return F->isIntrinsic() && (Name.startswith("llvm.sqrt") || Name.startswith("llvm.fabs") || Name.startswith("llvm.copysign")); @@ -1346,14 +1378,16 @@ /// Return `Function`s from `RawSubtreeValues`. static SetVector -getFunctionsFromRawSubtreeValues(SetVector RawSubtreeValues) { +getFunctionsFromRawSubtreeValues(SetVector RawSubtreeValues, + bool AllowCUDALibDevice) { SetVector SubtreeFunctions; for (Value *It : RawSubtreeValues) { Function *F = dyn_cast(It); if (F) { - assert(isValidFunctionInKernel(F) && "Code should have bailed out by " - "this point if an invalid function " - "were present in a kernel."); + assert(isValidFunctionInKernel(F, AllowCUDALibDevice) && + "Code should have bailed out by " + "this point if an invalid function " + "were present in a kernel."); SubtreeFunctions.insert(F); } } @@ -1407,8 +1441,11 @@ make_filter_range(SubtreeValues, isValidSubtreeValue); SetVector ValidSubtreeValues(ValidSubtreeValuesIt.begin(), ValidSubtreeValuesIt.end()); + + bool AllowCUDALibDevice = Arch == GPUArch::NVPTX64; + SetVector ValidSubtreeFunctions( - getFunctionsFromRawSubtreeValues(SubtreeValues)); + getFunctionsFromRawSubtreeValues(SubtreeValues, AllowCUDALibDevice)); // @see IslNodeBuilder::getReferencesInSubtree SetVector ReplacedValues; @@ -2232,6 +2269,49 @@ return ASMStream.str(); } +bool GPUNodeBuilder::requiresCUDALibDevice() { + for (Function &F : GPUModule->functions()) { + if (!F.isDeclaration()) + continue; + + std::string CUDALibDeviceFunc = getCUDALibDeviceFuntion(&F); + if (CUDALibDeviceFunc.length() != 0) { + F.setName(CUDALibDeviceFunc); + return true; + } + } + + return false; +} + +void GPUNodeBuilder::addCUDALibDevice() { + if (Arch != GPUArch::NVPTX64) + return; + + if (requiresCUDALibDevice()) { + SMDiagnostic Error; + + errs() << CUDALibDevice << "\n"; + auto LibDeviceModule = + parseIRFile(CUDALibDevice, Error, GPUModule->getContext()); + + if (!LibDeviceModule) { + BuildSuccessful = false; + report_fatal_error("Could not find or load libdevice. Skipping GPU " + "kernel generation. Please set -polly-acc-libdevice " + "accordingly.\n"); + return; + } + + Linker L(*GPUModule); + + // Set an nvptx64 target triple to avoid linker warnings. The original + // triple of the libdevice files are nvptx-unknown-unknown. + LibDeviceModule->setTargetTriple(Triple::normalize("nvptx64-nvidia-cuda")); + L.linkInModule(std::move(LibDeviceModule), Linker::LinkOnlyNeeded); + } +} + std::string GPUNodeBuilder::finalizeKernelFunction() { if (verifyModule(*GPUModule)) { @@ -2247,6 +2327,8 @@ return ""; } + addCUDALibDevice(); + if (DumpKernelIR) outs() << *GPUModule << "\n"; @@ -3116,10 +3198,12 @@ /// /// If this basic block does something with a `Function` other than calling /// a function that we support in a kernel, return true. - bool containsInvalidKernelFunctionInBlock(const BasicBlock *BB) { + bool containsInvalidKernelFunctionInBlock(const BasicBlock *BB, + bool AllowCUDALibDevice) { for (const Instruction &Inst : *BB) { const CallInst *Call = dyn_cast(&Inst); - if (Call && isValidFunctionInKernel(Call->getCalledFunction())) { + if (Call && isValidFunctionInKernel(Call->getCalledFunction(), + AllowCUDALibDevice)) { continue; } @@ -3135,16 +3219,17 @@ } /// Return whether the Scop S uses functions in a way that we do not support. - bool containsInvalidKernelFunction(const Scop &S) { + bool containsInvalidKernelFunction(const Scop &S, bool AllowCUDALibDevice) { for (auto &Stmt : S) { if (Stmt.isBlockStmt()) { - if (containsInvalidKernelFunctionInBlock(Stmt.getBasicBlock())) + if (containsInvalidKernelFunctionInBlock(Stmt.getBasicBlock(), + AllowCUDALibDevice)) return true; } else { assert(Stmt.isRegionStmt() && "Stmt was neither block nor region statement"); for (const BasicBlock *BB : Stmt.getRegion()->blocks()) - if (containsInvalidKernelFunctionInBlock(BB)) + if (containsInvalidKernelFunctionInBlock(BB, AllowCUDALibDevice)) return true; } } @@ -3232,7 +3317,8 @@ // kernel. This may lead to a kernel trying to call a function on the host. // This also allows us to prevent codegen from trying to take the // address of an intrinsic function to send to the kernel. - if (containsInvalidKernelFunction(CurrentScop)) { + if (containsInvalidKernelFunction(CurrentScop, + Architecture == GPUArch::NVPTX64)) { DEBUG( dbgs() << "Scop contains function which cannot be materialised in a GPU " Index: polly/trunk/test/GPGPU/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll =================================================================== --- polly/trunk/test/GPGPU/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll +++ polly/trunk/test/GPGPU/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll @@ -0,0 +1,3 @@ +define float @__nv_expf(float %a) { + ret float %a +} Index: polly/trunk/test/GPGPU/libdevice-functions-copied-into-kernel.ll =================================================================== --- polly/trunk/test/GPGPU/libdevice-functions-copied-into-kernel.ll +++ polly/trunk/test/GPGPU/libdevice-functions-copied-into-kernel.ll @@ -0,0 +1,74 @@ +; RUN: opt %loadPolly -analyze -polly-scops < %s \ +; RUN: -polly-acc-libdevice=%S/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll \ +; RUN: | FileCheck %s --check-prefix=SCOP +; RUN: opt %loadPolly -analyze -polly-codegen-ppcg -polly-acc-dump-kernel-ir \ +; RUN: -polly-acc-libdevice=%S/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll \ +; RUN: < %s | FileCheck %s --check-prefix=KERNEL-IR +; RUN: opt %loadPolly -S -polly-codegen-ppcg < %s \ +; RUN: -polly-acc-libdevice=%S/Inputs/libdevice-functions-copied-into-kernel_libdevice.ll \ +; RUN: | FileCheck %s --check-prefix=HOST-IR + +; Test that we do recognise and codegen a kernel that has functions that can +; be mapped to NVIDIA's libdevice + +; REQUIRES: pollyacc + +; Check that we model the kernel as a scop. +; SCOP: Function: f +; SCOP-NEXT: Region: %entry.split---%for.end + +; Check that the intrinsic call is present in the kernel IR. +; KERNEL-IR: %p_expf = tail call float @__nv_expf(float %A.arr.i.val_p_scalar_) + +; Check that kernel launch is generated in host IR. +; the declare would not be generated unless a call to a kernel exists. +; HOST-IR: declare void @polly_launchKernel(i8*, i32, i32, i32, i32, i32, i8*) + + +; void f(float *A, float *B, int N) { +; for(int i = 0; i < N; i++) { +; float tmp0 = A[i]; +; float tmp1 = expf(tmp1); +; B[i] = tmp1; +; } +; } + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" + +define void @f(float* %A, float* %B, i32 %N) { +entry: + br label %entry.split + +entry.split: ; preds = %entry + %cmp1 = icmp sgt i32 %N, 0 + br i1 %cmp1, label %for.body.lr.ph, label %for.end + +for.body.lr.ph: ; preds = %entry.split + br label %for.body + +for.body: ; preds = %for.body.lr.ph, %for.body + %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] + %A.arr.i = getelementptr inbounds float, float* %A, i64 %indvars.iv + %A.arr.i.val = load float, float* %A.arr.i, align 4 + ; Call to intrinsics that should be part of the kernel. + %expf = tail call float @expf(float %A.arr.i.val) + %B.arr.i = getelementptr inbounds float, float* %B, i64 %indvars.iv + store float %expf, float* %B.arr.i, align 4 + + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %wide.trip.count = zext i32 %N to i64 + %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count + br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge + +for.cond.for.end_crit_edge: ; preds = %for.body + br label %for.end + +for.end: ; preds = %for.cond.for.end_crit_edge, %entry.split + ret void +} + +; Function Attrs: nounwind readnone +declare float @expf(float) #0 + +attributes #0 = { nounwind readnone } + Index: polly/trunk/test/lit.site.cfg.in =================================================================== --- polly/trunk/test/lit.site.cfg.in +++ polly/trunk/test/lit.site.cfg.in @@ -31,6 +31,11 @@ key, = e.args lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key)) +# excludes: A list of directories to exclude from the testsuite. The 'Inputs' +# subdirectories contain auxiliary inputs for various tests in their parent +# directories. +config.excludes = ['Inputs'] + if config.link_polly_into_tools == '' or \ config.link_polly_into_tools.lower() == '0' or \ config.link_polly_into_tools.lower() == 'n' or \