Index: lib/Target/NVPTX/CMakeLists.txt =================================================================== --- lib/Target/NVPTX/CMakeLists.txt +++ lib/Target/NVPTX/CMakeLists.txt @@ -20,7 +20,7 @@ NVPTXImageOptimizer.cpp NVPTXInstrInfo.cpp NVPTXLowerAggrCopies.cpp - NVPTXLowerStructArgs.cpp + NVPTXLowerKernelArgs.cpp NVPTXMCExpr.cpp NVPTXPrologEpilogPass.cpp NVPTXRegisterInfo.cpp Index: lib/Target/NVPTX/NVPTX.h =================================================================== --- lib/Target/NVPTX/NVPTX.h +++ lib/Target/NVPTX/NVPTX.h @@ -69,7 +69,7 @@ MachineFunctionPass *createNVPTXPrologEpilogPass(); MachineFunctionPass *createNVPTXReplaceImageHandlesPass(); FunctionPass *createNVPTXImageOptimizerPass(); -FunctionPass *createNVPTXLowerStructArgsPass(); +FunctionPass *createNVPTXLowerKernelArgsPass(); bool isImageOrSamplerVal(const Value *, const Module *); Index: lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp =================================================================== --- /dev/null +++ lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp @@ -0,0 +1,197 @@ +//===-- NVPTXLowerKernelArgs.cpp - Lower kernel arguments -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Pointer arguments to kernel functions need to be lowered specially. +// +// 1. Copy byval struct args to local memory. This is a preparation for handling +// cases like +// +// kernel void foo(struct A arg, ...) +// { +// struct A *p = &arg; +// ... +// ... = p->filed1 ... (this is no generic address for .param) +// p->filed2 = ... (this is no write access to .param) +// } +// +// 2. Convert non-byval pointer arguments of CUDA kernels to pointers in the +// global address space. This allows later optimizations to emit +// ld.global.*/st.global.* for accessing these pointer arguments. For +// example, +// +// define void @foo(float* %input) { +// %v = load float, float* %input, align 4 +// ... +// } +// +// becomes +// +// define void @foo(float* %input) { +// %input2 = bitcast float* %input to i8* +// %cvt = call i8 addrspace(1)* @llvm.nvvm.ptr.gen.to.global.p1i8.p0i8(i8* %input2) +// %input3 = bitcast i8 addrspace(1)* %cvt to float addrspace(1)* +// %input4 = addrspacecast float addrspace(1)* %input3 to float* +// %v = load float, float* %input4, align 4 +// ... +// } +// +// Later, NVPTXFavorNonGenericAddrSpaces will optimize +// +// %input4 = addrspacecast float addrspace(1)* %input3 to float* +// %v = load float, float* %input4, align 4 +// +// to +// +// %v = load float, float addrspace(1)* %input3, align 4 +//===----------------------------------------------------------------------===// + +#include "NVPTX.h" +#include "NVPTXUtilities.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/Pass.h" + +using namespace llvm; + +namespace llvm { +void initializeNVPTXLowerKernelArgsPass(PassRegistry &); +} + +namespace { +class NVPTXLowerKernelArgs : public FunctionPass { + bool runOnFunction(Function &F) override; + + // handle byval parameters + void handleByValParam(Argument *); + // handle non-byval pointer parameters + void handlePointerParam(Argument *); + // a helper function to convert a pointer argument in generic to a pointer in + // the target address space. + Value *convertToPointerInAddrSpace(Value *Arg, enum AddressSpace TargetAS, + Instruction *InsertPt); + +public: + static char ID; // Pass identification, replacement for typeid + NVPTXLowerKernelArgs() : FunctionPass(ID) {} + const char *getPassName() const override { + return "Lower pointer arguments of CUDA kernels"; + } +}; +} // namespace + +char NVPTXLowerKernelArgs::ID = 1; + +INITIALIZE_PASS(NVPTXLowerKernelArgs, "nvptx-lower-kernel-args", + "Lower kernel arguments (NVPTX)", false, false) + +// ============================================================================= +// If the function had a byval struct ptr arg, say foo(%struct.x *byval %d), +// then add the following instructions to the first basic block: +// +// %temp = alloca %struct.x, align 8 +// %tt1 = bitcast %struct.x * %d to i8 * +// %tt2 = llvm.nvvm.cvt.gen.to.param %tt2 +// %tempd = bitcast i8 addrspace(101) * to %struct.x addrspace(101) * +// %tv = load %struct.x addrspace(101) * %tempd +// store %struct.x %tv, %struct.x * %temp, align 8 +// +// The above code allocates some space in the stack and copies the incoming +// struct from param space to local space. +// Then replace all occurences of %d by %temp. +// ============================================================================= +void NVPTXLowerKernelArgs::handleByValParam(Argument *Arg) { + Function *Func = Arg->getParent(); + Instruction *FirstInst = &(Func->getEntryBlock().front()); + PointerType *PType = dyn_cast(Arg->getType()); + + assert(PType && "Expecting pointer type in handleByValParam"); + + Type *StructType = PType->getElementType(); + AllocaInst *AllocA = new AllocaInst(StructType, Arg->getName(), FirstInst); + // Set the alignment to alignment of the byval parameter. This is because, + // later load/stores assume that alignment, and we are going to replace + // the use of the byval parameter with this alloca instruction. + AllocA->setAlignment(Func->getParamAlignment(Arg->getArgNo() + 1)); + Arg->replaceAllUsesWith(AllocA); + + Value *ArgInParam = + convertToPointerInAddrSpace(Arg, ADDRESS_SPACE_PARAM, FirstInst); + LoadInst *LI = new LoadInst(ArgInParam, Arg->getName(), FirstInst); + new StoreInst(LI, AllocA, FirstInst); +} + +Value *NVPTXLowerKernelArgs::convertToPointerInAddrSpace( + Value *Arg, enum AddressSpace TargetAS, Instruction *InsertPt) { + // Get the llvm.nvvm.ptr.gen.to. intrinsic. + Type *Int8PtrInTargetAS = Type::getInt8PtrTy(Arg->getContext(), TargetAS); + Type *Int8PtrInGenericAS = + Type::getInt8PtrTy(Arg->getContext(), ADDRESS_SPACE_GENERIC); + Type *CvtTypes[] = {Int8PtrInTargetAS, Int8PtrInGenericAS}; + Function *CvtFunc = Intrinsic::getDeclaration( + InsertPt->getModule(), Intrinsic::nvvm_ptr_gen_to_param, CvtTypes); + + // llvm.nvvm.ptr.gen.to.* converts i8* only. Therefore, we bitcast the + // argument to i8*, call the conversion function, and bitcast the result back + // to the original type in the target address space. + Value *ArgInI8Star = + new BitCastInst(Arg, Int8PtrInGenericAS, Arg->getName(), InsertPt); + CallInst *CallCVT = CallInst::Create(CvtFunc, ArgInI8Star, "cvt", InsertPt); + return new BitCastInst( + CallCVT, + PointerType::get(Arg->getType()->getPointerElementType(), TargetAS), + Arg->getName(), InsertPt); +} + +void NVPTXLowerKernelArgs::handlePointerParam(Argument *Arg) { + assert(!Arg->hasByValAttr() && + "byval params should be handled by handleByValParam"); + + Instruction *FirstInst = Arg->getParent()->getEntryBlock().begin(); + // Collect the original uses of Arg. + SmallVector UsesOfArg; + for (Use &U : Arg->uses()) + UsesOfArg.push_back(&U); + + Value *ArgInGlobal = + convertToPointerInAddrSpace(Arg, ADDRESS_SPACE_GLOBAL, FirstInst); + Value *ArgInGeneric = new AddrSpaceCastInst(ArgInGlobal, Arg->getType(), + Arg->getName(), FirstInst); + + // Replace all the original uses, avoiding modifying Arg's user added in + // convertToPointerInAddrSpace. + for (Use *U : UsesOfArg) + U->set(ArgInGeneric); +} + + +// ============================================================================= +// Main function for this pass. +// ============================================================================= +bool NVPTXLowerKernelArgs::runOnFunction(Function &F) { + // Skip non-kernels. See the comments at the top of this file. + if (!isKernelFunction(F)) + return false; + + for (Argument &Arg : F.args()) { + if (Arg.getType()->isPointerTy()) { + if (Arg.hasByValAttr()) + handleByValParam(&Arg); + else + handlePointerParam(&Arg); + } + } + return true; +} + +FunctionPass *llvm::createNVPTXLowerKernelArgsPass() { + return new NVPTXLowerKernelArgs(); +} Index: lib/Target/NVPTX/NVPTXLowerStructArgs.cpp =================================================================== --- lib/Target/NVPTX/NVPTXLowerStructArgs.cpp +++ /dev/null @@ -1,136 +0,0 @@ -//===-- NVPTXLowerStructArgs.cpp - Copy struct args to local memory =====--===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Copy struct args to local memory. This is needed for kernel functions only. -// This is a preparation for handling cases like -// -// kernel void foo(struct A arg, ...) -// { -// struct A *p = &arg; -// ... -// ... = p->filed1 ... (this is no generic address for .param) -// p->filed2 = ... (this is no write access to .param) -// } -// -//===----------------------------------------------------------------------===// - -#include "NVPTX.h" -#include "NVPTXUtilities.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Type.h" -#include "llvm/Pass.h" - -using namespace llvm; - -namespace llvm { -void initializeNVPTXLowerStructArgsPass(PassRegistry &); -} - -namespace { -class NVPTXLowerStructArgs : public FunctionPass { - bool runOnFunction(Function &F) override; - - void handleStructPtrArgs(Function &); - void handleParam(Argument *); - -public: - static char ID; // Pass identification, replacement for typeid - NVPTXLowerStructArgs() : FunctionPass(ID) {} - const char *getPassName() const override { - return "Copy structure (byval *) arguments to stack"; - } -}; -} // namespace - -char NVPTXLowerStructArgs::ID = 1; - -INITIALIZE_PASS(NVPTXLowerStructArgs, "nvptx-lower-struct-args", - "Lower structure arguments (NVPTX)", false, false) - -void NVPTXLowerStructArgs::handleParam(Argument *Arg) { - Function *Func = Arg->getParent(); - Instruction *FirstInst = &(Func->getEntryBlock().front()); - PointerType *PType = dyn_cast(Arg->getType()); - - assert(PType && "Expecting pointer type in handleParam"); - - Type *StructType = PType->getElementType(); - AllocaInst *AllocA = new AllocaInst(StructType, Arg->getName(), FirstInst); - - /* Set the alignment to alignment of the byval parameter. This is because, - * later load/stores assume that alignment, and we are going to replace - * the use of the byval parameter with this alloca instruction. - */ - AllocA->setAlignment(Func->getParamAlignment(Arg->getArgNo() + 1)); - - Arg->replaceAllUsesWith(AllocA); - - // Get the cvt.gen.to.param intrinsic - Type *CvtTypes[] = { - Type::getInt8PtrTy(Func->getParent()->getContext(), ADDRESS_SPACE_PARAM), - Type::getInt8PtrTy(Func->getParent()->getContext(), - ADDRESS_SPACE_GENERIC)}; - Function *CvtFunc = Intrinsic::getDeclaration( - Func->getParent(), Intrinsic::nvvm_ptr_gen_to_param, CvtTypes); - - Value *BitcastArgs[] = { - new BitCastInst(Arg, Type::getInt8PtrTy(Func->getParent()->getContext(), - ADDRESS_SPACE_GENERIC), - Arg->getName(), FirstInst)}; - CallInst *CallCVT = - CallInst::Create(CvtFunc, BitcastArgs, "cvt_to_param", FirstInst); - - BitCastInst *BitCast = new BitCastInst( - CallCVT, PointerType::get(StructType, ADDRESS_SPACE_PARAM), - Arg->getName(), FirstInst); - LoadInst *LI = new LoadInst(BitCast, Arg->getName(), FirstInst); - new StoreInst(LI, AllocA, FirstInst); -} - -// ============================================================================= -// If the function had a struct ptr arg, say foo(%struct.x *byval %d), then -// add the following instructions to the first basic block : -// -// %temp = alloca %struct.x, align 8 -// %tt1 = bitcast %struct.x * %d to i8 * -// %tt2 = llvm.nvvm.cvt.gen.to.param %tt2 -// %tempd = bitcast i8 addrspace(101) * to %struct.x addrspace(101) * -// %tv = load %struct.x addrspace(101) * %tempd -// store %struct.x %tv, %struct.x * %temp, align 8 -// -// The above code allocates some space in the stack and copies the incoming -// struct from param space to local space. -// Then replace all occurences of %d by %temp. -// ============================================================================= -void NVPTXLowerStructArgs::handleStructPtrArgs(Function &F) { - for (Argument &Arg : F.args()) { - if (Arg.getType()->isPointerTy() && Arg.hasByValAttr()) { - handleParam(&Arg); - } - } -} - -// ============================================================================= -// Main function for this pass. -// ============================================================================= -bool NVPTXLowerStructArgs::runOnFunction(Function &F) { - // Skip non-kernels. See the comments at the top of this file. - if (!isKernelFunction(F)) - return false; - - handleStructPtrArgs(F); - return true; -} - -FunctionPass *llvm::createNVPTXLowerStructArgsPass() { - return new NVPTXLowerStructArgs(); -} Index: lib/Target/NVPTX/NVPTXTargetMachine.cpp =================================================================== --- lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -53,7 +53,7 @@ void initializeNVPTXAllocaHoistingPass(PassRegistry &); void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&); void initializeNVPTXFavorNonGenericAddrSpacesPass(PassRegistry &); -void initializeNVPTXLowerStructArgsPass(PassRegistry &); +void initializeNVPTXLowerKernelArgsPass(PassRegistry &); } extern "C" void LLVMInitializeNVPTXTarget() { @@ -69,7 +69,7 @@ initializeNVPTXAssignValidGlobalNamesPass(*PassRegistry::getPassRegistry()); initializeNVPTXFavorNonGenericAddrSpacesPass( *PassRegistry::getPassRegistry()); - initializeNVPTXLowerStructArgsPass(*PassRegistry::getPassRegistry()); + initializeNVPTXLowerKernelArgsPass(*PassRegistry::getPassRegistry()); } static std::string computeDataLayout(bool is64Bit) { Index: test/CodeGen/NVPTX/bug21465.ll =================================================================== --- test/CodeGen/NVPTX/bug21465.ll +++ test/CodeGen/NVPTX/bug21465.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -nvptx-lower-struct-args -S | FileCheck %s +; RUN: opt < %s -nvptx-lower-kernel-args -S | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64" target triple = "nvptx64-unknown-unknown" @@ -8,7 +8,7 @@ ; Function Attrs: nounwind define void @_Z11TakesStruct1SPi(%struct.S* byval nocapture readonly %input, i32* nocapture %output) #0 { entry: -; CHECK-LABEL @_Z22TakesStruct1SPi +; CHECK-LABEL: @_Z11TakesStruct1SPi ; CHECK: bitcast %struct.S* %input to i8* ; CHECK: call i8 addrspace(101)* @llvm.nvvm.ptr.gen.to.param.p101i8.p0i8 %b = getelementptr inbounds %struct.S, %struct.S* %input, i64 0, i32 1 Index: test/CodeGen/NVPTX/lower-kernel-ptr-arg.ll =================================================================== --- /dev/null +++ test/CodeGen/NVPTX/lower-kernel-ptr-arg.ll @@ -0,0 +1,24 @@ +; RUN: opt < %s -nvptx-lower-kernel-args -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64" +target triple = "nvptx64-unknown-unknown" + +; Verify that both %input and %output are converted to global pointers and then +; addrspacecast'ed back to the original type. +define void @kernel(float* %input, float* %output) { +; CHECK-LABEL: @kernel( +; CHECK-NEXT: bitcast float* %output to i8* +; CHECK-NEXT: call i8 addrspace(1)* @llvm.nvvm.ptr.gen.to.param.p1i8.p0i8 +; CHECK-NEXT: bitcast i8 addrspace(1)* {{[^ ]+}} to float addrspace(1)* +; CHECK-NEXT: addrspacecast float addrspace(1)* {{[^ ]+}} to float* +; CHECK-NEXT: bitcast float* %input to i8* +; CHECK-NEXT: call i8 addrspace(1)* @llvm.nvvm.ptr.gen.to.param.p1i8.p0i8 +; CHECK-NEXT: bitcast i8 addrspace(1)* {{[^ ]+}} to float addrspace(1)* +; CHECK-NEXT: addrspacecast float addrspace(1)* {{[^ ]+}} to float* + %1 = load float, float* %input, align 4 + store float %1, float* %output, align 4 + ret void +} + +!nvvm.annotations = !{!0} +!0 = !{void (float*, float*)* @kernel, !"kernel", i32 1}