diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -226,6 +226,9 @@ if (LangOpts.OpenMPIRBuilder) { OMPBuilder.reset(new llvm::OpenMPIRBuilder(TheModule)); OMPBuilder->initialize(); + OMPBuilder->setIntPtrTy(IntPtrTy); + OMPBuilder->setSizeTy(SizeTy); + OMPBuilder->setVoidPtrTy(VoidPtrTy); } } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -95,6 +95,19 @@ ///{ namespace types { +/// Target specific types. + +extern Type *IntPtrTy; +extern Type *SizeTy; + +extern Type *VoidPtr; + +extern Type *Void2Ptr; +extern Type *Int8PtrPtr; + +extern Type *Void3Ptr; +extern Type *Int8PtrPtrPtr; + #define OMP_TYPE(VarName, InitValue) extern Type *VarName; #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ extern ArrayType *VarName##Ty; \ @@ -110,6 +123,11 @@ /// Helper to initialize all types defined in OpenMP/OMPKinds.def. void initializeTypes(Module &M); +/// setters for target specific types +void initializeIntPtrTy(Type *IntPtrTy); +void initializeSizeTy(Type *SizeTy); +void initializeVoidPtrTy(Type *VoidPtrTy); + /// Helper to uninitialize all types defined in OpenMP/OMPKinds.def. void uninitializeTypes(); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -35,6 +35,11 @@ /// before any other method and only once! void initialize(); + /// setters for target specific types + void setIntPtrTy(IntegerType *IntPtrTy); + void setSizeTy(IntegerType *SizeTy); + void setVoidPtrTy(PointerType *VoidPtrTy); + /// Finalize the underlying module, e.g., by outlining regions. void finalize(); @@ -44,6 +49,23 @@ /// Type used throughout for insertion points. using InsertPointTy = IRBuilder<>::InsertPoint; + /// Restores insertion point to \p IP + void restoreIP(InsertPointTy IP) { Builder.restoreIP(IP); } + + /// Set Insert point to BasicBlock /p IPBB , and Iterator /p IP. + void setInsertPoint(BasicBlock *IPBB, BasicBlock::iterator IP) { + Builder.SetInsertPoint(IPBB, IP); + } + + /// Set Insert point to BasicBlock /p IPBB + void setInsertPoint(BasicBlock *IPBB) { Builder.SetInsertPoint(IPBB); } + + /// Set Insert point to Instruction /p IPII + void setInsertPoint(Instruction *IPII) { Builder.SetInsertPoint(IPII); } + + /// return a copy of the current insertion point information + InsertPointTy SaveIP() { return Builder.saveIP(); } + /// Callback type for variable finalization (think destructors). /// /// \param CodeGenIP is the insertion point at which the finalization code @@ -299,6 +321,11 @@ StringMap, BumpPtrAllocator> InternalVars; public: + /// Return the current thread ID. + /// + /// \param Loc The insert and source location description. + Value *getOrCreateThreadID(const LocationDescription &Loc); + /// Generator for '#omp master' /// /// \param Loc The insert and source location description. @@ -324,6 +351,62 @@ FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst); + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param IP insertion block for copyin conditional + /// \param MasterVarPtr a pointer to the master variable + /// \param PrivateVarPtr a pointer to the threadprivate variable + /// \param IntPtrTy Pointer size type + /// + /// \returns The insertion point where copying operation to be emitted. + InsertPointTy CreateCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, + Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, + bool BranchtoEnd = true); + + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param Loc The insert and source location description. + /// \param Size Size of allocated memory space + /// \param Allocator Allocator information instruction + /// \param name Name of call Instruction for OMP_alloc + /// + /// \returns CallInst to the OMP_Alloc call + CallInst *CreateOMPAlloc(const LocationDescription &Loc, Value *Size, + Value *Allocator, std::string name = ""); + + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param Loc The insert and source location description. + /// \param Addr Address of memory space to be freed + /// \param Allocator Allocator information instruction + /// \param name Name of call Instruction for OMP_Free + /// + /// \returns CallInst to the OMP_Free call + CallInst *CreateOMPFree(const LocationDescription &Loc, Value *Addr, + Value *Allocator, std::string name = ""); + + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param Loc The insert and source location description. + /// \param Pointer pointer to data to be cached + /// \param Size size of data to be cached + /// \param Name Name of call Instruction for callinst + /// + /// \returns CallInst to the thread private cache call. + CallInst *CreateCachedThreadPrivate(const LocationDescription &Loc, + llvm::Value *Pointer, + llvm::ConstantInt *Size, + const llvm::Twine &Name = Twine(" ")); + private: /// Common interface for generating entry calls for OMP Directives. /// if the directive has a region/body, It will set the insertion diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def --- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -164,6 +164,9 @@ OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, __VA_ARGS__) __OMP_FUNCTION_TYPE(ParallelTask, true, Void, Int32Ptr, Int32Ptr) +__OMP_FUNCTION_TYPE(KmpcCtorTy, false, VoidPtr, VoidPtr) +__OMP_FUNCTION_TYPE(KmpcCopyCtorTy, false, VoidPtr, VoidPtr, VoidPtr) +__OMP_FUNCTION_TYPE(KmpcDtorTy, false, Void, VoidPtr) #undef __OMP_FUNCTION_TYPE #undef OMP_FUNCTION_TYPE @@ -228,9 +231,14 @@ __OMP_RTL(__kmpc_master, false, Int32, IdentPtr, Int32) __OMP_RTL(__kmpc_end_master, false, Void, IdentPtr, Int32) __OMP_RTL(__kmpc_critical, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy) -__OMP_RTL(__kmpc_critical_with_hint, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy, Int32) +__OMP_RTL(__kmpc_critical_with_hint, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy, IntPtrTy) __OMP_RTL(__kmpc_end_critical, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy) +__OMP_RTL(__kmpc_alloc, false, VoidPtr, Int32, SizeTy, VoidPtr) +__OMP_RTL(__kmpc_free, false, Void, Int32, VoidPtr, VoidPtr) +__OMP_RTL(__kmpc_threadprivate_cached, false, VoidPtr, IdentPtr, Int32, VoidPtr, SizeTy, Void3Ptr) +__OMP_RTL(__kmpc_threadprivate_register, false, Void, IdentPtr, VoidPtr, KmpcCtorTy, KmpcCopyCtorTy, KmpcDtorTy) + __OMP_RTL(__last, false, Void, ) #undef __OMP_RTL diff --git a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp --- a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp @@ -54,6 +54,21 @@ PointerType *llvm::omp::types::VarName##Ptr = nullptr; #include "llvm/Frontend/OpenMP/OMPKinds.def" +// The following are target-dependent types +// Pointer Size type +Type *llvm::omp::types::IntPtrTy = nullptr; +// Size of Integer type +Type *llvm::omp::types::SizeTy = nullptr; +// Pointer to Int8 pointer type +Type *llvm::omp::types::Int8PtrPtr = nullptr; +// Pointer to pointer of Int8 Pointer type +Type *llvm::omp::types::Int8PtrPtrPtr = nullptr; +// Void pointer type +Type *llvm::omp::types::VoidPtr = nullptr; +// Pointer to Void pointer of pointer type +Type *llvm::omp::types::Void2Ptr = nullptr; +// pointer to pointer of Void pointer type +Type *llvm::omp::types::Void3Ptr = nullptr; ///} void llvm::omp::types::initializeTypes(Module &M) { @@ -65,6 +80,16 @@ // the llvm::PointerTypes of them for easy access later. StructType *T; #define OMP_TYPE(VarName, InitValue) VarName = InitValue; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + + llvm::omp::types::IntPtrTy = Int32; + llvm::omp::types::SizeTy = Int32; + llvm::omp::types::VoidPtr = Int8Ptr; + llvm::omp::types::Int8PtrPtr = Int8Ptr->getPointerTo(); + llvm::omp::types::Void2Ptr = VoidPtr->getPointerTo(); + llvm::omp::types::Int8PtrPtrPtr = Int8PtrPtr->getPointerTo(); + llvm::omp::types::Void3Ptr = Void2Ptr->getPointerTo(); + #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ VarName##Ty = ArrayType::get(ElemTy, ArraySize); \ VarName##PtrTy = PointerType::getUnqual(VarName##Ty); @@ -80,8 +105,35 @@ #include "llvm/Frontend/OpenMP/OMPKinds.def" } +void llvm::omp::types::initializeIntPtrTy(Type *IntPtrTy) { + llvm::omp::types::IntPtrTy = (IntPtrTy) ? IntPtrTy : Int32; +} + +void llvm::omp::types::initializeSizeTy(Type *SizeTy) { + llvm::omp::types::SizeTy = (SizeTy) ? SizeTy : Int32; +} + +void llvm::omp::types::initializeVoidPtrTy(Type *VoidPtrTy) { + if (VoidPtr) { + llvm::omp::types::VoidPtr = VoidPtrTy; + llvm::omp::types::Void2Ptr = VoidPtr->getPointerTo(); + llvm::omp::types::Void3Ptr = VoidPtr->getPointerTo()->getPointerTo(); + } +} + void llvm::omp::types::uninitializeTypes() { + IntPtrTy = nullptr; + SizeTy = nullptr; + Int8PtrPtr = nullptr; + Int8PtrPtrPtr = nullptr; + VoidPtr = nullptr; + Void2Ptr = nullptr; + Void3Ptr = nullptr; + #define OMP_TYPE(VarName, InitValue) VarName = nullptr; +#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ + VarName##Ty = nullptr; \ + VarName##PtrTy = nullptr; #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ VarName = nullptr; \ VarName##Ptr = nullptr; diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -93,6 +93,18 @@ void OpenMPIRBuilder::initialize() { initializeTypes(M); } +void OpenMPIRBuilder::setIntPtrTy(IntegerType *IntPtrTy) { + initializeIntPtrTy(IntPtrTy); +} + +void OpenMPIRBuilder::setSizeTy(IntegerType *SizeTy) { + initializeSizeTy(SizeTy); +} + +void OpenMPIRBuilder::setVoidPtrTy(PointerType *VoidPtrTy) { + initializeVoidPtrTy(VoidPtrTy); +} + void OpenMPIRBuilder::finalize() { for (OutlineInfo &OI : OutlineInfos) { assert(!OI.Blocks.empty() && @@ -576,9 +588,10 @@ "Unexpected finalization stack state!"); Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); - assert(PRegPreFiniTI->getNumSuccessors() == 1 && - PRegPreFiniTI->getSuccessor(0) == PRegExitBB && - "Unexpected CFG structure!"); + + // assert(PRegPreFiniTI->getNumSuccessors() == 1 && + // PRegPreFiniTI->getSuccessor(0) == PRegExitBB && + // "Unexpected CFG structure!"); InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); FiniCB(PreFiniIP); @@ -912,6 +925,93 @@ ExitCall->getIterator()); } +OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks( + InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, bool BranchtoEnd) { + if (!IP.isSet()) + return IP; + + IRBuilder<>::InsertPointGuard IPG(Builder); + + BasicBlock *OMP_Entry = IP.getBlock(); + Function *CurFn = OMP_Entry->getParent(); + BasicBlock *CopyBegin = + BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn); + BasicBlock *CopyEnd = nullptr; + // If entry block is terminated, split to preserve the branch to following + // basic block, otherwise, leave everything as is. + if (isa_and_nonnull(OMP_Entry->getTerminator())) { + CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), + "copyin.not.master.end"); + OMP_Entry->getTerminator()->eraseFromParent(); + } else { + CopyEnd = + BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn); + } + + Builder.SetInsertPoint(OMP_Entry); + Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); + Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); + Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); + Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); + + Builder.SetInsertPoint(CopyBegin); + if (BranchtoEnd) { + BranchInst *br = Builder.CreateBr(CopyEnd); + Builder.SetInsertPoint(br); + } + + return Builder.saveIP(); +} + +CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc, + Value *Size, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Size, Allocator}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_alloc); + + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc, + Value *Addr, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Addr, Allocator}; + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_free); + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate( + const LocationDescription &Loc, llvm::Value *Pointer, + llvm::ConstantInt *Size, const llvm::Twine &Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Constant *ThreadPrivateCache = + getOrCreateOMPInternalVariable(Int8PtrPtr, Name); + llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_threadprivate_cached); + + return Builder.CreateCall(Fn, Args); +} + std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) {