diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -226,6 +226,8 @@ if (LangOpts.OpenMPIRBuilder) { OMPBuilder.reset(new llvm::OpenMPIRBuilder(TheModule)); OMPBuilder->initialize(); + OMPBuilder->setIntPtrTy(IntPtrTy); + OMPBuilder->setSizeTy(SizeTy); } } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -95,6 +95,14 @@ ///{ namespace types { +/// Target specific types. + +extern Type *IntPtrTy; +extern Type *SizeTy; + +extern Type *Int8PtrPtr; +extern Type *Int8PtrPtrPtr; + #define OMP_TYPE(VarName, InitValue) extern Type *VarName; #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ extern ArrayType *VarName##Ty; \ @@ -110,6 +118,11 @@ /// Helper to initialize all types defined in OpenMP/OMPKinds.def. void initializeTypes(Module &M); +/// setters for target specific types +void initializeIntPtrTy(Type *IntPtrTy); +void initializeSizeTy(Type *SizeTy); +void initializeVoidPtrTy(Type *VoidPtrTy); + /// Helper to uninitialize all types defined in OpenMP/OMPKinds.def. void uninitializeTypes(); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -35,6 +35,11 @@ /// before any other method and only once! void initialize(); + /// setters for target specific types + void setIntPtrTy(IntegerType *IntPtrTy); + void setSizeTy(IntegerType *SizeTy); + void setVoidPtrTy(PointerType *VoidPtrTy); + /// Finalize the underlying module, e.g., by outlining regions. void finalize(); @@ -44,6 +49,23 @@ /// Type used throughout for insertion points. using InsertPointTy = IRBuilder<>::InsertPoint; + /// Restores insertion point to \p IP + void restoreIP(InsertPointTy IP) { Builder.restoreIP(IP); } + + /// Set Insert point to BasicBlock /p IPBB , and Iterator /p IP. + void setInsertPoint(BasicBlock *IPBB, BasicBlock::iterator IP) { + Builder.SetInsertPoint(IPBB, IP); + } + + /// Set Insert point to BasicBlock /p IPBB + void setInsertPoint(BasicBlock *IPBB) { Builder.SetInsertPoint(IPBB); } + + /// Set Insert point to Instruction /p IPII + void setInsertPoint(Instruction *IPII) { Builder.SetInsertPoint(IPII); } + + /// return a copy of the current insertion point information + InsertPointTy SaveIP() { return Builder.saveIP(); } + /// Callback type for variable finalization (think destructors). /// /// \param CodeGenIP is the insertion point at which the finalization code @@ -299,6 +321,7 @@ StringMap, BumpPtrAllocator> InternalVars; public: + /// Generator for '#omp master' /// /// \param Loc The insert and source location description. @@ -310,7 +333,7 @@ BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB); - /// Generator for '#omp master' + /// Generator for '#omp Critical' /// /// \param Loc The insert and source location description. /// \param BodyGenCB Callback that will generate the region body code. @@ -324,6 +347,58 @@ FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst); + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param IP insertion block for copyin conditional + /// \param MasterVarPtr a pointer to the master variable + /// \param PrivateVarPtr a pointer to the threadprivate variable + /// \param IntPtrTy Pointer size type + /// \param BranchtoEnd Create a branch between the copyin.not.master blocks + // and copy.in.end block + /// + /// \returns The insertion point where copying operation to be emitted. + InsertPointTy CreateCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, + Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, + bool BranchtoEnd = true); + + /// Create a runtime call for kmpc_Alloc + /// + /// \param Loc The insert and source location description. + /// \param Size Size of allocated memory space + /// \param Allocator Allocator information instruction + /// \param name Name of call Instruction for OMP_alloc + /// + /// \returns CallInst to the OMP_Alloc call + CallInst *CreateOMPAlloc(const LocationDescription &Loc, Value *Size, + Value *Allocator, std::string name = ""); + + /// Create a runtime call for kmpc_free + /// + /// \param Loc The insert and source location description. + /// \param Addr Address of memory space to be freed + /// \param Allocator Allocator information instruction + /// \param name Name of call Instruction for OMP_Free + /// + /// \returns CallInst to the OMP_Free call + CallInst *CreateOMPFree(const LocationDescription &Loc, Value *Addr, + Value *Allocator, std::string name = ""); + + /// Create a runtime call for kmpc_free + /// + /// \param Loc The insert and source location description. + /// \param Pointer pointer to data to be cached + /// \param Size size of data to be cached + /// \param Name Name of call Instruction for callinst + /// + /// \returns CallInst to the thread private cache call. + CallInst *CreateCachedThreadPrivate(const LocationDescription &Loc, + llvm::Value *Pointer, + llvm::ConstantInt *Size, + const llvm::Twine &Name = Twine(" ")); + private: /// Common interface for generating entry calls for OMP Directives. /// if the directive has a region/body, It will set the insertion diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def --- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -239,7 +239,7 @@ __OMP_RTL(__kmpc_master, false, Int32, IdentPtr, Int32) __OMP_RTL(__kmpc_end_master, false, Void, IdentPtr, Int32) __OMP_RTL(__kmpc_critical, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy) -__OMP_RTL(__kmpc_critical_with_hint, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy, Int32) +__OMP_RTL(__kmpc_critical_with_hint, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy, IntPtrTy) __OMP_RTL(__kmpc_end_critical, false, Void, IdentPtr, Int32, KmpCriticalNamePtrTy) __OMP_RTL(__kmpc_alloc, false, VoidPtr, Int32, SizeTy, VoidPtr) diff --git a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp --- a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp @@ -54,6 +54,16 @@ PointerType *llvm::omp::types::VarName##Ptr = nullptr; #include "llvm/Frontend/OpenMP/OMPKinds.def" +// The following are target-dependent types +// Pointer Size type +Type *llvm::omp::types::IntPtrTy = nullptr; +// Size of Integer type +Type *llvm::omp::types::SizeTy = nullptr; + +// Pointer to Int8 pointer type +Type *llvm::omp::types::Int8PtrPtr = nullptr; +// Pointer to pointer of Int8 Pointer type +Type *llvm::omp::types::Int8PtrPtrPtr = nullptr; ///} void llvm::omp::types::initializeTypes(Module &M) { @@ -65,6 +75,13 @@ // the llvm::PointerTypes of them for easy access later. StructType *T; #define OMP_TYPE(VarName, InitValue) VarName = InitValue; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + + llvm::omp::types::IntPtrTy = Int32; + llvm::omp::types::SizeTy = Int32; + llvm::omp::types::Int8PtrPtr = Int8Ptr->getPointerTo(); + llvm::omp::types::Int8PtrPtrPtr = Int8PtrPtr->getPointerTo(); + #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ VarName##Ty = ArrayType::get(ElemTy, ArraySize); \ VarName##PtrTy = PointerType::getUnqual(VarName##Ty); @@ -80,8 +97,30 @@ #include "llvm/Frontend/OpenMP/OMPKinds.def" } +void llvm::omp::types::initializeIntPtrTy(Type *IntPtrTy) { + llvm::omp::types::IntPtrTy = (IntPtrTy) ? IntPtrTy : Int32; +} + +void llvm::omp::types::initializeSizeTy(Type *SizeTy) { + llvm::omp::types::SizeTy = (SizeTy) ? SizeTy : Int32; +} + +void llvm::omp::types::initializeVoidPtrTy(Type *VoidPtrTy) { + llvm::omp::types::VoidPtr = (VoidPtrTy) ? VoidPtrTy : Int8Ptr; + llvm::omp::types::VoidPtrPtr = VoidPtr->getPointerTo(); + llvm::omp::types::VoidPtrPtrPtr = VoidPtrPtr->getPointerTo(); +} + void llvm::omp::types::uninitializeTypes() { + IntPtrTy = nullptr; + SizeTy = nullptr; + Int8PtrPtr = nullptr; + Int8PtrPtrPtr = nullptr; + #define OMP_TYPE(VarName, InitValue) VarName = nullptr; +#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ + VarName##Ty = nullptr; \ + VarName##PtrTy = nullptr; #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ VarName = nullptr; \ VarName##Ptr = nullptr; diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -93,6 +93,18 @@ void OpenMPIRBuilder::initialize() { initializeTypes(M); } +void OpenMPIRBuilder::setIntPtrTy(IntegerType *IntPtrTy) { + initializeIntPtrTy(IntPtrTy); +} + +void OpenMPIRBuilder::setSizeTy(IntegerType *SizeTy) { + initializeSizeTy(SizeTy); +} + +void OpenMPIRBuilder::setVoidPtrTy(PointerType *VoidPtrTy) { + initializeVoidPtrTy(VoidPtrTy); +} + void OpenMPIRBuilder::finalize() { for (OutlineInfo &OI : OutlineInfos) { assert(!OI.Blocks.empty() && @@ -576,9 +588,6 @@ "Unexpected finalization stack state!"); Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); - assert(PRegPreFiniTI->getNumSuccessors() == 1 && - PRegPreFiniTI->getSuccessor(0) == PRegExitBB && - "Unexpected CFG structure!"); InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); FiniCB(PreFiniIP); @@ -912,6 +921,106 @@ ExitCall->getIterator()); } +OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks( + InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, bool BranchtoEnd) { + if (!IP.isSet()) + return IP; + + IRBuilder<>::InsertPointGuard IPG(Builder); + + // creates the following CFG structure + // OMP_Entry : (MasterAddr != PrivateAddr)? + // F T + // | \ + // | copin.not.master + // | / + // v / + // copyin.not.master.end + // | + // v + // OMP.Entry.Next + + BasicBlock *OMP_Entry = IP.getBlock(); + Function *CurFn = OMP_Entry->getParent(); + BasicBlock *CopyBegin = + BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn); + BasicBlock *CopyEnd = nullptr; + + // If entry block is terminated, split to preserve the branch to following + // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is. + if (isa_and_nonnull(OMP_Entry->getTerminator())) { + CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), + "copyin.not.master.end"); + OMP_Entry->getTerminator()->eraseFromParent(); + } else { + CopyEnd = + BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn); + } + + Builder.SetInsertPoint(OMP_Entry); + Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); + Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); + Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); + Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); + + Builder.SetInsertPoint(CopyBegin); + if (BranchtoEnd) { + BranchInst *br = Builder.CreateBr(CopyEnd); + Builder.SetInsertPoint(br); + } + + return Builder.saveIP(); +} + +CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc, + Value *Size, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Size, Allocator}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_alloc); + + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc, + Value *Addr, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Addr, Allocator}; + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_free); + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate( + const LocationDescription &Loc, llvm::Value *Pointer, + llvm::ConstantInt *Size, const llvm::Twine &Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Constant *ThreadPrivateCache = + getOrCreateOMPInternalVariable(Int8PtrPtr, Name); + llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_threadprivate_cached); + + return Builder.CreateCall(Fn, Args); +} + std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) { diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -779,4 +779,41 @@ EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy); } +TEST_F(OpenMPIRBuilderTest, CopyinBlocks) { + using InsertPointTy = OpenMPIRBuilder::InsertPointTy; + OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.initialize(); + F->setName("func"); + IRBuilder<> Builder(BB); + + OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); + + AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); + IntegerType* Int32 = Type::getInt32Ty(M->getContext()); + AllocaInst* MasterAddress = Builder.CreateAlloca(Int32->getPointerTo()); + AllocaInst* PrivAddress = Builder.CreateAlloca(Int32->getPointerTo()); + + BasicBlock *EntryBB = BB; + + OMPBuilder.CreateCopyinClauseBlocks(Builder.saveIP(), MasterAddress, PrivAddress, Int32, /*BranchtoEnd*/true); + + BranchInst* EntryBr = dyn_cast_or_null(EntryBB->getTerminator()); + + EXPECT_NE(EntryBr, nullptr); + EXPECT_TRUE(EntryBr->isConditional()); + + BasicBlock* NotMasterBB = EntryBr->getSuccessor(0); + BasicBlock* CopyinEnd = EntryBr->getSuccessor(1); + CmpInst* CMP = dyn_cast_or_null(EntryBr->getCondition()); + + EXPECT_NE(CMP, nullptr); + EXPECT_NE(NotMasterBB, nullptr); + EXPECT_NE(CopyinEnd, nullptr); + + BranchInst* NotMasterBr = dyn_cast_or_null(NotMasterBB->getTerminator()); + EXPECT_NE(NotMasterBr, nullptr); + EXPECT_FALSE(NotMasterBr->isConditional()); + EXPECT_EQ(CopyinEnd,NotMasterBr->getSuccessor(0)); +} + } // namespace