diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -225,7 +225,7 @@ // OpenMP-IR-Builder is opt-in only. if (LangOpts.OpenMPIRBuilder) { OMPBuilder.reset(new llvm::OpenMPIRBuilder(TheModule)); - OMPBuilder->initialize(); + OMPBuilder->initialize(SizeTy); } } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -95,6 +95,14 @@ ///{ namespace types { +/// Target specific types. + +extern Type *IntPtrTy; +extern Type *SizeTy; + +extern Type *Int8PtrPtr; +extern Type *Int8PtrPtrPtr; + #define OMP_TYPE(VarName, InitValue) extern Type *VarName; #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ extern ArrayType *VarName##Ty; \ @@ -108,7 +116,7 @@ #include "llvm/Frontend/OpenMP/OMPKinds.def" /// Helper to initialize all types defined in OpenMP/OMPKinds.def. -void initializeTypes(Module &M); +void initializeTypes(Module &M, Type *Size_T); /// Helper to uninitialize all types defined in OpenMP/OMPKinds.def. void uninitializeTypes(); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -33,7 +33,7 @@ /// Initialize the internal state, this will put structures types and /// potentially other helpers into the underlying module. Must be called /// before any other method and only once! - void initialize(); + void initialize(IntegerType* SizeTy); /// Finalize the underlying module, e.g., by outlining regions. void finalize(); @@ -44,6 +44,23 @@ /// Type used throughout for insertion points. using InsertPointTy = IRBuilder<>::InsertPoint; + /// Restores insertion point to \p IP + void restoreIP(InsertPointTy IP) { Builder.restoreIP(IP); } + + /// Set Insert point to BasicBlock \p IPBB , and Iterator /p IP. + void setInsertPoint(BasicBlock *IPBB, BasicBlock::iterator IP) { + Builder.SetInsertPoint(IPBB, IP); + } + + /// Set Insert point to BasicBlock \p IPBB + void setInsertPoint(BasicBlock *IPBB) { Builder.SetInsertPoint(IPBB); } + + /// Set Insert point to Instruction \p IPII + void setInsertPoint(Instruction *IPII) { Builder.SetInsertPoint(IPII); } + + /// return a copy of the current insertion point information + InsertPointTy SaveIP() { return Builder.saveIP(); } + /// Callback type for variable finalization (think destructors). /// /// \param CodeGenIP is the insertion point at which the finalization code @@ -299,6 +316,7 @@ StringMap, BumpPtrAllocator> InternalVars; public: + /// Generator for '#omp master' /// /// \param Loc The insert and source location description. @@ -310,7 +328,7 @@ BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB); - /// Generator for '#omp master' + /// Generator for '#omp critical' /// /// \param Loc The insert and source location description. /// \param BodyGenCB Callback that will generate the region body code. @@ -324,6 +342,58 @@ FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst); + /// Generate conditional branch and relevant BasicBlocks through which private + /// threads copy the 'copyin' variables from Master copy to threadprivate + /// copies. + /// + /// \param IP insertion block for copyin conditional + /// \param MasterVarPtr a pointer to the master variable + /// \param PrivateVarPtr a pointer to the threadprivate variable + /// \param IntPtrTy Pointer size type + /// \param BranchtoEnd Create a branch between the copyin.not.master blocks + // and copy.in.end block + /// + /// \returns The insertion point where copying operation to be emitted. + InsertPointTy CreateCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, + Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, + bool BranchtoEnd = true); + + /// Create a runtime call for kmpc_Alloc + /// + /// \param Loc The insert and source location description. + /// \param Size Size of allocated memory space + /// \param Allocator Allocator information instruction + /// \param Name Name of call Instruction for OMP_alloc + /// + /// \returns CallInst to the OMP_Alloc call + CallInst *CreateOMPAlloc(const LocationDescription &Loc, Value *Size, + Value *Allocator, std::string Name = ""); + + /// Create a runtime call for kmpc_free + /// + /// \param Loc The insert and source location description. + /// \param Addr Address of memory space to be freed + /// \param Allocator Allocator information instruction + /// \param Name Name of call Instruction for OMP_Free + /// + /// \returns CallInst to the OMP_Free call + CallInst *CreateOMPFree(const LocationDescription &Loc, Value *Addr, + Value *Allocator, std::string Name = ""); + + /// Create a runtime call for kmpc_threadprivate_cached + /// + /// \param Loc The insert and source location description. + /// \param Pointer pointer to data to be cached + /// \param Size size of data to be cached + /// \param Name Name of call Instruction for callinst + /// + /// \returns CallInst to the thread private cache call. + CallInst *CreateCachedThreadPrivate(const LocationDescription &Loc, + llvm::Value *Pointer, + llvm::ConstantInt *Size, + const llvm::Twine &Name = Twine("")); + private: /// Common interface for generating entry calls for OMP Directives. /// if the directive has a region/body, It will set the insertion @@ -382,7 +452,7 @@ /// \param Parts different parts of the final name that needs separation /// \param FirstSeparator First separator used between the initial two /// parts of the name. - /// \param Separator separator used between all of the rest consecutinve + /// \param Separator separator used between all of the rest consecutive /// parts of the name static std::string getNameWithSeparators(ArrayRef Parts, StringRef FirstSeparator, diff --git a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp --- a/llvm/lib/Frontend/OpenMP/OMPConstants.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPConstants.cpp @@ -54,17 +54,37 @@ PointerType *llvm::omp::types::VarName##Ptr = nullptr; #include "llvm/Frontend/OpenMP/OMPKinds.def" +// The following are target-dependent types +// Pointer Size type +Type *llvm::omp::types::IntPtrTy = nullptr; +// Size of Integer type +Type *llvm::omp::types::SizeTy = nullptr; + +// Pointer to Int8 pointer type +Type *llvm::omp::types::Int8PtrPtr = nullptr; +// Pointer to pointer of Int8 Pointer type +Type *llvm::omp::types::Int8PtrPtrPtr = nullptr; ///} -void llvm::omp::types::initializeTypes(Module &M) { +void llvm::omp::types::initializeTypes(Module &M, Type* Size_T) { if (Void) return; LLVMContext &Ctx = M.getContext(); + assert (Size_T && Size_T->isIntegerTy() + && "Type is not initialized to an Integer Type!"); + // Create all simple and struct types exposed by the runtime and remember // the llvm::PointerTypes of them for easy access later. StructType *T; #define OMP_TYPE(VarName, InitValue) VarName = InitValue; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + + llvm::omp::types::IntPtrTy = Size_T; + llvm::omp::types::SizeTy = Size_T; + llvm::omp::types::Int8PtrPtr = Int8Ptr->getPointerTo(); + llvm::omp::types::Int8PtrPtrPtr = Int8PtrPtr->getPointerTo(); + #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ VarName##Ty = ArrayType::get(ElemTy, ArraySize); \ VarName##PtrTy = PointerType::getUnqual(VarName##Ty); @@ -81,7 +101,15 @@ } void llvm::omp::types::uninitializeTypes() { + IntPtrTy = nullptr; + SizeTy = nullptr; + Int8PtrPtr = nullptr; + Int8PtrPtrPtr = nullptr; + #define OMP_TYPE(VarName, InitValue) VarName = nullptr; +#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ + VarName##Ty = nullptr; \ + VarName##PtrTy = nullptr; #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ VarName = nullptr; \ VarName##Ptr = nullptr; diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -91,7 +91,9 @@ return Fn; } -void OpenMPIRBuilder::initialize() { initializeTypes(M); } +void OpenMPIRBuilder::initialize(IntegerType* SizeTy) { + initializeTypes(M, SizeTy); +} void OpenMPIRBuilder::finalize() { for (OutlineInfo &OI : OutlineInfos) { @@ -576,9 +578,6 @@ "Unexpected finalization stack state!"); Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); - assert(PRegPreFiniTI->getNumSuccessors() == 1 && - PRegPreFiniTI->getSuccessor(0) == PRegExitBB && - "Unexpected CFG structure!"); InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); FiniCB(PreFiniIP); @@ -912,6 +911,104 @@ ExitCall->getIterator()); } +OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks( + InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, + llvm::IntegerType *IntPtrTy, bool BranchtoEnd) { + if (!IP.isSet()) + return IP; + + IRBuilder<>::InsertPointGuard IPG(Builder); + + // creates the following CFG structure + // OMP_Entry : (MasterAddr != PrivateAddr)? + // F T + // | \ + // | copin.not.master + // | / + // v / + // copyin.not.master.end + // | + // v + // OMP.Entry.Next + + BasicBlock *OMP_Entry = IP.getBlock(); + Function *CurFn = OMP_Entry->getParent(); + BasicBlock *CopyBegin = + BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn); + BasicBlock *CopyEnd = nullptr; + + // If entry block is terminated, split to preserve the branch to following + // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is. + if (isa_and_nonnull(OMP_Entry->getTerminator())) { + CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), + "copyin.not.master.end"); + OMP_Entry->getTerminator()->eraseFromParent(); + } else { + CopyEnd = + BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn); + } + + Builder.SetInsertPoint(OMP_Entry); + Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); + Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); + Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); + Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); + + Builder.SetInsertPoint(CopyBegin); + if (BranchtoEnd) + Builder.SetInsertPoint(Builder.CreateBr(CopyEnd)); + + return Builder.saveIP(); +} + +CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc, + Value *Size, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Size, Allocator}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_alloc); + + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc, + Value *Addr, Value *Allocator, + std::string Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Value *Args[] = {ThreadId, Addr, Allocator}; + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_free); + return Builder.CreateCall(Fn, Args, Name); +} + +CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate( + const LocationDescription &Loc, llvm::Value *Pointer, + llvm::ConstantInt *Size, const llvm::Twine &Name) { + IRBuilder<>::InsertPointGuard IPG(Builder); + Builder.restoreIP(Loc.IP); + + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Ident = getOrCreateIdent(SrcLocStr); + Value *ThreadId = getOrCreateThreadID(Ident); + Constant *ThreadPrivateCache = + getOrCreateOMPInternalVariable(Int8PtrPtr, Name); + llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache}; + + Function *Fn = getOrCreateRuntimeFunction(OMPRTL___kmpc_threadprivate_cached); + + return Builder.CreateCall(Fn, Args); +} + std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) { diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -56,9 +56,9 @@ CallGraphUpdater &CGUpdater) : M(*(*SCC.begin())->getParent()), SCC(SCC), ModuleSlice(ModuleSlice), OMPBuilder(M), CGUpdater(CGUpdater) { - initializeTypes(M); + initializeTypes(M,M.getDataLayout().getIntPtrType(M.getContext())); initializeRuntimeFunctions(); - OMPBuilder.initialize(); + OMPBuilder.initialize(M.getDataLayout().getIntPtrType(M.getContext())); } /// Generic information that describes a runtime function diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -62,7 +62,7 @@ TEST_F(OpenMPIRBuilderTest, CreateBarrier) { OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); IRBuilder<> Builder(BB); @@ -102,7 +102,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); @@ -157,7 +157,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); @@ -218,7 +218,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); @@ -272,7 +272,7 @@ TEST_F(OpenMPIRBuilderTest, DbgLoc) { OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -308,7 +308,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -405,7 +405,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -516,7 +516,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -625,7 +625,7 @@ TEST_F(OpenMPIRBuilderTest, MasterDirective) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -704,7 +704,7 @@ TEST_F(OpenMPIRBuilderTest, CriticalDirective) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); - OMPBuilder.initialize(); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); F->setName("func"); IRBuilder<> Builder(BB); @@ -779,4 +779,41 @@ EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy); } +TEST_F(OpenMPIRBuilderTest, CopyinBlocks) { + using InsertPointTy = OpenMPIRBuilder::InsertPointTy; + OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.initialize(M->getDataLayout().getIntPtrType(M->getContext())); + F->setName("func"); + IRBuilder<> Builder(BB); + + OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); + + AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); + IntegerType* Int32 = Type::getInt32Ty(M->getContext()); + AllocaInst* MasterAddress = Builder.CreateAlloca(Int32->getPointerTo()); + AllocaInst* PrivAddress = Builder.CreateAlloca(Int32->getPointerTo()); + + BasicBlock *EntryBB = BB; + + OMPBuilder.CreateCopyinClauseBlocks(Builder.saveIP(), MasterAddress, PrivAddress, Int32, /*BranchtoEnd*/true); + + BranchInst* EntryBr = dyn_cast_or_null(EntryBB->getTerminator()); + + EXPECT_NE(EntryBr, nullptr); + EXPECT_TRUE(EntryBr->isConditional()); + + BasicBlock* NotMasterBB = EntryBr->getSuccessor(0); + BasicBlock* CopyinEnd = EntryBr->getSuccessor(1); + CmpInst* CMP = dyn_cast_or_null(EntryBr->getCondition()); + + EXPECT_NE(CMP, nullptr); + EXPECT_NE(NotMasterBB, nullptr); + EXPECT_NE(CopyinEnd, nullptr); + + BranchInst* NotMasterBr = dyn_cast_or_null(NotMasterBB->getTerminator()); + EXPECT_NE(NotMasterBr, nullptr); + EXPECT_FALSE(NotMasterBr->isConditional()); + EXPECT_EQ(CopyinEnd,NotMasterBr->getSuccessor(0)); +} + } // namespace