diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1400,10 +1400,15 @@ Address address = Address::invalid(); Address AllocaAddr = Address::invalid(); - Address OpenMPLocalAddr = - getLangOpts().OpenMP - ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D) - : Address::invalid(); + Address OpenMPLocalAddr = Address::invalid(); + if (CGM.getOpenMPIRBuilder()) + OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D); + else + OpenMPLocalAddr = + getLangOpts().OpenMP + ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D) + : Address::invalid(); + bool NRVO = getLangOpts().ElideConstructors && D.isNRVOVariable(); if (getLangOpts().OpenMP && OpenMPLocalAddr.isValid()) { diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -2398,7 +2398,13 @@ static LValue EmitThreadPrivateVarDeclLValue( CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr, llvm::Type *RealVarTy, SourceLocation Loc) { - Addr = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); + if (CGF.CGM.getOpenMPIRBuilder()) + Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( + CGF, VD, Addr, Loc); + else + Addr = + CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); + Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy); return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); } diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -23,6 +23,7 @@ #include "clang/AST/StmtOpenMP.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/PrettyStackTrace.h" +#include "llvm/Frontend/OpenMP/OMPConstants.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" @@ -1564,6 +1565,96 @@ const OMPExecutableDirective &, llvm::SmallVectorImpl &) {} +Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable( + CodeGenFunction &CGF, const VarDecl *VD) { + CodeGenModule &CGM = CGF.CGM; + auto OMPBuilder = CGM.getOpenMPIRBuilder(); + assert(OMPBuilder && "OMPIRBuilder does not exist!"); + + if (!VD) + return Address::invalid(); + const VarDecl *CVD = VD->getCanonicalDecl(); + if (!CVD->hasAttr()) + return Address::invalid(); + const auto *AA = CVD->getAttr(); + // Use the default allocation. + if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc && + !AA->getAllocator()) + return Address::invalid(); + llvm::Value *Size; + CharUnits Align = CGM.getContext().getDeclAlign(CVD); + if (CVD->getType()->isVariablyModifiedType()) { + Size = CGF.getTypeSize(CVD->getType()); + // Align the size: ((size + align - 1) / align) * align + Size = CGF.Builder.CreateNUWAdd( + Size, CGM.getSize(Align - CharUnits::fromQuantity(1))); + Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); + Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); + } else { + CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType()); + Size = CGM.getSize(Sz.alignTo(Align)); + } + + assert(AA->getAllocator() && + "Expected allocator expression for non-default allocator."); + llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); + // According to the standard, the original allocator type is a enum (integer). + // Convert to pointer type, if required. + if (Allocator->getType()->isIntegerTy()) + Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); + else if (Allocator->getType()->isPointerTy()) + Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, + CGM.VoidPtrTy); + + llvm::Value *Addr = OMPBuilder->CreateOMPAlloc( + CGF.Builder, Size, Allocator, + getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", ".")); + llvm::CallInst *FreeCI = + OMPBuilder->CreateOMPFree(CGF.Builder, Addr, Allocator); + + CGF.EHStack.pushCleanup(NormalAndEHCleanup, FreeCI); + Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + Addr, + CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), + getNameWithSeparators({CVD->getName(), ".addr"}, ".", ".")); + return Address(Addr, Align); +} + +Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( + CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, + SourceLocation Loc) { + CodeGenModule &CGM = CGF.CGM; + if (CGM.getLangOpts().OpenMPUseTLS && + CGM.getContext().getTargetInfo().isTLSSupported()) + return VDAddr; + + llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder(); + assert(OMPBuilder && "OpenMPIRBuilder is not initialized or used."); + + llvm::Type *VarTy = VDAddr.getElementType(); + llvm::Value *Data = + CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); + llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)); + std::string Suffix = getNameWithSeparators({"cache", ""}); + llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix); + + llvm::CallInst *ThreadPrivateCacheCall = + OMPBuilder->CreateCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); + + return Address(ThreadPrivateCacheCall, VDAddr.getAlignment()); +} + +std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators( + ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) { + SmallString<128> Buffer; + llvm::raw_svector_ostream OS(Buffer); + StringRef Sep = FirstSeparator; + for (StringRef Part : Parts) { + OS << Sep << Part; + Sep = Separator; + } + return OS.str().str(); +} void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) { // Check if we have any if clause associated with the directive. diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -26,6 +26,7 @@ #include "clang/AST/ExprCXX.h" #include "clang/AST/ExprObjC.h" #include "clang/AST/ExprOpenMP.h" +#include "clang/AST/StmtOpenMP.h" #include "clang/AST/Type.h" #include "clang/Basic/ABI.h" #include "clang/Basic/CapturedStmt.h" @@ -80,6 +81,7 @@ class OMPUseDeviceAddrClause; class ReturnsNonNullAttr; class SVETypeFlags; +class OMPExecutableDirective; namespace analyze_os_log { class OSLogBufferLayout; @@ -259,114 +261,6 @@ unsigned Index; }; - // Helper class for the OpenMP IR Builder. Allows reusability of code used for - // region body, and finalization codegen callbacks. This will class will also - // contain privatization functions used by the privatization call backs - struct OMPBuilderCBHelpers { - - using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - - /// Emit the Finalization for an OMP region - /// \param CGF The Codegen function this belongs to - /// \param IP Insertion point for generating the finalization code. - static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) { - CGBuilderTy::InsertPointGuard IPG(CGF.Builder); - assert(IP.getBlock()->end() != IP.getPoint() && - "OpenMP IR Builder should cause terminated block!"); - - llvm::BasicBlock *IPBB = IP.getBlock(); - llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor(); - assert(DestBB && "Finalization block should have one successor!"); - - // erase and replace with cleanup branch. - IPBB->getTerminator()->eraseFromParent(); - CGF.Builder.SetInsertPoint(IPBB); - CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(DestBB); - CGF.EmitBranchThroughCleanup(Dest); - } - - /// Emit the body of an OMP region - /// \param CGF The Codegen function this belongs to - /// \param RegionBodyStmt The body statement for the OpenMP region being - /// generated - /// \param CodeGenIP Insertion point for generating the body code. - /// \param FiniBB The finalization basic block - static void EmitOMPRegionBody(CodeGenFunction &CGF, - const Stmt *RegionBodyStmt, - InsertPointTy CodeGenIP, - llvm::BasicBlock &FiniBB) { - llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); - if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator()) - CodeGenIPBBTI->eraseFromParent(); - - CGF.Builder.SetInsertPoint(CodeGenIPBB); - - CGF.EmitStmt(RegionBodyStmt); - - if (CGF.Builder.saveIP().isSet()) - CGF.Builder.CreateBr(&FiniBB); - } - - /// RAII for preserving necessary info during Outlined region body codegen. - class OutlinedRegionBodyRAII { - - llvm::AssertingVH OldAllocaIP; - CodeGenFunction::JumpDest OldReturnBlock; - CodeGenFunction &CGF; - - public: - OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP, - llvm::BasicBlock &RetBB) - : CGF(cgf) { - assert(AllocaIP.isSet() && - "Must specify Insertion point for allocas of outlined function"); - OldAllocaIP = CGF.AllocaInsertPt; - CGF.AllocaInsertPt = &*AllocaIP.getPoint(); - - OldReturnBlock = CGF.ReturnBlock; - CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB); - } - - ~OutlinedRegionBodyRAII() { - CGF.AllocaInsertPt = OldAllocaIP; - CGF.ReturnBlock = OldReturnBlock; - } - }; - - /// RAII for preserving necessary info during inlined region body codegen. - class InlinedRegionBodyRAII { - - llvm::AssertingVH OldAllocaIP; - CodeGenFunction &CGF; - - public: - InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP, - llvm::BasicBlock &FiniBB) - : CGF(cgf) { - // Alloca insertion block should be in the entry block of the containing - // function so it expects an empty AllocaIP in which case will reuse the - // old alloca insertion point, or a new AllocaIP in the same block as - // the old one - assert((!AllocaIP.isSet() || - CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) && - "Insertion point should be in the entry block of containing " - "function!"); - OldAllocaIP = CGF.AllocaInsertPt; - if (AllocaIP.isSet()) - CGF.AllocaInsertPt = &*AllocaIP.getPoint(); - - // TODO: Remove the call, after making sure the counter is not used by - // the EHStack. - // Since this is an inlined region, it should not modify the - // ReturnBlock, and should reuse the one for the enclosing outlined - // region. So, the JumpDest being return by the function is discarded - (void)CGF.getJumpDestInCurrentScope(&FiniBB); - } - - ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; } - }; - }; - CodeGenModule &CGM; // Per-module state. const TargetInfo &Target; @@ -1695,6 +1589,168 @@ CallArgList OldCXXInheritedCtorInitExprArgs; }; + // Helper class for the OpenMP IR Builder. Allows reusability of code used for + // region body, and finalization codegen callbacks. This will class will also + // contain privatization functions used by the privatization call backs + // + // TODO: this is temporary class for things that are being moved out of + // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or + // utility function for use with the OMPBuilder. Once that move to use the + // OMPBuilder is done, everything here will either become part of CodeGenFunc. + // directly, or a new helper class that will contain functions used by both + // this and the OMPBuilder + + struct OMPBuilderCBHelpers { + + OMPBuilderCBHelpers() = delete; + OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete; + OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete; + + using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; + + /// Cleanup action for allocate support. + class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup { + + private: + llvm::CallInst *RTLFnCI; + + public: + OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) { + RLFnCI->removeFromParent(); + } + + void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { + if (!CGF.HaveInsertPoint()) + return; + CGF.Builder.Insert(RTLFnCI); + } + }; + + /// Returns address of the threadprivate variable for the current + /// thread. This Also create any necessary OMP runtime calls. + /// + /// \param VD VarDecl for Threadprivate variable. + /// \param VDAddr Address of the Vardecl + /// \param Loc The location where the barrier directive was encountered + static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, Address VDAddr, + SourceLocation Loc); + + /// Gets the OpenMP-specific address of the local variable /p VD. + static Address getAddressOfLocalVariable(CodeGenFunction &CGF, + const VarDecl *VD); + /// Get the platform-specific name separator. + /// \param Parts different parts of the final name that needs separation + /// \param FirstSeparator First separator used between the initial two + /// parts of the name. + /// \param Separator separator used between all of the rest consecutinve + /// parts of the name + static std::string getNameWithSeparators(ArrayRef Parts, + StringRef FirstSeparator = ".", + StringRef Separator = "."); + /// Emit the Finalization for an OMP region + /// \param CGF The Codegen function this belongs to + /// \param IP Insertion point for generating the finalization code. + static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) { + CGBuilderTy::InsertPointGuard IPG(CGF.Builder); + assert(IP.getBlock()->end() != IP.getPoint() && + "OpenMP IR Builder should cause terminated block!"); + + llvm::BasicBlock *IPBB = IP.getBlock(); + llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor(); + assert(DestBB && "Finalization block should have one successor!"); + + // erase and replace with cleanup branch. + IPBB->getTerminator()->eraseFromParent(); + CGF.Builder.SetInsertPoint(IPBB); + CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(DestBB); + CGF.EmitBranchThroughCleanup(Dest); + } + + /// Emit the body of an OMP region + /// \param CGF The Codegen function this belongs to + /// \param RegionBodyStmt The body statement for the OpenMP region being + /// generated + /// \param CodeGenIP Insertion point for generating the body code. + /// \param FiniBB The finalization basic block + static void EmitOMPRegionBody(CodeGenFunction &CGF, + const Stmt *RegionBodyStmt, + InsertPointTy CodeGenIP, + llvm::BasicBlock &FiniBB) { + llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); + if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator()) + CodeGenIPBBTI->eraseFromParent(); + + CGF.Builder.SetInsertPoint(CodeGenIPBB); + + CGF.EmitStmt(RegionBodyStmt); + + if (CGF.Builder.saveIP().isSet()) + CGF.Builder.CreateBr(&FiniBB); + } + + /// RAII for preserving necessary info during Outlined region body codegen. + class OutlinedRegionBodyRAII { + + llvm::AssertingVH OldAllocaIP; + CodeGenFunction::JumpDest OldReturnBlock; + CGBuilderTy::InsertPoint IP; + CodeGenFunction &CGF; + + public: + OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP, + llvm::BasicBlock &RetBB) + : CGF(cgf) { + assert(AllocaIP.isSet() && + "Must specify Insertion point for allocas of outlined function"); + OldAllocaIP = CGF.AllocaInsertPt; + CGF.AllocaInsertPt = &*AllocaIP.getPoint(); + IP = CGF.Builder.saveIP(); + + OldReturnBlock = CGF.ReturnBlock; + CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB); + } + + ~OutlinedRegionBodyRAII() { + CGF.AllocaInsertPt = OldAllocaIP; + CGF.ReturnBlock = OldReturnBlock; + CGF.Builder.restoreIP(IP); + } + }; + + /// RAII for preserving necessary info during inlined region body codegen. + class InlinedRegionBodyRAII { + + llvm::AssertingVH OldAllocaIP; + CodeGenFunction &CGF; + + public: + InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP, + llvm::BasicBlock &FiniBB) + : CGF(cgf) { + // Alloca insertion block should be in the entry block of the containing + // function so it expects an empty AllocaIP in which case will reuse the + // old alloca insertion point, or a new AllocaIP in the same block as + // the old one + assert((!AllocaIP.isSet() || + CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) && + "Insertion point should be in the entry block of containing " + "function!"); + OldAllocaIP = CGF.AllocaInsertPt; + if (AllocaIP.isSet()) + CGF.AllocaInsertPt = &*AllocaIP.getPoint(); + + // TODO: Remove the call, after making sure the counter is not used by + // the EHStack. + // Since this is an inlined region, it should not modify the + // ReturnBlock, and should reuse the one for the enclosing outlined + // region. So, the JumpDest being return by the function is discarded + (void)CGF.getJumpDestInCurrentScope(&FiniBB); + } + + ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; } + }; + }; private: /// CXXThisDecl - When generating code for a C++ member function, /// this will hold the implicit 'this' declaration.