diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 924f036a0020..c80a5bbb2401 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -1,1153 +1,1155 @@ //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file /// /// This file implements the OpenMPIRBuilder class, which is used as a /// convenient way to create LLVM instructions for OpenMP directives. /// //===----------------------------------------------------------------------===// #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/IR/CFG.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/MDBuilder.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/CodeExtractor.h" #include #define DEBUG_TYPE "openmp-ir-builder" using namespace llvm; using namespace omp; static cl::opt OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden, cl::desc("Use optimistic attributes describing " "'as-if' properties of runtime calls."), cl::init(false)); void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) { LLVMContext &Ctx = Fn.getContext(); #define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet; #include "llvm/Frontend/OpenMP/OMPKinds.def" // Add attributes to the new declaration. switch (FnID) { #define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \ case Enum: \ Fn.setAttributes( \ AttributeList::get(Ctx, FnAttrSet, RetAttrSet, ArgAttrSets)); \ break; #include "llvm/Frontend/OpenMP/OMPKinds.def" default: // Attributes are optional. break; } } FunctionCallee OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) { FunctionType *FnTy = nullptr; Function *Fn = nullptr; // Try to find the declation in the module first. switch (FnID) { #define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \ case Enum: \ FnTy = FunctionType::get(ReturnType, ArrayRef{__VA_ARGS__}, \ IsVarArg); \ Fn = M.getFunction(Str); \ break; #include "llvm/Frontend/OpenMP/OMPKinds.def" } if (!Fn) { // Create a new declaration if we need one. switch (FnID) { #define OMP_RTL(Enum, Str, ...) \ case Enum: \ Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \ break; #include "llvm/Frontend/OpenMP/OMPKinds.def" } // Add information if the runtime function takes a callback function if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) { if (!Fn->hasMetadata(LLVMContext::MD_callback)) { LLVMContext &Ctx = Fn->getContext(); MDBuilder MDB(Ctx); // Annotate the callback behavior of the runtime function: // - The callback callee is argument number 2 (microtask). // - The first two arguments of the callback callee are unknown (-1). // - All variadic arguments to the runtime function are passed to the // callback callee. Fn->addMetadata( LLVMContext::MD_callback, *MDNode::get(Ctx, {MDB.createCallbackEncoding( 2, {-1, -1}, /* VarArgsArePassed */ true)})); } } LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName() << " with type " << *Fn->getFunctionType() << "\n"); addAttributes(FnID, *Fn); } else { LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName() << " with type " << *Fn->getFunctionType() << "\n"); } assert(Fn && "Failed to create OpenMP runtime function"); // Cast the function to the expected type if necessary Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo()); return {FnTy, C}; } Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) { FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID); auto *Fn = dyn_cast(RTLFn.getCallee()); assert(Fn && "Failed to create OpenMP runtime function pointer"); return Fn; } void OpenMPIRBuilder::initialize() { initializeTypes(M); } void OpenMPIRBuilder::finalize() { SmallPtrSet ParallelRegionBlockSet; SmallVector Blocks; for (OutlineInfo &OI : OutlineInfos) { ParallelRegionBlockSet.clear(); Blocks.clear(); OI.collectBlocks(ParallelRegionBlockSet, Blocks); Function *OuterFn = OI.EntryBB->getParent(); CodeExtractorAnalysisCache CEAC(*OuterFn); CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr, /* AggregateArgs */ false, /* BlockFrequencyInfo */ nullptr, /* BranchProbabilityInfo */ nullptr, /* AssumptionCache */ nullptr, /* AllowVarArgs */ true, /* AllowAlloca */ true, /* Suffix */ ".omp_par"); LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n"); LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName() << " Exit: " << OI.ExitBB->getName() << "\n"); assert(Extractor.isEligible() && "Expected OpenMP outlining to be possible!"); Function *OutlinedFn = Extractor.extractCodeRegion(CEAC); LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n"); LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n"); assert(OutlinedFn->getReturnType()->isVoidTy() && "OpenMP outlined functions should not return a value!"); // For compability with the clang CG we move the outlined function after the // one with the parallel region. OutlinedFn->removeFromParent(); M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn); // Remove the artificial entry introduced by the extractor right away, we // made our own entry block after all. { BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock(); assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB); assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry); OI.EntryBB->moveBefore(&ArtificialEntry); ArtificialEntry.eraseFromParent(); } assert(&OutlinedFn->getEntryBlock() == OI.EntryBB); assert(OutlinedFn && OutlinedFn->getNumUses() == 1); // Run a user callback, e.g. to add attributes. if (OI.PostOutlineCB) OI.PostOutlineCB(*OutlinedFn); } // Allow finalize to be called multiple times. OutlineInfos.clear(); } Value *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr, IdentFlag LocFlags, unsigned Reserve2Flags) { // Enable "C-mode". LocFlags |= OMP_IDENT_FLAG_KMPC; Value *&Ident = IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}]; if (!Ident) { Constant *I32Null = ConstantInt::getNullValue(Int32); Constant *IdentData[] = { I32Null, ConstantInt::get(Int32, uint32_t(LocFlags)), ConstantInt::get(Int32, Reserve2Flags), I32Null, SrcLocStr}; Constant *Initializer = ConstantStruct::get( cast(IdentPtr->getPointerElementType()), IdentData); // Look for existing encoding of the location + flags, not needed but // minimizes the difference to the existing solution while we transition. for (GlobalVariable &GV : M.getGlobalList()) if (GV.getType() == IdentPtr && GV.hasInitializer()) if (GV.getInitializer() == Initializer) return Ident = &GV; auto *GV = new GlobalVariable(M, IdentPtr->getPointerElementType(), /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer); GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); GV->setAlignment(Align(8)); Ident = GV; } return Ident; } Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr) { Constant *&SrcLocStr = SrcLocStrMap[LocStr]; if (!SrcLocStr) { Constant *Initializer = ConstantDataArray::getString(M.getContext(), LocStr); // Look for existing encoding of the location, not needed but minimizes the // difference to the existing solution while we transition. for (GlobalVariable &GV : M.getGlobalList()) if (GV.isConstant() && GV.hasInitializer() && GV.getInitializer() == Initializer) return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr); SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "", /* AddressSpace */ 0, &M); } return SrcLocStr; } Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName, StringRef FileName, unsigned Line, unsigned Column) { SmallString<128> Buffer; Buffer.push_back(';'); Buffer.append(FileName); Buffer.push_back(';'); Buffer.append(FunctionName); Buffer.push_back(';'); Buffer.append(std::to_string(Line)); Buffer.push_back(';'); Buffer.append(std::to_string(Column)); Buffer.push_back(';'); Buffer.push_back(';'); return getOrCreateSrcLocStr(Buffer.str()); } Constant *OpenMPIRBuilder::getOrCreateDefaultSrcLocStr() { return getOrCreateSrcLocStr(";unknown;unknown;0;0;;"); } Constant * OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc) { DILocation *DIL = Loc.DL.get(); if (!DIL) return getOrCreateDefaultSrcLocStr(); - StringRef FileName = - !DIL->getFilename().empty() ? DIL->getFilename() : M.getName(); + StringRef FileName = M.getName(); + if (DIFile *DIF = DIL->getFile()) + if (Optional Source = DIF->getSource()) + FileName = *Source; StringRef Function = DIL->getScope()->getSubprogram()->getName(); Function = !Function.empty() ? Function : Loc.IP.getBlock()->getParent()->getName(); return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(), DIL->getColumn()); } Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) { return Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident, "omp_global_thread_num"); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateBarrier(const LocationDescription &Loc, Directive DK, bool ForceSimpleCall, bool CheckCancelFlag) { if (!updateToLocation(Loc)) return Loc.IP; return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind, bool ForceSimpleCall, bool CheckCancelFlag) { // Build call __kmpc_cancel_barrier(loc, thread_id) or // __kmpc_barrier(loc, thread_id); IdentFlag BarrierLocFlags; switch (Kind) { case OMPD_for: BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR; break; case OMPD_sections: BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS; break; case OMPD_single: BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE; break; case OMPD_barrier: BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL; break; default: BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL; break; } Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags), getOrCreateThreadID(getOrCreateIdent(SrcLocStr))}; // If we are in a cancellable parallel region, barriers are cancellation // points. // TODO: Check why we would force simple calls or to ignore the cancel flag. bool UseCancelBarrier = !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel); Value *Result = Builder.CreateCall(getOrCreateRuntimeFunctionPtr( UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier : OMPRTL___kmpc_barrier), Args); if (UseCancelBarrier && CheckCancelFlag) emitCancelationCheckImpl(Result, OMPD_parallel); return Builder.saveIP(); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCancel(const LocationDescription &Loc, Value *IfCondition, omp::Directive CanceledDirective) { if (!updateToLocation(Loc)) return Loc.IP; // LLVM utilities like blocks with terminators. auto *UI = Builder.CreateUnreachable(); Instruction *ThenTI = UI, *ElseTI = nullptr; if (IfCondition) SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI); Builder.SetInsertPoint(ThenTI); Value *CancelKind = nullptr; switch (CanceledDirective) { #define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \ case DirectiveEnum: \ CancelKind = Builder.getInt32(Value); \ break; #include "llvm/Frontend/OpenMP/OMPKinds.def" default: llvm_unreachable("Unknown cancel kind!"); } Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind}; Value *Result = Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args); // The actual cancel logic is shared with others, e.g., cancel_barriers. emitCancelationCheckImpl(Result, CanceledDirective); // Update the insertion point and remove the terminator we introduced. Builder.SetInsertPoint(UI->getParent()); UI->eraseFromParent(); return Builder.saveIP(); } void OpenMPIRBuilder::emitCancelationCheckImpl( Value *CancelFlag, omp::Directive CanceledDirective) { assert(isLastFinalizationInfoCancellable(CanceledDirective) && "Unexpected cancellation!"); // For a cancel barrier we create two new blocks. BasicBlock *BB = Builder.GetInsertBlock(); BasicBlock *NonCancellationBlock; if (Builder.GetInsertPoint() == BB->end()) { // TODO: This branch will not be needed once we moved to the // OpenMPIRBuilder codegen completely. NonCancellationBlock = BasicBlock::Create( BB->getContext(), BB->getName() + ".cont", BB->getParent()); } else { NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint()); BB->getTerminator()->eraseFromParent(); Builder.SetInsertPoint(BB); } BasicBlock *CancellationBlock = BasicBlock::Create( BB->getContext(), BB->getName() + ".cncl", BB->getParent()); // Jump to them based on the return value. Value *Cmp = Builder.CreateIsNull(CancelFlag); Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock, /* TODO weight */ nullptr, nullptr); // From the cancellation block we finalize all variables and go to the // post finalization block that is known to the FiniCB callback. Builder.SetInsertPoint(CancellationBlock); auto &FI = FinalizationStack.back(); FI.FiniCB(Builder.saveIP()); // The continuation block is where code generation continues. Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); } IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel( const LocationDescription &Loc, InsertPointTy OuterAllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) { if (!updateToLocation(Loc)) return Loc.IP; Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadID = getOrCreateThreadID(Ident); if (NumThreads) { // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads) Value *Args[] = { Ident, ThreadID, Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)}; Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args); } if (ProcBind != OMP_PROC_BIND_default) { // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind) Value *Args[] = { Ident, ThreadID, ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)}; Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args); } BasicBlock *InsertBB = Builder.GetInsertBlock(); Function *OuterFn = InsertBB->getParent(); // Vector to remember instructions we used only during the modeling but which // we want to delete at the end. SmallVector ToBeDeleted; // Change the location to the outer alloca insertion point to create and // initialize the allocas we pass into the parallel region. Builder.restoreIP(OuterAllocaIP); AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr"); AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr"); // If there is an if condition we actually use the TIDAddr and ZeroAddr in the // program, otherwise we only need them for modeling purposes to get the // associated arguments in the outlined function. In the former case, // initialize the allocas properly, in the latter case, delete them later. if (IfCondition) { Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr); Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr); } else { ToBeDeleted.push_back(TIDAddr); ToBeDeleted.push_back(ZeroAddr); } // Create an artificial insertion point that will also ensure the blocks we // are about to split are not degenerated. auto *UI = new UnreachableInst(Builder.getContext(), InsertBB); Instruction *ThenTI = UI, *ElseTI = nullptr; if (IfCondition) SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI); BasicBlock *ThenBB = ThenTI->getParent(); BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry"); BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region"); BasicBlock *PRegPreFiniBB = PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize"); BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit"); auto FiniCBWrapper = [&](InsertPointTy IP) { // Hide "open-ended" blocks from the given FiniCB by setting the right jump // target to the region exit block. if (IP.getBlock()->end() == IP.getPoint()) { IRBuilder<>::InsertPointGuard IPG(Builder); Builder.restoreIP(IP); Instruction *I = Builder.CreateBr(PRegExitBB); IP = InsertPointTy(I->getParent(), I->getIterator()); } assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && "Unexpected insertion point for finalization call!"); return FiniCB(IP); }; FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable}); // Generate the privatization allocas in the block that will become the entry // of the outlined function. Builder.SetInsertPoint(PRegEntryBB->getTerminator()); InsertPointTy InnerAllocaIP = Builder.saveIP(); AllocaInst *PrivTIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr.local"); Instruction *PrivTID = Builder.CreateLoad(PrivTIDAddr, "tid"); // Add some fake uses for OpenMP provided arguments. ToBeDeleted.push_back(Builder.CreateLoad(TIDAddr, "tid.addr.use")); ToBeDeleted.push_back(Builder.CreateLoad(ZeroAddr, "zero.addr.use")); // ThenBB // | // V // PRegionEntryBB <- Privatization allocas are placed here. // | // V // PRegionBodyBB <- BodeGen is invoked here. // | // V // PRegPreFiniBB <- The block we will start finalization from. // | // V // PRegionExitBB <- A common exit to simplify block collection. // LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n"); // Let the caller create the body. assert(BodyGenCB && "Expected body generation callback!"); InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin()); BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB); LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n"); FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call); if (auto *F = dyn_cast(RTLFn.getCallee())) { if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) { llvm::LLVMContext &Ctx = F->getContext(); MDBuilder MDB(Ctx); // Annotate the callback behavior of the __kmpc_fork_call: // - The callback callee is argument number 2 (microtask). // - The first two arguments of the callback callee are unknown (-1). // - All variadic arguments to the __kmpc_fork_call are passed to the // callback callee. F->addMetadata( llvm::LLVMContext::MD_callback, *llvm::MDNode::get( Ctx, {MDB.createCallbackEncoding(2, {-1, -1}, /* VarArgsArePassed */ true)})); } } OutlineInfo OI; OI.PostOutlineCB = [=](Function &OutlinedFn) { // Add some known attributes. OutlinedFn.addParamAttr(0, Attribute::NoAlias); OutlinedFn.addParamAttr(1, Attribute::NoAlias); OutlinedFn.addFnAttr(Attribute::NoUnwind); OutlinedFn.addFnAttr(Attribute::NoRecurse); assert(OutlinedFn.arg_size() >= 2 && "Expected at least tid and bounded tid as arguments"); unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2; CallInst *CI = cast(OutlinedFn.user_back()); CI->getParent()->setName("omp_parallel"); Builder.SetInsertPoint(CI); // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn); Value *ForkCallArgs[] = { Ident, Builder.getInt32(NumCapturedVars), Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)}; SmallVector RealArgs; RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs)); RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end()); Builder.CreateCall(RTLFn, RealArgs); LLVM_DEBUG(dbgs() << "With fork_call placed: " << *Builder.GetInsertBlock()->getParent() << "\n"); InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end()); // Initialize the local TID stack location with the argument value. Builder.SetInsertPoint(PrivTID); Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin(); Builder.CreateStore(Builder.CreateLoad(OutlinedAI), PrivTIDAddr); // If no "if" clause was present we do not need the call created during // outlining, otherwise we reuse it in the serialized parallel region. if (!ElseTI) { CI->eraseFromParent(); } else { // If an "if" clause was present we are now generating the serialized // version into the "else" branch. Builder.SetInsertPoint(ElseTI); // Build calls __kmpc_serialized_parallel(&Ident, GTid); Value *SerializedParallelCallArgs[] = {Ident, ThreadID}; Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel), SerializedParallelCallArgs); // OutlinedFn(>id, &zero, CapturedStruct); CI->removeFromParent(); Builder.Insert(CI); // __kmpc_end_serialized_parallel(&Ident, GTid); Value *EndArgs[] = {Ident, ThreadID}; Builder.CreateCall( getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel), EndArgs); LLVM_DEBUG(dbgs() << "With serialized parallel region: " << *Builder.GetInsertBlock()->getParent() << "\n"); } for (Instruction *I : ToBeDeleted) I->eraseFromParent(); }; // Adjust the finalization stack, verify the adjustment, and call the // finalize function a last time to finalize values between the pre-fini // block and the exit block if we left the parallel "the normal way". auto FiniInfo = FinalizationStack.pop_back_val(); (void)FiniInfo; assert(FiniInfo.DK == OMPD_parallel && "Unexpected finalization stack state!"); Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); FiniCB(PreFiniIP); OI.EntryBB = PRegEntryBB; OI.ExitBB = PRegExitBB; SmallPtrSet ParallelRegionBlockSet; SmallVector Blocks; OI.collectBlocks(ParallelRegionBlockSet, Blocks); // Ensure a single exit node for the outlined region by creating one. // We might have multiple incoming edges to the exit now due to finalizations, // e.g., cancel calls that cause the control flow to leave the region. BasicBlock *PRegOutlinedExitBB = PRegExitBB; PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt()); PRegOutlinedExitBB->setName("omp.par.outlined.exit"); Blocks.push_back(PRegOutlinedExitBB); CodeExtractorAnalysisCache CEAC(*OuterFn); CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr, /* AggregateArgs */ false, /* BlockFrequencyInfo */ nullptr, /* BranchProbabilityInfo */ nullptr, /* AssumptionCache */ nullptr, /* AllowVarArgs */ true, /* AllowAlloca */ true, /* Suffix */ ".omp_par"); // Find inputs to, outputs from the code region. BasicBlock *CommonExit = nullptr; SetVector Inputs, Outputs, SinkingCands, HoistingCands; Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit); Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands); LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n"); FunctionCallee TIDRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num); auto PrivHelper = [&](Value &V) { if (&V == TIDAddr || &V == ZeroAddr) return; SmallVector Uses; for (Use &U : V.uses()) if (auto *UserI = dyn_cast(U.getUser())) if (ParallelRegionBlockSet.count(UserI->getParent())) Uses.push_back(&U); Value *ReplacementValue = nullptr; CallInst *CI = dyn_cast(&V); if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) { ReplacementValue = PrivTID; } else { Builder.restoreIP( PrivCB(InnerAllocaIP, Builder.saveIP(), V, ReplacementValue)); assert(ReplacementValue && "Expected copy/create callback to set replacement value!"); if (ReplacementValue == &V) return; } for (Use *UPtr : Uses) UPtr->set(ReplacementValue); }; for (Value *Input : Inputs) { LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n"); PrivHelper(*Input); } LLVM_DEBUG({ for (Value *Output : Outputs) LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n"); }); assert(Outputs.empty() && "OpenMP outlining should not produce live-out values!"); LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n"); LLVM_DEBUG({ for (auto *BB : Blocks) dbgs() << " PBR: " << BB->getName() << "\n"; }); // Register the outlined info. addOutlineInfo(std::move(OI)); InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end()); UI->eraseFromParent(); return AfterIP; } void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) { // Build call void __kmpc_flush(ident_t *loc) Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Args[] = {getOrCreateIdent(SrcLocStr)}; Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args); } void OpenMPIRBuilder::CreateFlush(const LocationDescription &Loc) { if (!updateToLocation(Loc)) return; emitFlush(Loc); } void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) { // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 // global_tid); Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *Args[] = {Ident, getOrCreateThreadID(Ident)}; // Ignore return result until untied tasks are supported. Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait), Args); } void OpenMPIRBuilder::CreateTaskwait(const LocationDescription &Loc) { if (!updateToLocation(Loc)) return; emitTaskwaitImpl(Loc); } void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) { // Build call __kmpc_omp_taskyield(loc, thread_id, 0); Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Constant *I32Null = ConstantInt::getNullValue(Int32); Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null}; Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield), Args); } void OpenMPIRBuilder::CreateTaskyield(const LocationDescription &Loc) { if (!updateToLocation(Loc)) return; emitTaskyieldImpl(Loc); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateMaster(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB) { if (!updateToLocation(Loc)) return Loc.IP; Directive OMPD = Directive::OMPD_master; Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadId = getOrCreateThreadID(Ident); Value *Args[] = {Ident, ThreadId}; Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master); Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args); Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master); Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB, /*Conditional*/ true, /*hasFinalize*/ true); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCritical( const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) { if (!updateToLocation(Loc)) return Loc.IP; Directive OMPD = Directive::OMPD_critical; Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadId = getOrCreateThreadID(Ident); Value *LockVar = getOMPCriticalRegionLock(CriticalName); Value *Args[] = {Ident, ThreadId, LockVar}; SmallVector EnterArgs(std::begin(Args), std::end(Args)); Function *RTFn = nullptr; if (HintInst) { // Add Hint to entry Args and create call EnterArgs.push_back(HintInst); RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint); } else { RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical); } Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs); Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical); Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB, /*Conditional*/ false, /*hasFinalize*/ true); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion( Directive OMPD, Instruction *EntryCall, Instruction *ExitCall, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional, bool HasFinalize) { if (HasFinalize) FinalizationStack.push_back({FiniCB, OMPD, /*IsCancellable*/ false}); // Create inlined region's entry and body blocks, in preparation // for conditional creation BasicBlock *EntryBB = Builder.GetInsertBlock(); Instruction *SplitPos = EntryBB->getTerminator(); if (!isa_and_nonnull(SplitPos)) SplitPos = new UnreachableInst(Builder.getContext(), EntryBB); BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end"); BasicBlock *FiniBB = EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize"); Builder.SetInsertPoint(EntryBB->getTerminator()); emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional); // generate body BodyGenCB(/* AllocaIP */ InsertPointTy(), /* CodeGenIP */ Builder.saveIP(), *FiniBB); // If we didn't emit a branch to FiniBB during body generation, it means // FiniBB is unreachable (e.g. while(1);). stop generating all the // unreachable blocks, and remove anything we are not going to use. auto SkipEmittingRegion = FiniBB->hasNPredecessors(0); if (SkipEmittingRegion) { FiniBB->eraseFromParent(); ExitCall->eraseFromParent(); // Discard finalization if we have it. if (HasFinalize) { assert(!FinalizationStack.empty() && "Unexpected finalization stack state!"); FinalizationStack.pop_back(); } } else { // emit exit call and do any needed finalization. auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt()); assert(FiniBB->getTerminator()->getNumSuccessors() == 1 && FiniBB->getTerminator()->getSuccessor(0) == ExitBB && "Unexpected control flow graph state!!"); emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize); assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && "Unexpected Control Flow State!"); MergeBlockIntoPredecessor(FiniBB); } // If we are skipping the region of a non conditional, remove the exit // block, and clear the builder's insertion point. assert(SplitPos->getParent() == ExitBB && "Unexpected Insertion point location!"); if (!Conditional && SkipEmittingRegion) { ExitBB->eraseFromParent(); Builder.ClearInsertionPoint(); } else { auto merged = MergeBlockIntoPredecessor(ExitBB); BasicBlock *ExitPredBB = SplitPos->getParent(); auto InsertBB = merged ? ExitPredBB : ExitBB; if (!isa_and_nonnull(SplitPos)) SplitPos->eraseFromParent(); Builder.SetInsertPoint(InsertBB); } return Builder.saveIP(); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry( Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) { // if nothing to do, Return current insertion point. if (!Conditional) return Builder.saveIP(); BasicBlock *EntryBB = Builder.GetInsertBlock(); Value *CallBool = Builder.CreateIsNotNull(EntryCall); auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body"); auto *UI = new UnreachableInst(Builder.getContext(), ThenBB); // Emit thenBB and set the Builder's insertion point there for // body generation next. Place the block after the current block. Function *CurFn = EntryBB->getParent(); CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB); // Move Entry branch to end of ThenBB, and replace with conditional // branch (If-stmt) Instruction *EntryBBTI = EntryBB->getTerminator(); Builder.CreateCondBr(CallBool, ThenBB, ExitBB); EntryBBTI->removeFromParent(); Builder.SetInsertPoint(UI); Builder.Insert(EntryBBTI); UI->eraseFromParent(); Builder.SetInsertPoint(ThenBB->getTerminator()); // return an insertion point to ExitBB. return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt()); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit( omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall, bool HasFinalize) { Builder.restoreIP(FinIP); // If there is finalization to do, emit it before the exit call if (HasFinalize) { assert(!FinalizationStack.empty() && "Unexpected finalization stack state!"); FinalizationInfo Fi = FinalizationStack.pop_back_val(); assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!"); Fi.FiniCB(FinIP); BasicBlock *FiniBB = FinIP.getBlock(); Instruction *FiniBBTI = FiniBB->getTerminator(); // set Builder IP for call creation Builder.SetInsertPoint(FiniBBTI); } // place the Exitcall as last instruction before Finalization block terminator ExitCall->removeFromParent(); Builder.Insert(ExitCall); return IRBuilder<>::InsertPoint(ExitCall->getParent(), ExitCall->getIterator()); } OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks( InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, llvm::IntegerType *IntPtrTy, bool BranchtoEnd) { if (!IP.isSet()) return IP; IRBuilder<>::InsertPointGuard IPG(Builder); // creates the following CFG structure // OMP_Entry : (MasterAddr != PrivateAddr)? // F T // | \ // | copin.not.master // | / // v / // copyin.not.master.end // | // v // OMP.Entry.Next BasicBlock *OMP_Entry = IP.getBlock(); Function *CurFn = OMP_Entry->getParent(); BasicBlock *CopyBegin = BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn); BasicBlock *CopyEnd = nullptr; // If entry block is terminated, split to preserve the branch to following // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is. if (isa_and_nonnull(OMP_Entry->getTerminator())) { CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), "copyin.not.master.end"); OMP_Entry->getTerminator()->eraseFromParent(); } else { CopyEnd = BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn); } Builder.SetInsertPoint(OMP_Entry); Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); Builder.SetInsertPoint(CopyBegin); if (BranchtoEnd) Builder.SetInsertPoint(Builder.CreateBr(CopyEnd)); return Builder.saveIP(); } CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc, Value *Size, Value *Allocator, std::string Name) { IRBuilder<>::InsertPointGuard IPG(Builder); Builder.restoreIP(Loc.IP); Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadId = getOrCreateThreadID(Ident); Value *Args[] = {ThreadId, Size, Allocator}; Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc); return Builder.CreateCall(Fn, Args, Name); } CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc, Value *Addr, Value *Allocator, std::string Name) { IRBuilder<>::InsertPointGuard IPG(Builder); Builder.restoreIP(Loc.IP); Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadId = getOrCreateThreadID(Ident); Value *Args[] = {ThreadId, Addr, Allocator}; Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free); return Builder.CreateCall(Fn, Args, Name); } CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate( const LocationDescription &Loc, llvm::Value *Pointer, llvm::ConstantInt *Size, const llvm::Twine &Name) { IRBuilder<>::InsertPointGuard IPG(Builder); Builder.restoreIP(Loc.IP); Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); Value *Ident = getOrCreateIdent(SrcLocStr); Value *ThreadId = getOrCreateThreadID(Ident); Constant *ThreadPrivateCache = getOrCreateOMPInternalVariable(Int8PtrPtr, Name); llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache}; Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached); return Builder.CreateCall(Fn, Args); } std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) { SmallString<128> Buffer; llvm::raw_svector_ostream OS(Buffer); StringRef Sep = FirstSeparator; for (StringRef Part : Parts) { OS << Sep << Part; Sep = Separator; } return OS.str().str(); } Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable( llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) { // TODO: Replace the twine arg with stringref to get rid of the conversion // logic. However This is taken from current implementation in clang as is. // Since this method is used in many places exclusively for OMP internal use // we will keep it as is for temporarily until we move all users to the // builder and then, if possible, fix it everywhere in one go. SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); Out << Name; StringRef RuntimeName = Out.str(); auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first; if (Elem.second) { assert(Elem.second->getType()->getPointerElementType() == Ty && "OMP internal variable has different type than requested"); } else { // TODO: investigate the appropriate linkage type used for the global // variable for possibly changing that to internal or private, or maybe // create different versions of the function for different OMP internal // variables. Elem.second = new llvm::GlobalVariable( M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty), Elem.first(), /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal, AddressSpace); } return Elem.second; } Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) { std::string Prefix = Twine("gomp_critical_user_", CriticalName).str(); std::string Name = getNameWithSeparators({Prefix, "var"}, ".", "."); return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name); } // Create all simple and struct types exposed by the runtime and remember // the llvm::PointerTypes of them for easy access later. void OpenMPIRBuilder::initializeTypes(Module &M) { LLVMContext &Ctx = M.getContext(); StructType *T; #define OMP_TYPE(VarName, InitValue) VarName = InitValue; #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ VarName##Ty = ArrayType::get(ElemTy, ArraySize); \ VarName##PtrTy = PointerType::getUnqual(VarName##Ty); #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \ VarName##Ptr = PointerType::getUnqual(VarName); #define OMP_STRUCT_TYPE(VarName, StructName, ...) \ T = M.getTypeByName(StructName); \ if (!T) \ T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \ VarName = T; \ VarName##Ptr = PointerType::getUnqual(T); #include "llvm/Frontend/OpenMP/OMPKinds.def" } void OpenMPIRBuilder::OutlineInfo::collectBlocks( SmallPtrSetImpl &BlockSet, SmallVectorImpl &BlockVector) { SmallVector Worklist; BlockSet.insert(EntryBB); BlockSet.insert(ExitBB); Worklist.push_back(EntryBB); while (!Worklist.empty()) { BasicBlock *BB = Worklist.pop_back_val(); BlockVector.push_back(BB); for (BasicBlock *SuccBB : successors(BB)) if (BlockSet.insert(SuccBB).second) Worklist.push_back(SuccBB); } } diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp index edd2c8f5dd88..d671e65f8f49 100644 --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -1,1023 +1,1024 @@ //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/Frontend/OpenMP/OMPConstants.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "gtest/gtest.h" using namespace llvm; using namespace omp; namespace { class OpenMPIRBuilderTest : public testing::Test { protected: void SetUp() override { M.reset(new Module("MyModule", Ctx)); FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)}, /*isVarArg=*/false); F = Function::Create(FTy, Function::ExternalLinkage, "", M.get()); BB = BasicBlock::Create(Ctx, "", F); DIBuilder DIB(*M); - auto File = DIB.createFile("test.dbg", "/"); + auto File = DIB.createFile("test.dbg", "/src", llvm::None, + Optional("/src/test.dbg")); auto CU = DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0); auto Type = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None)); auto SP = DIB.createFunction( CU, "foo", "", File, 1, Type, 1, DINode::FlagZero, DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized); F->setSubprogram(SP); auto Scope = DIB.createLexicalBlockFile(SP, File, 0); DIB.finalize(); DL = DebugLoc::get(3, 7, Scope); } void TearDown() override { BB = nullptr; M.reset(); } LLVMContext Ctx; std::unique_ptr M; Function *F; BasicBlock *BB; DebugLoc DL; }; TEST_F(OpenMPIRBuilderTest, CreateBarrier) { OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); IRBuilder<> Builder(BB); OMPBuilder.CreateBarrier({IRBuilder<>::InsertPoint()}, OMPD_for); EXPECT_TRUE(M->global_empty()); EXPECT_EQ(M->size(), 1U); EXPECT_EQ(F->size(), 1U); EXPECT_EQ(BB->size(), 0U); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); OMPBuilder.CreateBarrier(Loc, OMPD_for); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 1U); EXPECT_EQ(BB->size(), 2U); CallInst *GTID = dyn_cast(&BB->front()); EXPECT_NE(GTID, nullptr); EXPECT_EQ(GTID->getNumArgOperands(), 1U); EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); CallInst *Barrier = dyn_cast(GTID->getNextNode()); EXPECT_NE(Barrier, nullptr); EXPECT_EQ(Barrier->getNumArgOperands(), 2U); EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier"); EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); EXPECT_EQ(cast(Barrier)->getArgOperand(1), GTID); Builder.CreateUnreachable(); EXPECT_FALSE(verifyModule(*M, &errs())); } TEST_F(OpenMPIRBuilderTest, CreateCancel) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); auto FiniCB = [&](InsertPointTy IP) { ASSERT_NE(IP.getBlock(), nullptr); ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); auto NewIP = OMPBuilder.CreateCancel(Loc, nullptr, OMPD_parallel); Builder.restoreIP(NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 4U); EXPECT_EQ(BB->size(), 4U); CallInst *GTID = dyn_cast(&BB->front()); EXPECT_NE(GTID, nullptr); EXPECT_EQ(GTID->getNumArgOperands(), 1U); EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); CallInst *Cancel = dyn_cast(GTID->getNextNode()); EXPECT_NE(Cancel, nullptr); EXPECT_EQ(Cancel->getNumArgOperands(), 3U); EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel"); EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory()); EXPECT_EQ(Cancel->getNumUses(), 1U); Instruction *CancelBBTI = Cancel->getParent()->getTerminator(); EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock()); EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB); EXPECT_EQ(cast(Cancel)->getArgOperand(1), GTID); OMPBuilder.popFinalizationCB(); Builder.CreateUnreachable(); EXPECT_FALSE(verifyModule(*M, &errs())); } TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); auto FiniCB = [&](InsertPointTy IP) { ASSERT_NE(IP.getBlock(), nullptr); ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); auto NewIP = OMPBuilder.CreateCancel(Loc, Builder.getTrue(), OMPD_parallel); Builder.restoreIP(NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 7U); EXPECT_EQ(BB->size(), 1U); ASSERT_TRUE(isa(BB->getTerminator())); ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U); BB = BB->getTerminator()->getSuccessor(0); EXPECT_EQ(BB->size(), 4U); CallInst *GTID = dyn_cast(&BB->front()); EXPECT_NE(GTID, nullptr); EXPECT_EQ(GTID->getNumArgOperands(), 1U); EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); CallInst *Cancel = dyn_cast(GTID->getNextNode()); EXPECT_NE(Cancel, nullptr); EXPECT_EQ(Cancel->getNumArgOperands(), 3U); EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel"); EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory()); EXPECT_EQ(Cancel->getNumUses(), 1U); Instruction *CancelBBTI = Cancel->getParent()->getTerminator(); EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(), NewIP.getBlock()); EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 1U); EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB); EXPECT_EQ(cast(Cancel)->getArgOperand(1), GTID); OMPBuilder.popFinalizationCB(); Builder.CreateUnreachable(); EXPECT_FALSE(verifyModule(*M, &errs())); } TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); new UnreachableInst(Ctx, CBB); auto FiniCB = [&](InsertPointTy IP) { ASSERT_NE(IP.getBlock(), nullptr); ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); BranchInst::Create(CBB, IP.getBlock()); }; OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); auto NewIP = OMPBuilder.CreateBarrier(Loc, OMPD_for); Builder.restoreIP(NewIP); EXPECT_FALSE(M->global_empty()); EXPECT_EQ(M->size(), 3U); EXPECT_EQ(F->size(), 4U); EXPECT_EQ(BB->size(), 4U); CallInst *GTID = dyn_cast(&BB->front()); EXPECT_NE(GTID, nullptr); EXPECT_EQ(GTID->getNumArgOperands(), 1U); EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); CallInst *Barrier = dyn_cast(GTID->getNextNode()); EXPECT_NE(Barrier, nullptr); EXPECT_EQ(Barrier->getNumArgOperands(), 2U); EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier"); EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); EXPECT_EQ(Barrier->getNumUses(), 1U); Instruction *BarrierBBTI = Barrier->getParent()->getTerminator(); EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U); EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock()); EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U); EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 1U); EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB); EXPECT_EQ(cast(Barrier)->getArgOperand(1), GTID); OMPBuilder.popFinalizationCB(); Builder.CreateUnreachable(); EXPECT_FALSE(verifyModule(*M, &errs())); } TEST_F(OpenMPIRBuilderTest, DbgLoc) { OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); OMPBuilder.CreateBarrier(Loc, OMPD_for); CallInst *GTID = dyn_cast(&BB->front()); CallInst *Barrier = dyn_cast(GTID->getNextNode()); EXPECT_EQ(GTID->getDebugLoc(), DL); EXPECT_EQ(Barrier->getDebugLoc(), DL); EXPECT_TRUE(isa(Barrier->getOperand(0))); if (!isa(Barrier->getOperand(0))) return; GlobalVariable *Ident = cast(Barrier->getOperand(0)); EXPECT_TRUE(Ident->hasInitializer()); if (!Ident->hasInitializer()) return; Constant *Initializer = Ident->getInitializer(); EXPECT_TRUE( isa(Initializer->getOperand(4)->stripPointerCasts())); GlobalVariable *SrcStrGlob = cast(Initializer->getOperand(4)->stripPointerCasts()); if (!SrcStrGlob) return; EXPECT_TRUE(isa(SrcStrGlob->getInitializer())); ConstantDataArray *SrcSrc = dyn_cast(SrcStrGlob->getInitializer()); if (!SrcSrc) return; - EXPECT_EQ(SrcSrc->getAsCString(), ";test.dbg;foo;3;7;;"); + EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;"); } TEST_F(OpenMPIRBuilderTest, ParallelSimple) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); AllocaInst *PrivAI = nullptr; unsigned NumBodiesGenerated = 0; unsigned NumPrivatizedVars = 0; unsigned NumFinalizationPoints = 0; auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumBodiesGenerated; Builder.restoreIP(AllocaIP); PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); Builder.CreateStore(F->arg_begin(), PrivAI); Builder.restoreIP(CodeGenIP); Value *PrivLoad = Builder.CreateLoad(PrivAI, "local.use"); Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad); Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); Builder.SetInsertPoint(ThenTerm); Builder.CreateBr(&ContinuationIP); ThenTerm->eraseFromParent(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &VPtr, Value *&ReplacementValue) -> InsertPointTy { ++NumPrivatizedVars; if (!isa(VPtr)) { EXPECT_EQ(&VPtr, F->arg_begin()); ReplacementValue = &VPtr; return CodeGenIP; } // Trivial copy (=firstprivate). Builder.restoreIP(AllocaIP); Type *VTy = VPtr.getType()->getPointerElementType(); Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload"); ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy"); Builder.restoreIP(CodeGenIP); Builder.CreateStore(V, ReplacementValue); return CodeGenIP; }; auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 1U); EXPECT_EQ(NumFinalizationPoints, 1U); Builder.restoreIP(AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_NE(PrivAI, nullptr); Function *OutlinedFn = PrivAI->getFunction(); EXPECT_NE(F, OutlinedFn); EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind)); EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn->hasInternalLinkage()); EXPECT_EQ(OutlinedFn->arg_size(), 3U); EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent()); EXPECT_EQ(OutlinedFn->getNumUses(), 1U); User *Usr = OutlinedFn->user_back(); ASSERT_TRUE(isa(Usr)); CallInst *ForkCI = dyn_cast(Usr->user_back()); ASSERT_NE(ForkCI, nullptr); EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); EXPECT_EQ(ForkCI->getNumArgOperands(), 4U); EXPECT_TRUE(isa(ForkCI->getArgOperand(0))); EXPECT_EQ(ForkCI->getArgOperand(1), ConstantInt::get(Type::getInt32Ty(Ctx), 1U)); EXPECT_EQ(ForkCI->getArgOperand(2), Usr); EXPECT_EQ(ForkCI->getArgOperand(3), F->arg_begin()); } TEST_F(OpenMPIRBuilderTest, ParallelNested) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); unsigned NumInnerBodiesGenerated = 0; unsigned NumOuterBodiesGenerated = 0; unsigned NumFinalizationPoints = 0; auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumInnerBodiesGenerated; }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &VPtr, Value *&ReplacementValue) -> InsertPointTy { // Trivial copy (=firstprivate). Builder.restoreIP(AllocaIP); Type *VTy = VPtr.getType()->getPointerElementType(); Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload"); ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy"); Builder.restoreIP(CodeGenIP); Builder.CreateStore(V, ReplacementValue); return CodeGenIP; }; auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumOuterBodiesGenerated; Builder.restoreIP(CodeGenIP); BasicBlock *CGBB = CodeGenIP.getBlock(); BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint()); CGBB->getTerminator()->eraseFromParent(); ; IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel( InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); Builder.restoreIP(AfterIP); Builder.CreateBr(NewBB); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); EXPECT_EQ(NumInnerBodiesGenerated, 1U); EXPECT_EQ(NumOuterBodiesGenerated, 1U); EXPECT_EQ(NumFinalizationPoints, 2U); Builder.restoreIP(AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_EQ(M->size(), 5U); for (Function &OutlinedFn : *M) { if (F == &OutlinedFn || OutlinedFn.isDeclaration()) continue; EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasInternalLinkage()); EXPECT_EQ(OutlinedFn.arg_size(), 2U); EXPECT_EQ(OutlinedFn.getNumUses(), 1U); User *Usr = OutlinedFn.user_back(); ASSERT_TRUE(isa(Usr)); CallInst *ForkCI = dyn_cast(Usr->user_back()); ASSERT_NE(ForkCI, nullptr); EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); EXPECT_EQ(ForkCI->getNumArgOperands(), 3U); EXPECT_TRUE(isa(ForkCI->getArgOperand(0))); EXPECT_EQ(ForkCI->getArgOperand(1), ConstantInt::get(Type::getInt32Ty(Ctx), 0U)); EXPECT_EQ(ForkCI->getArgOperand(2), Usr); } } TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); unsigned NumInnerBodiesGenerated = 0; unsigned NumOuterBodiesGenerated = 0; unsigned NumFinalizationPoints = 0; auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumInnerBodiesGenerated; }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &VPtr, Value *&ReplacementValue) -> InsertPointTy { // Trivial copy (=firstprivate). Builder.restoreIP(AllocaIP); Type *VTy = VPtr.getType()->getPointerElementType(); Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload"); ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy"); Builder.restoreIP(CodeGenIP); Builder.CreateStore(V, ReplacementValue); return CodeGenIP; }; auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumOuterBodiesGenerated; Builder.restoreIP(CodeGenIP); BasicBlock *CGBB = CodeGenIP.getBlock(); BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint()); BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt()); CGBB->getTerminator()->eraseFromParent(); ; NewBB1->getTerminator()->eraseFromParent(); ; IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.CreateParallel( InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); Builder.restoreIP(AfterIP1); Builder.CreateBr(NewBB1); IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.CreateParallel( InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); Builder.restoreIP(AfterIP2); Builder.CreateBr(NewBB2); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); EXPECT_EQ(NumInnerBodiesGenerated, 2U); EXPECT_EQ(NumOuterBodiesGenerated, 1U); EXPECT_EQ(NumFinalizationPoints, 3U); Builder.restoreIP(AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_EQ(M->size(), 6U); for (Function &OutlinedFn : *M) { if (F == &OutlinedFn || OutlinedFn.isDeclaration()) continue; EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasInternalLinkage()); EXPECT_EQ(OutlinedFn.arg_size(), 2U); unsigned NumAllocas = 0; for (Instruction &I : instructions(OutlinedFn)) NumAllocas += isa(I); EXPECT_EQ(NumAllocas, 1U); EXPECT_EQ(OutlinedFn.getNumUses(), 1U); User *Usr = OutlinedFn.user_back(); ASSERT_TRUE(isa(Usr)); CallInst *ForkCI = dyn_cast(Usr->user_back()); ASSERT_NE(ForkCI, nullptr); EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); EXPECT_EQ(ForkCI->getNumArgOperands(), 3U); EXPECT_TRUE(isa(ForkCI->getArgOperand(0))); EXPECT_EQ(ForkCI->getArgOperand(1), ConstantInt::get(Type::getInt32Ty(Ctx), 0U)); EXPECT_EQ(ForkCI->getArgOperand(2), Usr); } } TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); AllocaInst *PrivAI = nullptr; unsigned NumBodiesGenerated = 0; unsigned NumPrivatizedVars = 0; unsigned NumFinalizationPoints = 0; auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumBodiesGenerated; Builder.restoreIP(AllocaIP); PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); Builder.CreateStore(F->arg_begin(), PrivAI); Builder.restoreIP(CodeGenIP); Value *PrivLoad = Builder.CreateLoad(PrivAI, "local.use"); Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad); Instruction *ThenTerm, *ElseTerm; SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), &ThenTerm, &ElseTerm); Builder.SetInsertPoint(ThenTerm); Builder.CreateBr(&ContinuationIP); ThenTerm->eraseFromParent(); }; auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &VPtr, Value *&ReplacementValue) -> InsertPointTy { ++NumPrivatizedVars; if (!isa(VPtr)) { EXPECT_EQ(&VPtr, F->arg_begin()); ReplacementValue = &VPtr; return CodeGenIP; } // Trivial copy (=firstprivate). Builder.restoreIP(AllocaIP); Type *VTy = VPtr.getType()->getPointerElementType(); Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload"); ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy"); Builder.restoreIP(CodeGenIP); Builder.CreateStore(V, ReplacementValue); return CodeGenIP; }; auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; // No destructors. }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, Builder.CreateIsNotNull(F->arg_begin()), nullptr, OMP_PROC_BIND_default, false); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 1U); EXPECT_EQ(NumFinalizationPoints, 1U); Builder.restoreIP(AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_NE(PrivAI, nullptr); Function *OutlinedFn = PrivAI->getFunction(); EXPECT_NE(F, OutlinedFn); EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn->hasInternalLinkage()); EXPECT_EQ(OutlinedFn->arg_size(), 3U); EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent()); ASSERT_EQ(OutlinedFn->getNumUses(), 2U); CallInst *DirectCI = nullptr; CallInst *ForkCI = nullptr; for (User *Usr : OutlinedFn->users()) { if (isa(Usr)) { ASSERT_EQ(DirectCI, nullptr); DirectCI = cast(Usr); } else { ASSERT_TRUE(isa(Usr)); ASSERT_EQ(Usr->getNumUses(), 1U); ASSERT_TRUE(isa(Usr->user_back())); ForkCI = cast(Usr->user_back()); } } EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); EXPECT_EQ(ForkCI->getNumArgOperands(), 4U); EXPECT_TRUE(isa(ForkCI->getArgOperand(0))); EXPECT_EQ(ForkCI->getArgOperand(1), ConstantInt::get(Type::getInt32Ty(Ctx), 1)); EXPECT_EQ(ForkCI->getArgOperand(3), F->arg_begin()); EXPECT_EQ(DirectCI->getCalledFunction(), OutlinedFn); EXPECT_EQ(DirectCI->getNumArgOperands(), 3U); EXPECT_TRUE(isa(DirectCI->getArgOperand(0))); EXPECT_TRUE(isa(DirectCI->getArgOperand(1))); EXPECT_EQ(DirectCI->getArgOperand(2), F->arg_begin()); } TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); unsigned NumBodiesGenerated = 0; unsigned NumPrivatizedVars = 0; unsigned NumFinalizationPoints = 0; CallInst *CheckedBarrier = nullptr; auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &ContinuationIP) { ++NumBodiesGenerated; Builder.restoreIP(CodeGenIP); // Create three barriers, two cancel barriers but only one checked. Function *CBFn, *BFn; Builder.restoreIP( OMPBuilder.CreateBarrier(Builder.saveIP(), OMPD_parallel)); CBFn = M->getFunction("__kmpc_cancel_barrier"); BFn = M->getFunction("__kmpc_barrier"); ASSERT_NE(CBFn, nullptr); ASSERT_EQ(BFn, nullptr); ASSERT_EQ(CBFn->getNumUses(), 1U); ASSERT_TRUE(isa(CBFn->user_back())); ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U); CheckedBarrier = cast(CBFn->user_back()); Builder.restoreIP( OMPBuilder.CreateBarrier(Builder.saveIP(), OMPD_parallel, true)); CBFn = M->getFunction("__kmpc_cancel_barrier"); BFn = M->getFunction("__kmpc_barrier"); ASSERT_NE(CBFn, nullptr); ASSERT_NE(BFn, nullptr); ASSERT_EQ(CBFn->getNumUses(), 1U); ASSERT_EQ(BFn->getNumUses(), 1U); ASSERT_TRUE(isa(BFn->user_back())); ASSERT_EQ(BFn->user_back()->getNumUses(), 0U); Builder.restoreIP(OMPBuilder.CreateBarrier(Builder.saveIP(), OMPD_parallel, false, false)); ASSERT_EQ(CBFn->getNumUses(), 2U); ASSERT_EQ(BFn->getNumUses(), 1U); ASSERT_TRUE(CBFn->user_back() != CheckedBarrier); ASSERT_TRUE(isa(CBFn->user_back())); ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U); }; auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value *&) -> InsertPointTy { ++NumPrivatizedVars; llvm_unreachable("No privatization callback call expected!"); }; FunctionType *FakeDestructorTy = FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)}, /*isVarArg=*/false); auto *FakeDestructor = Function::Create( FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get()); auto FiniCB = [&](InsertPointTy IP) { ++NumFinalizationPoints; Builder.restoreIP(IP); Builder.CreateCall(FakeDestructor, {Builder.getInt32(NumFinalizationPoints)}); }; IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), F->getEntryBlock().getFirstInsertionPt()); IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, Builder.CreateIsNotNull(F->arg_begin()), nullptr, OMP_PROC_BIND_default, true); EXPECT_EQ(NumBodiesGenerated, 1U); EXPECT_EQ(NumPrivatizedVars, 0U); EXPECT_EQ(NumFinalizationPoints, 2U); EXPECT_EQ(FakeDestructor->getNumUses(), 2U); Builder.restoreIP(AfterIP); Builder.CreateRetVoid(); OMPBuilder.finalize(); EXPECT_FALSE(verifyModule(*M, &errs())); BasicBlock *ExitBB = nullptr; for (const User *Usr : FakeDestructor->users()) { const CallInst *CI = dyn_cast(Usr); ASSERT_EQ(CI->getCalledFunction(), FakeDestructor); ASSERT_TRUE(isa(CI->getNextNode())); ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U); if (ExitBB) ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB); else ExitBB = CI->getNextNode()->getSuccessor(0); ASSERT_EQ(ExitBB->size(), 1U); if (!isa(ExitBB->front())) { ASSERT_TRUE(isa(ExitBB->front())); ASSERT_EQ(cast(ExitBB->front()).getNumSuccessors(), 1U); ASSERT_TRUE(isa( cast(ExitBB->front()).getSuccessor(0)->front())); } } } TEST_F(OpenMPIRBuilderTest, MasterDirective) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); AllocaInst *PrivAI = nullptr; BasicBlock *EntryBB = nullptr; BasicBlock *ExitBB = nullptr; BasicBlock *ThenBB = nullptr; auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &FiniBB) { if (AllocaIP.isSet()) Builder.restoreIP(AllocaIP); else Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt())); PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); Builder.CreateStore(F->arg_begin(), PrivAI); llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); Builder.restoreIP(CodeGenIP); // collect some info for checks later ExitBB = FiniBB.getUniqueSuccessor(); ThenBB = Builder.GetInsertBlock(); EntryBB = ThenBB->getUniquePredecessor(); // simple instructions for body Value *PrivLoad = Builder.CreateLoad(PrivAI, "local.use"); Builder.CreateICmpNE(F->arg_begin(), PrivLoad); }; auto FiniCB = [&](InsertPointTy IP) { BasicBlock *IPBB = IP.getBlock(); EXPECT_NE(IPBB->end(), IP.getPoint()); }; Builder.restoreIP(OMPBuilder.CreateMaster(Builder, BodyGenCB, FiniCB)); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_NE(EntryBBTI, nullptr); EXPECT_TRUE(isa(EntryBBTI)); BranchInst *EntryBr = cast(EntryBB->getTerminator()); EXPECT_TRUE(EntryBr->isConditional()); EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB); EXPECT_EQ(ThenBB->getUniqueSuccessor(), ExitBB); EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB); CmpInst *CondInst = cast(EntryBr->getCondition()); EXPECT_TRUE(isa(CondInst->getOperand(0))); CallInst *MasterEntryCI = cast(CondInst->getOperand(0)); EXPECT_EQ(MasterEntryCI->getNumArgOperands(), 2U); EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master"); EXPECT_TRUE(isa(MasterEntryCI->getArgOperand(0))); CallInst *MasterEndCI = nullptr; for (auto &FI : *ThenBB) { Instruction *cur = &FI; if (isa(cur)) { MasterEndCI = cast(cur); if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master") break; MasterEndCI = nullptr; } } EXPECT_NE(MasterEndCI, nullptr); EXPECT_EQ(MasterEndCI->getNumArgOperands(), 2U); EXPECT_TRUE(isa(MasterEndCI->getArgOperand(0))); EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1)); } TEST_F(OpenMPIRBuilderTest, CriticalDirective) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); BasicBlock *EntryBB = nullptr; auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, BasicBlock &FiniBB) { // collect some info for checks later EntryBB = FiniBB.getUniquePredecessor(); // actual start for bodyCB llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); EXPECT_EQ(EntryBB, CodeGenIPBB); // body begin Builder.restoreIP(CodeGenIP); Builder.CreateStore(F->arg_begin(), PrivAI); Value *PrivLoad = Builder.CreateLoad(PrivAI, "local.use"); Builder.CreateICmpNE(F->arg_begin(), PrivLoad); }; auto FiniCB = [&](InsertPointTy IP) { BasicBlock *IPBB = IP.getBlock(); EXPECT_NE(IPBB->end(), IP.getPoint()); }; Builder.restoreIP(OMPBuilder.CreateCritical(Builder, BodyGenCB, FiniCB, "testCRT", nullptr)); Value *EntryBBTI = EntryBB->getTerminator(); EXPECT_EQ(EntryBBTI, nullptr); CallInst *CriticalEntryCI = nullptr; for (auto &EI : *EntryBB) { Instruction *cur = &EI; if (isa(cur)) { CriticalEntryCI = cast(cur); if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical") break; CriticalEntryCI = nullptr; } } EXPECT_NE(CriticalEntryCI, nullptr); EXPECT_EQ(CriticalEntryCI->getNumArgOperands(), 3U); EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical"); EXPECT_TRUE(isa(CriticalEntryCI->getArgOperand(0))); CallInst *CriticalEndCI = nullptr; for (auto &FI : *EntryBB) { Instruction *cur = &FI; if (isa(cur)) { CriticalEndCI = cast(cur); if (CriticalEndCI->getCalledFunction()->getName() == "__kmpc_end_critical") break; CriticalEndCI = nullptr; } } EXPECT_NE(CriticalEndCI, nullptr); EXPECT_EQ(CriticalEndCI->getNumArgOperands(), 3U); EXPECT_TRUE(isa(CriticalEndCI->getArgOperand(0))); EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1)); PointerType *CriticalNamePtrTy = PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8)); EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2)); EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy); } TEST_F(OpenMPIRBuilderTest, CopyinBlocks) { OpenMPIRBuilder OMPBuilder(*M); OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); IntegerType* Int32 = Type::getInt32Ty(M->getContext()); AllocaInst* MasterAddress = Builder.CreateAlloca(Int32->getPointerTo()); AllocaInst* PrivAddress = Builder.CreateAlloca(Int32->getPointerTo()); BasicBlock *EntryBB = BB; OMPBuilder.CreateCopyinClauseBlocks(Builder.saveIP(), MasterAddress, PrivAddress, Int32, /*BranchtoEnd*/true); BranchInst* EntryBr = dyn_cast_or_null(EntryBB->getTerminator()); EXPECT_NE(EntryBr, nullptr); EXPECT_TRUE(EntryBr->isConditional()); BasicBlock* NotMasterBB = EntryBr->getSuccessor(0); BasicBlock* CopyinEnd = EntryBr->getSuccessor(1); CmpInst* CMP = dyn_cast_or_null(EntryBr->getCondition()); EXPECT_NE(CMP, nullptr); EXPECT_NE(NotMasterBB, nullptr); EXPECT_NE(CopyinEnd, nullptr); BranchInst* NotMasterBr = dyn_cast_or_null(NotMasterBB->getTerminator()); EXPECT_NE(NotMasterBr, nullptr); EXPECT_FALSE(NotMasterBr->isConditional()); EXPECT_EQ(CopyinEnd,NotMasterBr->getSuccessor(0)); } } // namespace