Index: include/clang/AST/DataRecursiveASTVisitor.h =================================================================== --- include/clang/AST/DataRecursiveASTVisitor.h +++ include/clang/AST/DataRecursiveASTVisitor.h @@ -2531,7 +2531,17 @@ template bool RecursiveASTVisitor::VisitOMPLinearClause(OMPLinearClause *C) { TRY_TO(TraverseStmt(C->getStep())); + TRY_TO(TraverseStmt(C->getCalcStep())); TRY_TO(VisitOMPClauseList(C)); + for (auto *E : C->inits()) { + TRY_TO(TraverseStmt(E)); + } + for (auto *E : C->updates()) { + TRY_TO(TraverseStmt(E)); + } + for (auto *E : C->finals()) { + TRY_TO(TraverseStmt(E)); + } return true; } Index: include/clang/AST/OpenMPClause.h =================================================================== --- include/clang/AST/OpenMPClause.h +++ include/clang/AST/OpenMPClause.h @@ -1356,7 +1356,10 @@ SourceLocation ColonLoc; /// \brief Sets the linear step for clause. - void setStep(Expr *Step) { *varlist_end() = Step; } + void setStep(Expr *Step) { *(getFinals().end()) = Step; } + + /// \brief Sets the expression to calculate linear step for clause. + void setCalcStep(Expr *CalcStep) { *(getFinals().end() + 1) = CalcStep; } /// \brief Build 'linear' clause with given number of variables \a NumVars. /// @@ -1383,6 +1386,46 @@ NumVars), ColonLoc(SourceLocation()) {} + /// \brief Gets the list of initial values for linear variables. + /// + /// There are NumVars expressions with initial values allocated after the + /// varlist, they are followed by NumVars update expressions (used to update + /// the linear variable's value on current iteration) and they are followed by + /// NumVars final expressions (used to calculate the linear variable's + /// value after the loop body). After these lists, there are 2 helper + /// expressions - linear step and a helper to calculate it before the + /// loop body (used when the linear step is not constant): + /// + /// { Vars[] /* in OMPVarListClause */; Inits[]; Updates[]; Finals[]; + /// Step; CalcStep; } + /// + MutableArrayRef getInits() { + return MutableArrayRef(varlist_end(), varlist_size()); + } + ArrayRef getInits() const { + return llvm::makeArrayRef(varlist_end(), varlist_size()); + } + + /// \brief Sets the list of update expressions for linear variables. + MutableArrayRef getUpdates() { + return MutableArrayRef(getInits().end(), varlist_size()); + } + ArrayRef getUpdates() const { + return llvm::makeArrayRef(getInits().end(), varlist_size()); + } + + /// \brief Sets the list of final update expressions for linear variables. + MutableArrayRef getFinals() { + return MutableArrayRef(getUpdates().end(), varlist_size()); + } + ArrayRef getFinals() const { + return llvm::makeArrayRef(getUpdates().end(), varlist_size()); + } + + /// \brief Sets the list of the initial values for linear variables. + /// \param IL List of expressions. + void setInits(ArrayRef IL); + public: /// \brief Creates clause with a list of variables \a VL and a linear step /// \a Step. @@ -1393,11 +1436,14 @@ /// \param ColonLoc Location of ':'. /// \param EndLoc Ending location of the clause. /// \param VL List of references to the variables. + /// \param IL List of initial values for the variables. /// \param Step Linear step. + /// \param CalcStep Calculation of the linear step. static OMPLinearClause *Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, - ArrayRef VL, Expr *Step); + ArrayRef VL, ArrayRef IL, + Expr *Step, Expr *CalcStep); /// \brief Creates an empty clause with the place for \a NumVars variables. /// @@ -1412,13 +1458,61 @@ SourceLocation getColonLoc() const { return ColonLoc; } /// \brief Returns linear step. - Expr *getStep() { return *varlist_end(); } + Expr *getStep() { return *(getFinals().end()); } /// \brief Returns linear step. - const Expr *getStep() const { return *varlist_end(); } + const Expr *getStep() const { return *(getFinals().end()); } + /// \brief Returns expression to calculate linear step. + Expr *getCalcStep() { return *(getFinals().end() + 1); } + /// \brief Returns expression to calculate linear step. + const Expr *getCalcStep() const { return *(getFinals().end() + 1); } + + /// \brief Sets the list of update expressions for linear variables. + /// \param UL List of expressions. + void setUpdates(ArrayRef UL); + + /// \brief Sets the list of final update expressions for linear variables. + /// \param FL List of expressions. + void setFinals(ArrayRef FL); + + typedef MutableArrayRef::iterator inits_iterator; + typedef ArrayRef::iterator inits_const_iterator; + typedef llvm::iterator_range inits_range; + typedef llvm::iterator_range inits_const_range; + + inits_range inits() { + return inits_range(getInits().begin(), getInits().end()); + } + inits_const_range inits() const { + return inits_const_range(getInits().begin(), getInits().end()); + } + + typedef MutableArrayRef::iterator updates_iterator; + typedef ArrayRef::iterator updates_const_iterator; + typedef llvm::iterator_range updates_range; + typedef llvm::iterator_range updates_const_range; + + updates_range updates() { + return updates_range(getUpdates().begin(), getUpdates().end()); + } + updates_const_range updates() const { + return updates_const_range(getUpdates().begin(), getUpdates().end()); + } + + typedef MutableArrayRef::iterator finals_iterator; + typedef ArrayRef::iterator finals_const_iterator; + typedef llvm::iterator_range finals_range; + typedef llvm::iterator_range finals_const_range; + + finals_range finals() { + return finals_range(getFinals().begin(), getFinals().end()); + } + finals_const_range finals() const { + return finals_const_range(getFinals().begin(), getFinals().end()); + } StmtRange children() { return StmtRange(reinterpret_cast(varlist_begin()), - reinterpret_cast(varlist_end() + 1)); + reinterpret_cast(getFinals().end() + 2)); } static bool classof(const OMPClause *T) { Index: include/clang/AST/RecursiveASTVisitor.h =================================================================== --- include/clang/AST/RecursiveASTVisitor.h +++ include/clang/AST/RecursiveASTVisitor.h @@ -2561,7 +2561,17 @@ template bool RecursiveASTVisitor::VisitOMPLinearClause(OMPLinearClause *C) { TRY_TO(TraverseStmt(C->getStep())); + TRY_TO(TraverseStmt(C->getCalcStep())); TRY_TO(VisitOMPClauseList(C)); + for (auto *E : C->inits()) { + TRY_TO(TraverseStmt(E)); + } + for (auto *E : C->updates()) { + TRY_TO(TraverseStmt(E)); + } + for (auto *E : C->finals()) { + TRY_TO(TraverseStmt(E)); + } return true; } Index: include/clang/AST/StmtOpenMP.h =================================================================== --- include/clang/AST/StmtOpenMP.h +++ include/clang/AST/StmtOpenMP.h @@ -95,6 +95,7 @@ /// This iterator visits only those declarations that meet some run-time /// criteria. template class filtered_clause_iterator { + protected: ArrayRef::const_iterator Current; ArrayRef::const_iterator End; FilterPredicate Pred; @@ -126,6 +127,27 @@ bool operator!() { return Current == End; } operator bool() { return Current != End; } + bool empty() const { return Current == End; } + }; + + /// \brief A filter to iterate over 'linear' clauses using a C++ range + /// for loop. + struct linear_filter : public filtered_clause_iterator< + std::function > { + linear_filter(ArrayRef Arr) + : filtered_clause_iterator(Arr, [](const OMPClause *C)->bool { + return C->getClauseKind() == OMPC_linear; + }) {} + const OMPLinearClause *operator*() const { + return cast(*Current); + } + const OMPLinearClause *operator->() const { + return cast(*Current); + } + friend linear_filter begin(const linear_filter &range) { return range; } + friend linear_filter end(const linear_filter &range) { + return linear_filter(ArrayRef(range.End, range.End)); + } }; /// \brief Gets a single clause of the specified kind \a K associated with the @@ -410,6 +432,8 @@ Expr *IterationVarRef; /// \brief Loop last iteration number. Expr *LastIteration; + /// \brief Loop number of iterations. + Expr *NumIterations; /// \brief Calculation of last iteration. Expr *CalcLastIteration; /// \brief Loop pre-condition. @@ -447,8 +471,9 @@ /// worksharing ones). bool builtAll() { return IterationVarRef != nullptr && LastIteration != nullptr && - PreCond != nullptr && Cond != nullptr && - SeparatedCond != nullptr && Init != nullptr && Inc != nullptr; + NumIterations != nullptr && PreCond != nullptr && + Cond != nullptr && SeparatedCond != nullptr && Init != nullptr && + Inc != nullptr; } /// \brief Initialize all the fields to null. Index: lib/AST/Stmt.cpp =================================================================== --- lib/AST/Stmt.cpp +++ lib/AST/Stmt.cpp @@ -1291,27 +1291,56 @@ return new (Mem) OMPSharedClause(N); } -OMPLinearClause *OMPLinearClause::Create(const ASTContext &C, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ColonLoc, - SourceLocation EndLoc, - ArrayRef VL, Expr *Step) { +void OMPLinearClause::setInits(ArrayRef IL) { + assert(IL.size() == varlist_size() && + "Number of inits is not the same as the preallocated buffer"); + std::copy(IL.begin(), IL.end(), varlist_end()); +} + +void OMPLinearClause::setUpdates(ArrayRef UL) { + assert(UL.size() == varlist_size() && + "Number of updates is not the same as the preallocated buffer"); + std::copy(UL.begin(), UL.end(), getInits().end()); +} + +void OMPLinearClause::setFinals(ArrayRef FL) { + assert(FL.size() == varlist_size() && + "Number of final updates is not the same as the preallocated buffer"); + std::copy(FL.begin(), FL.end(), getUpdates().end()); +} + +OMPLinearClause * +OMPLinearClause::Create(const ASTContext &C, SourceLocation StartLoc, + SourceLocation LParenLoc, SourceLocation ColonLoc, + SourceLocation EndLoc, ArrayRef VL, + ArrayRef IL, Expr *Step, Expr *CalcStep) { + // Allocate space for 4 lists (Vars, Inits, Updates, Finals) and 2 expressions + // (Step and CalcStep). void *Mem = C.Allocate(llvm::RoundUpToAlignment(sizeof(OMPLinearClause), llvm::alignOf()) + - sizeof(Expr *) * (VL.size() + 1)); + (4 * VL.size() + 2) * sizeof(Expr *)); OMPLinearClause *Clause = new (Mem) OMPLinearClause(StartLoc, LParenLoc, ColonLoc, EndLoc, VL.size()); Clause->setVarRefs(VL); + Clause->setInits(IL); + // Fill update and final expressions with zeroes, they are provided later, + // after the directive construction. + std::fill(Clause->getInits().end(), Clause->getInits().end() + VL.size(), + nullptr); + std::fill(Clause->getUpdates().end(), Clause->getUpdates().end() + VL.size(), + nullptr); Clause->setStep(Step); + Clause->setCalcStep(CalcStep); return Clause; } OMPLinearClause *OMPLinearClause::CreateEmpty(const ASTContext &C, unsigned NumVars) { + // Allocate space for 4 lists (Vars, Inits, Updates, Finals) and 2 expressions + // (Step and CalcStep). void *Mem = C.Allocate(llvm::RoundUpToAlignment(sizeof(OMPLinearClause), llvm::alignOf()) + - sizeof(Expr *) * (NumVars + 1)); + (4 * NumVars + 2) * sizeof(Expr *)); return new (Mem) OMPLinearClause(NumVars); } Index: lib/AST/StmtProfile.cpp =================================================================== --- lib/AST/StmtProfile.cpp +++ lib/AST/StmtProfile.cpp @@ -359,7 +359,17 @@ } void OMPClauseProfiler::VisitOMPLinearClause(const OMPLinearClause *C) { VisitOMPClauseList(C); + for (auto *E : C->inits()) { + Profiler->VisitStmt(E); + } + for (auto *E : C->updates()) { + Profiler->VisitStmt(E); + } + for (auto *E : C->finals()) { + Profiler->VisitStmt(E); + } Profiler->VisitStmt(C->getStep()); + Profiler->VisitStmt(C->getCalcStep()); } void OMPClauseProfiler::VisitOMPAlignedClause(const OMPAlignedClause *C) { VisitOMPClauseList(C); Index: lib/CodeGen/CGStmtOpenMP.cpp =================================================================== --- lib/CodeGen/CGStmtOpenMP.cpp +++ lib/CodeGen/CGStmtOpenMP.cpp @@ -266,6 +266,13 @@ for (auto I : S.updates()) { EmitIgnoredExpr(I); } + // Update the linear variables. + for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { + for (auto U : C->updates()) { + EmitIgnoredExpr(U); + } + } + // On a continue in the body, jump to the end. auto Continue = getJumpDestInCurrentScope("omp.body.continue"); BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); @@ -336,6 +343,12 @@ } ++IC; } + // Emit the final values of the linear variables. + for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { + for (auto F : C->finals()) { + EmitIgnoredExpr(F); + } + } } static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, @@ -381,6 +394,25 @@ } } +static void +EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, + CodeGenFunction::OMPPrivateScope &PrivateScope) { + for (auto Clause : OMPExecutableDirective::linear_filter(D.clauses())) { + for (auto *E : Clause->varlists()) { + auto VD = cast(cast(E)->getDecl()); + bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * { + // Emit var without initialization. + auto VarEmission = CGF.EmitAutoVarAlloca(*VD); + CGF.EmitAutoVarCleanups(VarEmission); + return VarEmission.getAllocatedAddress(); + }); + assert(IsRegistered && "linear var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } + } +} + void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { // Pragma 'simd' code depends on presence of 'lastprivate'. // If present, we have to separate last iteration of the loop: @@ -428,6 +460,14 @@ InlinedOpenMPRegionScopeRAII Region(*this, S); + // Emit inits for the linear variables. + for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { + for (auto Init : C->inits()) { + auto *D = cast(cast(Init)->getDecl()); + EmitVarDecl(*D); + } + } + // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); @@ -443,6 +483,17 @@ EmitIgnoredExpr(S.getCalcLastIteration()); } + // Emit the linear steps for the linear clauses. + // If a step is not constant, it is pre-calculated before the loop. + for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { + if (auto CS = cast_or_null(C->getCalcStep())) + if (auto SaveRef = cast(CS->getLHS())) { + EmitVarDecl(*cast(SaveRef->getDecl())); + // Emit calculation of the linear step. + EmitIgnoredExpr(CS); + } + } + if (SeparateIter) { // Emit: if (LastIteration > 0) - begin. RegionCounter Cnt = getPGORegionCounter(&S); @@ -455,6 +506,7 @@ { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); + EmitPrivateLinearVars(*this, S, LoopScope); EmitOMPPrivateClause(S, LoopScope); (void)LoopScope.Privatize(); EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), @@ -473,6 +525,7 @@ { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); + EmitPrivateLinearVars(*this, S, LoopScope); EmitOMPPrivateClause(S, LoopScope); (void)LoopScope.Privatize(); EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Index: lib/Sema/SemaOpenMP.cpp =================================================================== --- lib/Sema/SemaOpenMP.cpp +++ lib/Sema/SemaOpenMP.cpp @@ -622,6 +622,10 @@ PopExpressionEvaluationContext(); } +static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV, + Expr *NumIterations, Sema &SemaRef, + Scope *S); + namespace { class VarDeclFilterCCC : public CorrectionCandidateCallback { @@ -2915,6 +2919,7 @@ // Save results Built.IterationVarRef = IV.get(); Built.LastIteration = LastIteration.get(); + Built.NumIterations = NumIterations.get(); Built.CalcLastIteration = CalcLastIteration.get(); Built.PreCond = PreCond.get(); Built.Cond = Cond.get(); @@ -2958,6 +2963,16 @@ assert((CurContext->isDependentContext() || B.builtAll()) && "omp simd loop exprs were not built"); + if (!CurContext->isDependentContext()) { + // Finalize the clauses that need pre-built expressions for CodeGen. + for (auto C : Clauses) { + if (auto LC = dyn_cast(C)) + if (FinishOpenMPLinearClause(*LC, cast(B.IterationVarRef), + B.NumIterations, *this, CurScope)) + return StmtError(); + } + } + getCurFunction()->setHasBranchProtectedScope(); return OMPSimdDirective::Create(Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B); @@ -5231,11 +5246,13 @@ SourceLocation ColonLoc, SourceLocation EndLoc) { SmallVector Vars; + SmallVector Inits; for (auto &RefExpr : VarList) { assert(RefExpr && "NULL expr in OpenMP linear clause."); if (isa(RefExpr)) { // It will be analyzed later. Vars.push_back(RefExpr); + Inits.push_back(nullptr); continue; } @@ -5277,6 +5294,7 @@ if (QType->isDependentType() || QType->isInstantiationDependentType()) { // It will be analyzed later. Vars.push_back(DE); + Inits.push_back(nullptr); continue; } @@ -5322,14 +5340,27 @@ continue; } + // Build var to save initial value. + VarDecl *Init = BuildVarDecl(*this, ELoc, DE->getType(), ".linear.start"); + AddInitializerToDecl(Init, DefaultLvalueConversion(DE).get(), + /*DirectInit*/ false, /*TypeMayContainAuto*/ false); + CurContext->addDecl(Init); + Init->setIsUsed(); + auto InitRef = DeclRefExpr::Create( + Context, /*QualifierLoc*/ NestedNameSpecifierLoc(), + /*TemplateKWLoc*/ SourceLocation(), Init, + /*isEnclosingLocal*/ false, DE->getLocStart(), DE->getType(), + /*VK*/ VK_LValue); DSAStack->addDSA(VD, DE, OMPC_linear); Vars.push_back(DE); + Inits.push_back(InitRef); } if (Vars.empty()) return nullptr; Expr *StepExpr = Step; + Expr *CalcStepExpr = nullptr; if (Step && !Step->isValueDependent() && !Step->isTypeDependent() && !Step->isInstantiationDependent() && !Step->containsUnexpandedParameterPack()) { @@ -5339,17 +5370,85 @@ return nullptr; StepExpr = Val.get(); + // Build var to save the step value. + VarDecl *SaveVar = + BuildVarDecl(*this, StepLoc, StepExpr->getType(), ".linear.step"); + CurContext->addDecl(SaveVar); + SaveVar->setIsUsed(); + ExprResult SaveRef = + BuildDeclRefExpr(SaveVar, StepExpr->getType(), VK_LValue, StepLoc); + ExprResult CalcStep = + BuildBinOp(CurScope, StepLoc, BO_Assign, SaveRef.get(), StepExpr); + // Warn about zero linear step (it would be probably better specified as // making corresponding variables 'const'). llvm::APSInt Result; - if (StepExpr->isIntegerConstantExpr(Result, Context) && - !Result.isNegative() && !Result.isStrictlyPositive()) + bool IsConstant = StepExpr->isIntegerConstantExpr(Result, Context); + if (IsConstant && !Result.isNegative() && !Result.isStrictlyPositive()) Diag(StepLoc, diag::warn_omp_linear_step_zero) << Vars[0] << (Vars.size() > 1); + if (!IsConstant && CalcStep.isUsable()) { + // Calculate the step beforehand instead of doing this on each iteration. + // (This is not used if the number of iterations may be kfold-ed). + CalcStepExpr = CalcStep.get(); + } } return OMPLinearClause::Create(Context, StartLoc, LParenLoc, ColonLoc, EndLoc, - Vars, StepExpr); + Vars, Inits, StepExpr, CalcStepExpr); +} + +static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV, + Expr *NumIterations, Sema &SemaRef, + Scope *S) { + // Walk the vars and build update/final expressions for the CodeGen. + SmallVector Updates; + SmallVector Finals; + Expr *Step = Clause.getStep(); + Expr *CalcStep = Clause.getCalcStep(); + // OpenMP [2.14.3.7, linear clause] + // If linear-step is not specified it is assumed to be 1. + if (Step == nullptr) + Step = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(); + else if (CalcStep) + Step = cast(CalcStep)->getLHS(); + bool HasErrors = false; + auto CurInit = Clause.inits().begin(); + for (auto &RefExpr : Clause.varlists()) { + Expr *InitExpr = *CurInit; + + // Build privatized reference to the current linear var. + auto DE = cast(RefExpr); + auto PrivateRef = DeclRefExpr::Create( + SemaRef.Context, /*QualifierLoc*/ DE->getQualifierLoc(), + /*TemplateKWLoc*/ SourceLocation(), DE->getDecl(), + /* RefersToEnclosingVariableOrCapture */ true, DE->getLocStart(), + DE->getType(), /*VK*/ VK_LValue); + + // Build update: Var = InitExpr + IV * Step + ExprResult Update = + BuildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), PrivateRef, + InitExpr, IV, Step, /* Subtract */ false); + Update = SemaRef.ActOnFinishFullExpr(Update.get()); + + // Build final: Var = InitExpr + NumIterations * Step + ExprResult Final = + BuildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), RefExpr, InitExpr, + NumIterations, Step, /* Subtract */ false); + Final = SemaRef.ActOnFinishFullExpr(Final.get()); + if (!Update.isUsable() || !Final.isUsable()) { + Updates.push_back(nullptr); + Finals.push_back(nullptr); + HasErrors = true; + } else { + Updates.push_back(Update.get()); + Finals.push_back(Final.get()); + } + ++CurInit; + } + Clause.setUpdates(Updates); + Clause.setFinals(Finals); + return HasErrors; } OMPClause *Sema::ActOnOpenMPAlignedClause( Index: lib/Serialization/ASTReaderStmt.cpp =================================================================== --- lib/Serialization/ASTReaderStmt.cpp +++ lib/Serialization/ASTReaderStmt.cpp @@ -1928,7 +1928,20 @@ for (unsigned i = 0; i != NumVars; ++i) Vars.push_back(Reader->Reader.ReadSubExpr()); C->setVarRefs(Vars); + Vars.clear(); + for (unsigned i = 0; i != NumVars; ++i) + Vars.push_back(Reader->Reader.ReadSubExpr()); + C->setInits(Vars); + Vars.clear(); + for (unsigned i = 0; i != NumVars; ++i) + Vars.push_back(Reader->Reader.ReadSubExpr()); + C->setUpdates(Vars); + Vars.clear(); + for (unsigned i = 0; i != NumVars; ++i) + Vars.push_back(Reader->Reader.ReadSubExpr()); + C->setFinals(Vars); C->setStep(Reader->Reader.ReadSubExpr()); + C->setCalcStep(Reader->Reader.ReadSubExpr()); } void OMPClauseReader::VisitOMPAlignedClause(OMPAlignedClause *C) { Index: lib/Serialization/ASTWriterStmt.cpp =================================================================== --- lib/Serialization/ASTWriterStmt.cpp +++ lib/Serialization/ASTWriterStmt.cpp @@ -1820,9 +1820,20 @@ Record.push_back(C->varlist_size()); Writer->Writer.AddSourceLocation(C->getLParenLoc(), Record); Writer->Writer.AddSourceLocation(C->getColonLoc(), Record); - for (auto *VE : C->varlists()) + for (auto *VE : C->varlists()) { + Writer->Writer.AddStmt(VE); + } + for (auto *VE : C->inits()) { Writer->Writer.AddStmt(VE); + } + for (auto *VE : C->updates()) { + Writer->Writer.AddStmt(VE); + } + for (auto *VE : C->finals()) { + Writer->Writer.AddStmt(VE); + } Writer->Writer.AddStmt(C->getStep()); + Writer->Writer.AddStmt(C->getCalcStep()); } void OMPClauseWriter::VisitOMPAlignedClause(OMPAlignedClause *C) { Index: test/OpenMP/simd_codegen.cpp =================================================================== --- test/OpenMP/simd_codegen.cpp +++ test/OpenMP/simd_codegen.cpp @@ -7,6 +7,9 @@ #ifndef HEADER #define HEADER +long long get_val() { return 0; } +double *g_ptr; + // CHECK-LABEL: define {{.*void}} @{{.*}}simple{{.*}}(float* {{.+}}, float* {{.+}}, float* {{.+}}, float* {{.+}}) void simple(float *a, float *b, float *c, float *d) { #pragma omp simd @@ -33,7 +36,13 @@ } // CHECK: [[SIMPLE_LOOP1_END]] - #pragma omp simd + long long k = get_val(); + + #pragma omp simd linear(k : 3) +// CHECK: [[K0:%.+]] = call {{.*}}i64 @{{.*}}get_val +// CHECK-NEXT: store i64 [[K0]], i64* [[K_VAR:%[^,]+]] +// CHECK: [[K0LOAD:%.+]] = load i64, i64* [[K_VAR]] +// CHECK-NEXT: store i64 [[K0LOAD]], i64* [[LIN0:%[^,]+]] // CHECK: store i32 0, i32* [[OMP_IV2:%[^,]+]] // CHECK: [[IV2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID:[0-9]+]] @@ -47,17 +56,45 @@ // CHECK-NEXT: [[IV2_1:%.+]] = mul nsw i32 [[IV2_0]], 1 // CHECK-NEXT: [[LC_I_1:%.+]] = sub nsw i32 10, [[IV2_1]] // CHECK-NEXT: store i32 [[LC_I_1]], i32* {{.+}}, !llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]] - a[i]++; +// +// CHECK-NEXT: [[LIN0_1:%.+]] = load i64, i64* [[LIN0]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]] +// CHECK-NEXT: [[IV2_2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]] +// CHECK-NEXT: [[LIN_MUL1:%.+]] = mul nsw i32 [[IV2_2]], 3 +// CHECK-NEXT: [[LIN_EXT1:%.+]] = sext i32 [[LIN_MUL1]] to i64 +// CHECK-NEXT: [[LIN_ADD1:%.+]] = add nsw i64 [[LIN0_1]], [[LIN_EXT1]] +// Update of the privatized version of linear variable! +// CHECK-NEXT: store i64 [[LIN_ADD1]], i64* [[K_PRIVATIZED:%[^,]+]] + a[k]++; + k = k + 3; // CHECK: [[IV2_2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]] // CHECK-NEXT: [[ADD2_2:%.+]] = add nsw i32 [[IV2_2]], 1 // CHECK-NEXT: store i32 [[ADD2_2]], i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]] // br label {{.+}}, !llvm.loop ![[SIMPLE_LOOP2_ID]] } // CHECK: [[SIMPLE_LOOP2_END]] +// +// Update linear vars after loop, as the loop was operating on a private version. +// CHECK: [[LIN0_2:%.+]] = load i64, i64* [[LIN0]] +// CHECK-NEXT: [[LIN_ADD2:%.+]] = add nsw i64 [[LIN0_2]], 27 +// CHECK-NEXT: store i64 [[LIN_ADD2]], i64* [[K_VAR]] +// + + int lin = 12; + #pragma omp simd linear(lin : get_val()), linear(g_ptr) + +// Init linear private var. +// CHECK: store i32 12, i32* [[LIN_VAR:%[^,]+]] +// CHECK: [[LIN_LOAD:%.+]] = load i32, i32* [[LIN_VAR]] +// CHECK-NEXT: store i32 [[LIN_LOAD]], i32* [[LIN_START:%[^,]+]] +// CHECK: [[GLIN_LOAD:%.+]] = load double*, double** [[GLIN_VAR:@[^,]+]] +// CHECK-NEXT: store double* [[GLIN_LOAD]], double** [[GLIN_START:%[^,]+]] - #pragma omp simd // CHECK: store i64 0, i64* [[OMP_IV3:%[^,]+]] +// Remember linear step. +// CHECK: [[CALL_VAL:%.+]] = invoke +// CHECK: store i64 [[CALL_VAL]], i64* [[LIN_STEP:%[^,]+]] + // CHECK: [[IV3:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID:[0-9]+]] // CHECK-NEXT: [[CMP3:%.+]] = icmp ult i64 [[IV3]], 4 // CHECK-NEXT: br i1 [[CMP3]], label %[[SIMPLE_LOOP3_BODY:.+]], label %[[SIMPLE_LOOP3_END:[^,]+]] @@ -68,12 +105,34 @@ // CHECK-NEXT: [[LC_IT_1:%.+]] = mul i64 [[IV3_0]], 400 // CHECK-NEXT: [[LC_IT_2:%.+]] = sub i64 2000, [[LC_IT_1]] // CHECK-NEXT: store i64 [[LC_IT_2]], i64* {{.+}}, !llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] - a[it]++; +// +// Linear start and step are used to calculate current value of the linear variable. +// CHECK: [[LINSTART:.+]] = load i32, i32* [[LIN_START]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK: [[LINSTEP:.+]] = load i64, i64* [[LIN_STEP]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK-NOT: store i32 {{.+}}, i32* [[LIN_VAR]],{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK: [[GLINSTART:.+]] = load double*, double** [[GLIN_START]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK-NEXT: [[IV3_1:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK-NEXT: [[MUL:%.+]] = mul i64 [[IV3_1]], 1 +// CHECK-NEXT: [[GEP:%.+]] = getelementptr{{.*}}[[GLINSTART]]{{.*}}[[MUL]] +// CHECK-NEXT: store double* [[GEP]], double** [[G_PTR_CUR:%[^,]+]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] + *g_ptr++ = 0.0; +// CHECK: [[GEP_VAL:%.+]] = load double{{.*}}[[G_PTR_CUR]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] +// CHECK: store double{{.*}}[[GEP_VAL]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] + a[it + lin]++; +// CHECK: [[FLT_INC:%.+]] = fadd float +// CHECK-NEXT: store float [[FLT_INC]],{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] // CHECK: [[IV3_2:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] // CHECK-NEXT: [[ADD3_2:%.+]] = add i64 [[IV3_2]], 1 // CHECK-NEXT: store i64 [[ADD3_2]], i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]] } // CHECK: [[SIMPLE_LOOP3_END]] +// +// Linear start and step are used to calculate final value of the linear variables. +// CHECK: [[LINSTART:.+]] = load i32, i32* [[LIN_START]] +// CHECK: [[LINSTEP:.+]] = load i64, i64* [[LIN_STEP]] +// CHECK: store i32 {{.+}}, i32* [[LIN_VAR]], +// CHECK: [[GLINSTART:.+]] = load double*, double** [[GLIN_START]] +// CHECK: store double* {{.*}}[[GLIN_VAR]] #pragma omp simd // CHECK: store i32 0, i32* [[OMP_IV4:%[^,]+]] Index: tools/libclang/CIndex.cpp =================================================================== --- tools/libclang/CIndex.cpp +++ tools/libclang/CIndex.cpp @@ -2031,7 +2031,17 @@ } void OMPClauseEnqueue::VisitOMPLinearClause(const OMPLinearClause *C) { VisitOMPClauseList(C); + for (const auto *E : C->inits()) { + Visitor->AddStmt(E); + } + for (const auto *E : C->updates()) { + Visitor->AddStmt(E); + } + for (const auto *E : C->finals()) { + Visitor->AddStmt(E); + } Visitor->AddStmt(C->getStep()); + Visitor->AddStmt(C->getCalcStep()); } void OMPClauseEnqueue::VisitOMPAlignedClause(const OMPAlignedClause *C) { VisitOMPClauseList(C);