Index: include/clang/Basic/Attr.td =================================================================== --- include/clang/Basic/Attr.td +++ include/clang/Basic/Attr.td @@ -2868,6 +2868,8 @@ /// unroll_and_jam: attempt to unroll and jam loop if State == Enable. /// unroll_and_jam_count: unroll and jams loop 'Value' times. /// distribute: attempt to distribute loop if State == Enable + /// pipeline: disable pipelining loop if State == Disable + /// pipeline_ii_count: try to pipeline loop for only 'Value' value /// #pragma unroll directive /// : fully unrolls loop. @@ -2882,10 +2884,10 @@ let Args = [EnumArgument<"Option", "OptionType", ["vectorize", "vectorize_width", "interleave", "interleave_count", "unroll", "unroll_count", "unroll_and_jam", "unroll_and_jam_count", - "distribute"], + "pipeline", "pipeline_ii_count", "distribute"], ["Vectorize", "VectorizeWidth", "Interleave", "InterleaveCount", "Unroll", "UnrollCount", "UnrollAndJam", "UnrollAndJamCount", - "Distribute"]>, + "PipelineDisabled", "PipelineIICount", "Distribute"]>, EnumArgument<"State", "LoopHintState", ["enable", "disable", "numeric", "assume_safety", "full"], ["Enable", "Disable", "Numeric", "AssumeSafety", "Full"]>, @@ -2902,6 +2904,8 @@ case UnrollCount: return "unroll_count"; case UnrollAndJam: return "unroll_and_jam"; case UnrollAndJamCount: return "unroll_and_jam_count"; + case PipelineDisabled: return "pipeline"; + case PipelineIICount: return "pipeline_ii_count"; case Distribute: return "distribute"; } llvm_unreachable("Unhandled LoopHint option."); Index: include/clang/Basic/AttrDocs.td =================================================================== --- include/clang/Basic/AttrDocs.td +++ include/clang/Basic/AttrDocs.td @@ -2579,9 +2579,9 @@ let Content = [{ The ``#pragma clang loop`` directive allows loop optimization hints to be specified for the subsequent loop. The directive allows vectorization, -interleaving, and unrolling to be enabled or disabled. Vector width as well -as interleave and unrolling count can be manually specified. See -`language extensions +interleaving, and unrolling to be enabled or disabled. Pipelining could be disabled. +Vector width as well as interleave, unrolling count and Initiation interval for pipelining +can be manually specified. See `language extensions `_ for details. }]; @@ -2642,6 +2642,43 @@ }]; } +def PipelineHintDocs : Documentation { + let Category = DocCatStmt; + let Heading = "#pragma clang loop pipeline, #pragma clang loop pipeline_ii_count"; + let Content = [{ +Loop pipelining optimization hints can be specified with ``#pragma clang loop pipeline`` and +``#pragma loop pipeline_ii_count``. The pragma is placed immediately before a for, while, +do-while, or c++11 range-based for loop. + +Specifying ``#pragma clang loop pipeline(disable)`` avoids software pipelining optimization. +Only `disable` state could be specified for ``#pragma clang loop pipeline``: + +.. code-block:: c++ + + #pragma clang loop pipeline(disable) + for (...) { + ... + } + +Specifying the ii count parameter for ``#pragma loop pipeline_ii_count`` instructs software +pipeliner to use only specified initiation interval : + +.. code-block:: c++ + + #pragma loop pipeline_ii_count(10) + for (...) { + ... + } + +See +`language extensions +`_ +for further details including limitations of the pipeline hints. + }]; +} + + + def OpenCLUnrollHintDocs : Documentation { let Category = DocCatStmt; let Content = [{ Index: include/clang/Basic/DiagnosticParseKinds.td =================================================================== --- include/clang/Basic/DiagnosticParseKinds.td +++ include/clang/Basic/DiagnosticParseKinds.td @@ -1175,7 +1175,7 @@ "'enable'%select{|, 'full'}1%select{|, 'assume_safety'}2 or 'disable'}0">; def err_pragma_loop_invalid_option : Error< "%select{invalid|missing}0 option%select{ %1|}0; expected vectorize, " - "vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute">; + "vectorize_width, interleave, interleave_count, unroll, unroll_count, pipeline, pipeline_ii_count or distribute">; def err_pragma_fp_invalid_option : Error< "%select{invalid|missing}0 option%select{ %1|}0; expected contract">; @@ -1188,6 +1188,8 @@ def err_pragma_invalid_keyword : Error< "invalid argument; expected 'enable'%select{|, 'full'}0%select{|, 'assume_safety'}1 or 'disable'">; +def err_pragma_pipeline_invalid_keyword : Error< + "invalid argument; expected 'disable'">; // Pragma unroll support. def warn_pragma_unroll_cuda_value_in_parens : Warning< Index: lib/CodeGen/CGLoopInfo.h =================================================================== --- lib/CodeGen/CGLoopInfo.h +++ lib/CodeGen/CGLoopInfo.h @@ -66,6 +66,12 @@ /// Value for llvm.loop.distribute.enable metadata. LVEnableState DistributeEnable; + + /// Value for llvm.loop.pipeline.disable metadata + bool PipelineDisabled; + + /// Value for llvm.loop.pipeline.iicount metadata + unsigned PipelineIICount; }; /// Information used when generating a structured loop. @@ -166,6 +172,12 @@ /// \brief Set the unroll count for the next loop pushed. void setUnrollAndJamCount(unsigned C) { StagedAttrs.UnrollAndJamCount = C; } + /// Set the pipeline disabled state + void setPipelineDisabled(bool S) { StagedAttrs.PipelineDisabled = S; } + + /// Set the pipeline ii count + void setPipelineIICount(unsigned C) { StagedAttrs.PipelineIICount = C; } + private: /// Returns true if there is LoopInfo on the stack. bool hasInfo() const { return !Active.empty(); } Index: lib/CodeGen/CGLoopInfo.cpp =================================================================== --- lib/CodeGen/CGLoopInfo.cpp +++ lib/CodeGen/CGLoopInfo.cpp @@ -26,6 +26,8 @@ if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.UnrollAndJamCount == 0 && + Attrs.PipelineDisabled == false && + Attrs.PipelineIICount == 0 && Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && @@ -122,6 +124,20 @@ Args.push_back(MDNode::get(Ctx, Vals)); } + if(Attrs.PipelineDisabled){ + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.pipeline.disable"), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + + if(Attrs.PipelineIICount > 0){ + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.pipeline.iicount"), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt32Ty(Ctx), Attrs.PipelineIICount))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); @@ -133,7 +149,8 @@ UnrollEnable(LoopAttributes::Unspecified), UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0), InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0), - DistributeEnable(LoopAttributes::Unspecified) {} + DistributeEnable(LoopAttributes::Unspecified), + PipelineDisabled(false), PipelineIICount(0) {} void LoopAttributes::clear() { IsParallel = false; @@ -145,6 +162,8 @@ UnrollEnable = LoopAttributes::Unspecified; UnrollAndJamEnable = LoopAttributes::Unspecified; DistributeEnable = LoopAttributes::Unspecified; + PipelineDisabled = false; + PipelineIICount = 0; } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, @@ -223,10 +242,14 @@ case LoopHintAttr::Distribute: setDistributeState(false); break; + case LoopHintAttr::PipelineDisabled: + setPipelineDisabled(true); + break; case LoopHintAttr::UnrollCount: case LoopHintAttr::UnrollAndJamCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: + case LoopHintAttr::PipelineIICount: llvm_unreachable("Options cannot be disabled."); break; } @@ -250,6 +273,8 @@ case LoopHintAttr::UnrollAndJamCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineIICount: llvm_unreachable("Options cannot enabled."); break; } @@ -269,6 +294,8 @@ case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineIICount: llvm_unreachable("Options cannot be used to assume mem safety."); break; } @@ -288,6 +315,8 @@ case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineIICount: llvm_unreachable("Options cannot be used with 'full' hint."); break; } @@ -306,11 +335,15 @@ case LoopHintAttr::UnrollAndJamCount: setUnrollAndJamCount(ValueInt); break; + case LoopHintAttr::PipelineIICount: + setPipelineIICount(ValueInt); + break; case LoopHintAttr::Unroll: case LoopHintAttr::UnrollAndJam: case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: llvm_unreachable("Options cannot be assigned a value."); break; } Index: lib/Parse/ParsePragma.cpp =================================================================== --- lib/Parse/ParsePragma.cpp +++ lib/Parse/ParsePragma.cpp @@ -1057,20 +1057,23 @@ bool OptionUnroll = false; bool OptionUnrollAndJam = false; bool OptionDistribute = false; + bool OptionPipelineDisabled = false; bool StateOption = false; if (OptionInfo) { // Pragma Unroll does not specify an option. OptionUnroll = OptionInfo->isStr("unroll"); OptionUnrollAndJam = OptionInfo->isStr("unroll_and_jam"); OptionDistribute = OptionInfo->isStr("distribute"); + OptionPipelineDisabled = OptionInfo->isStr("pipeline"); StateOption = llvm::StringSwitch(OptionInfo->getName()) .Case("vectorize", true) .Case("interleave", true) .Default(false) || - OptionUnroll || OptionUnrollAndJam || OptionDistribute; + OptionUnroll || OptionUnrollAndJam || + OptionDistribute || OptionPipelineDisabled; } bool AssumeSafetyArg = - !OptionUnroll && !OptionUnrollAndJam && !OptionDistribute; + !OptionUnroll && !OptionUnrollAndJam && !OptionDistribute && !OptionPipelineDisabled; // Verify loop hint has an argument. if (Toks[0].is(tok::eof)) { ConsumeAnnotationToken(); @@ -1089,14 +1092,19 @@ bool Valid = StateInfo && llvm::StringSwitch(StateInfo->getName()) - .Cases("enable", "disable", true) + .Case("disable", true) + .Case("enable", !OptionPipelineDisabled) .Case("full", OptionUnroll || OptionUnrollAndJam) .Case("assume_safety", AssumeSafetyArg) .Default(false); if (!Valid) { - Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword) - << /*FullKeyword=*/(OptionUnroll || OptionUnrollAndJam) - << /*AssumeSafetyKeyword=*/AssumeSafetyArg; + if(OptionPipelineDisabled) { + Diag(Toks[0].getLocation(), diag::err_pragma_pipeline_invalid_keyword); + } else { + Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword) + << /*FullKeyword=*/(OptionUnroll || OptionUnrollAndJam) + << /*AssumeSafetyKeyword=*/AssumeSafetyArg; + } return false; } if (Toks.size() > 2) @@ -2809,6 +2817,8 @@ /// 'vectorize_width' '(' loop-hint-value ')' /// 'interleave_count' '(' loop-hint-value ')' /// 'unroll_count' '(' loop-hint-value ')' +/// 'pipeline' '(' disable ')' +/// 'pipeline_ii_count' '(' loop-hint-value ')' /// /// loop-hint-keyword: /// 'enable' @@ -2868,6 +2878,8 @@ .Case("vectorize_width", true) .Case("interleave_count", true) .Case("unroll_count", true) + .Case("pipeline", true) + .Case("pipeline_ii_count", true) .Default(false); if (!OptionValid) { PP.Diag(Tok.getLocation(), diag::err_pragma_loop_invalid_option) Index: lib/Sema/SemaStmtAttr.cpp =================================================================== --- lib/Sema/SemaStmtAttr.cpp +++ lib/Sema/SemaStmtAttr.cpp @@ -147,11 +147,15 @@ .Case("interleave_count", LoopHintAttr::InterleaveCount) .Case("unroll", LoopHintAttr::Unroll) .Case("unroll_count", LoopHintAttr::UnrollCount) + .Case("pipeline", LoopHintAttr::PipelineDisabled) + .Case("pipeline_ii_count", LoopHintAttr::PipelineIICount) .Case("distribute", LoopHintAttr::Distribute) .Default(LoopHintAttr::Vectorize); if (Option == LoopHintAttr::VectorizeWidth || Option == LoopHintAttr::InterleaveCount || - Option == LoopHintAttr::UnrollCount) { + Option == LoopHintAttr::UnrollCount || + Option == LoopHintAttr::PipelineIICount + ) { assert(ValueExpr && "Attribute must have a valid value expression."); if (S.CheckLoopHintExpr(ValueExpr, St->getBeginLoc())) return nullptr; @@ -159,7 +163,8 @@ } else if (Option == LoopHintAttr::Vectorize || Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll || - Option == LoopHintAttr::Distribute) { + Option == LoopHintAttr::Distribute || + Option == LoopHintAttr::PipelineDisabled) { assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument"); if (StateLoc->Ident->isStr("disable")) State = LoopHintAttr::Disable; @@ -182,8 +187,8 @@ static void CheckForIncompatibleAttributes(Sema &S, const SmallVectorImpl &Attrs) { - // There are 5 categories of loop hints attributes: vectorize, interleave, - // unroll, unroll_and_jam and distribute. Except for distribute they come + // There are 6 categories of loop hints attributes: vectorize, interleave, + // unroll, unroll_and_jam, pipeline and distribute. Except for distribute they come // in two variants: a state form and a numeric form. The state form // selectively defaults/enables/disables the transformation for the loop // (for unroll, default indicates full unrolling rather than enabling the @@ -198,6 +203,7 @@ {nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}, + {nullptr, nullptr}, {nullptr, nullptr}}; for (const auto *I : Attrs) { @@ -208,7 +214,7 @@ continue; LoopHintAttr::OptionType Option = LH->getOption(); - enum { Vectorize, Interleave, Unroll, UnrollAndJam, Distribute } Category; + enum { Vectorize, Interleave, Unroll, UnrollAndJam, Distribute, Pipeline } Category; switch (Option) { case LoopHintAttr::Vectorize: case LoopHintAttr::VectorizeWidth: @@ -230,6 +236,10 @@ // Perform the check for duplicated 'distribute' hints. Category = Distribute; break; + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineIICount: + Category = Pipeline; + break; }; assert(Category < sizeof(HintAttrs) / sizeof(HintAttrs[0])); @@ -238,6 +248,7 @@ if (Option == LoopHintAttr::Vectorize || Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll || Option == LoopHintAttr::UnrollAndJam || + Option == LoopHintAttr::PipelineDisabled || Option == LoopHintAttr::Distribute) { // Enable|Disable|AssumeSafety hint. For example, vectorize(enable). PrevAttr = CategoryState.StateAttr; Index: test/CodeGenCXX/pragma-pipeline.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/pragma-pipeline.cpp @@ -0,0 +1,47 @@ +// RUN: %clang_cc1 -triple hexagon -std=c++11 -emit-llvm -o - %s | FileCheck %s + +void pipeline_disabled(int *List, int Length, int Value) { + // CHECK-LABEL: define {{.*}} @_Z17pipeline_disabled +#pragma clang loop pipeline(disable) + for (int i = 0; i < Length; i++) { + // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_1:.*]] + List[i] = Value; + } +} + +void pipeline_not_disabled(int *List, int Length, int Value) { + // CHECK-LABEL: define {{.*}} @_Z21pipeline_not_disabled + for (int i = 0; i < Length; i++) { + List[i] = Value; + } +} + +void pipeline_ii_count(int *List, int Length, int Value) { + // CHECK-LABEL: define {{.*}} @_Z17pipeline_ii_count +#pragma clang loop pipeline_ii_count(10) + for (int i = 0; i < Length; i++) { + // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_3:.*]] + List[i] = Value; + } +} + +void pipeline_disabled_on_nested_loop(int *List, int Length, int Value) { + // CHECK-LABEL: define {{.*}} @_Z32pipeline_disabled_on_nested_loop + for (int i = 0; i < Length; i++) { +#pragma clang loop pipeline(disable) + for (int j = 0; j < Length; j++) { + // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_4:.*]] + List[i*Length + j] = Value; + } + } +} + +// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[PIPELINE_DISABLE:.*]]} +// CHECK: ![[PIPELINE_DISABLE]] = !{!"llvm.loop.pipeline.disable", i1 true} + +// CHECK-NOT:llvm.loop.pipeline + +// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[PIPELINE_II_10:.*]]} +// CHECK: ![[PIPELINE_II_10]] = !{!"llvm.loop.pipeline.iicount", i32 10} + +// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[PIPELINE_DISABLE]]} Index: test/Parser/pragma-loop.cpp =================================================================== --- test/Parser/pragma-loop.cpp +++ test/Parser/pragma-loop.cpp @@ -147,7 +147,7 @@ /* expected-error {{missing argument; expected 'enable', 'full' or 'disable'}} */ #pragma clang loop unroll() /* expected-error {{missing argument; expected 'enable' or 'disable'}} */ #pragma clang loop distribute() -/* expected-error {{missing option; expected vectorize, vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute}} */ #pragma clang loop +/* expected-error {{missing option; expected vectorize, vectorize_width, interleave, interleave_count, unroll, unroll_count, pipeline, pipeline_ii_count or distribute}} */ #pragma clang loop /* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop badkeyword /* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop badkeyword(enable) /* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop vectorize(enable) badkeyword(4) Index: test/Parser/pragma-pipeline.cpp =================================================================== --- /dev/null +++ test/Parser/pragma-pipeline.cpp @@ -0,0 +1,43 @@ +// RUN: %clang_cc1 -std=c++11 -verify %s + +// Note that this puts the expected lines before the directives to work around +// limitations in the -verify mode. + +void test(int *List, int Length, int Value) { + int i = 0; + +#pragma clang loop pipeline(disable) + for (int i = 0; i < Length; i++) { + List[i] = Value; + } + +#pragma clang loop pipeline_ii_count(10) + for (int i = 0; i < Length; i++) { + List[i] = Value; + } + +/* expected-error {{expected ')'}} */ #pragma clang loop pipeline(disable +/* expected-error {{invalid argument; expected 'disable'}} */ #pragma clang loop pipeline(enable) +/* expected-error {{invalid argument; expected 'disable'}} */ #pragma clang loop pipeline(error) +/* expected-error {{missing argument; expected an integer value}} */ #pragma clang loop pipeline_ii_count() +/* expected-error {{use of undeclared identifier 'error'}} */ #pragma clang loop pipeline_ii_count(error) +/* expected-error {{expected '('}} */ #pragma clang loop pipeline_ii_count 1 2 + for (int i = 0; i < Length; i++) { + for (int j = 0; j < Length; j++) { + List[i * Length + j] = Value; + } + } + +#pragma clang loop pipeline(disable) +/* expected-error {{expected a for, while, or do-while loop to follow '#pragma clang loop'}} */ int j = Length; +#pragma clang loop pipeline_ii_count(4) +/* expected-error {{expected a for, while, or do-while loop to follow '#pragma clang loop'}} */ int k = Length; + +#pragma clang loop pipeline(disable) +#pragma clang loop pipeline_ii_count(4) /* expected-error {{incompatible directives 'pipeline(disable)' and 'pipeline_ii_count(4)'}} */ + for (int i = 0; i < Length; i++) { + List[i] = Value; + } + +#pragma clang loop pipeline(disable) +/* expected-error {{expected statement}} */ } Index: test/Parser/pragma-unroll-and-jam.cpp =================================================================== --- test/Parser/pragma-unroll-and-jam.cpp +++ test/Parser/pragma-unroll-and-jam.cpp @@ -67,7 +67,7 @@ } // pragma clang unroll_and_jam is disabled for the moment -/* expected-error {{invalid option 'unroll_and_jam'; expected vectorize, vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute}} */ #pragma clang loop unroll_and_jam(4) +/* expected-error {{invalid option 'unroll_and_jam'; expected vectorize, vectorize_width, interleave, interleave_count, unroll, unroll_count, pipeline, pipeline_ii_count or distribute}} */ #pragma clang loop unroll_and_jam(4) for (int i = 0; i < Length; i++) { for (int j = 0; j < Length; j++) { List[i * Length + j] = Value;