diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h --- a/clang/include/clang/AST/Stmt.h +++ b/clang/include/clang/AST/Stmt.h @@ -1183,6 +1183,9 @@ /// \returns the likelihood of a statement. static Likelihood getLikelihood(const Stmt *S); + /// \returns the likelihood attribute of a statement. + static const Attr *getLikelihoodAttr(const Stmt *S); + /// \returns the likelihood of the 'then' branch of an 'if' statement. The /// 'else' branch is required to determine whether both branches specify the /// same likelihood, which affects the result. diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -1738,7 +1738,8 @@ In Clang, the attributes will be ignored if they're not placed on * the ``case`` or ``default`` label of a ``switch`` statement, -* or on the substatement of an ``if`` or ``else`` statement. +* or on the substatement of an ``if`` or ``else`` statement, +* or on the substatement of an ``for`` or ``while`` statement. The C++ Standard recommends to honor them on every statement in the path of execution, but that can be confusing: @@ -1769,8 +1770,7 @@ } -At the moment the attributes only have effect when used in an ``if``, ``else``, -or ``switch`` statement. +Below are some example usages of the likelihood attributes and their effects: .. code-block:: c++ @@ -1850,6 +1850,23 @@ break; } + for(int i = 0; i != size; ++i) [[likely]] { + ... // The loop is the likely path of execution + } + + for(const auto &E : Elements) [[likely]] { + ... // The loop is the likely path of execution + } + + while(i != size) [[unlikely]] { + ... // The loop is the unlikely path of execution + } // The generated code will optimize to skip the loop body + + while(true) [[unlikely]] { + ... // The attribute has no effect + } // Clang elides the comparison and generates an infinite + // loop + }]; } diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3157,6 +3157,11 @@ def warn_unhandled_ms_attribute_ignored : Warning< "__declspec attribute %0 is not supported">, InGroup; +def warn_attribute_has_no_effect_on_infinite_loop : Warning< + "attribute %0 has no effect when annotating an infinite loop">, + InGroup; +def note_attribute_has_no_effect_on_infinite_loop_here : Note< + "annotating the infinite loop here">; def err_decl_attribute_invalid_on_stmt : Error< "%0 attribute cannot be applied to a statement">; def err_stmt_attribute_invalid_on_decl : Error< diff --git a/clang/lib/AST/Stmt.cpp b/clang/lib/AST/Stmt.cpp --- a/clang/lib/AST/Stmt.cpp +++ b/clang/lib/AST/Stmt.cpp @@ -158,6 +158,10 @@ return ::getLikelihood(S).first; } +const Attr *Stmt::getLikelihoodAttr(const Stmt *S) { + return ::getLikelihood(S).second; +} + Stmt::Likelihood Stmt::getLikelihood(const Stmt *Then, const Stmt *Else) { Likelihood LHT = ::getLikelihood(Then).first; Likelihood LHE = ::getLikelihood(Else).first; diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -18,6 +18,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/StmtVisitor.h" #include "clang/Basic/Builtins.h" +#include "clang/Basic/DiagnosticSema.h" #include "clang/Basic/PrettyStackTrace.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/TargetInfo.h" @@ -806,14 +807,22 @@ llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - Builder.CreateCondBr( - BoolCondVal, LoopBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } + } else if (const Attr *A = Stmt::getLikelihoodAttr(S.getBody())) { + CGM.getDiags().Report(A->getLocation(), + diag::warn_attribute_has_no_effect_on_infinite_loop) + << A << A->getRange(); + CGM.getDiags().Report( + S.getWhileLoc(), + diag::note_attribute_has_no_effect_on_infinite_loop_here) + << SourceRange(S.getWhileLoc(), S.getRParenLoc()); } // Emit the loop body. We have to emit this in a cleanup scope @@ -961,9 +970,9 @@ // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -1042,9 +1051,9 @@ // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1407,6 +1407,13 @@ llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond, uint64_t LoopCount) const; + /// Calculate the branch weight for PGO data or the likelihood attribute. + /// The function tries to get the weight of \ref createProfileWeightsForLoop. + /// If that fails it gets the weight of \ref createBranchWeights. + llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount, + const Stmt *Body) const; + public: /// Increment the profiler's counter for the given statement by \p StepV. /// If \p StepV is null, the default increment is 1. diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -2555,3 +2555,12 @@ llvm::MDBuilder MDHelper(CGM.getLLVMContext()); return MDHelper.createBranchWeights(LHW->first, LHW->second); } + +llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop( + const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const { + llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount); + if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) + Weights = createBranchWeights(Stmt::getLikelihood(Body)); + + return Weights; +} diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp --- a/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp @@ -69,6 +69,7 @@ // CHECK-NOT: br {{.*}} %if.end{{.*}} !prof if (b) + // CHECK: br {{.*}} !prof !7 while (B()) [[unlikely]] { b = false; } } @@ -96,6 +97,7 @@ // CHECK-NOT: br {{.*}} %if.end{{.*}} !prof if (b) + // CHECK: br {{.*}} !prof !7 for (; B();) [[unlikely]] {} } diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp new file mode 100644 --- /dev/null +++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp @@ -0,0 +1,60 @@ +// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify +// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s + +void wl(int e){ + // CHECK-LABEL: define{{.*}}wl + // CHECK: br {{.*}} !prof !6 + while(e) [[likely]] ++e; +} + +void wu(int e){ + // CHECK-LABEL: define{{.*}}wu + // CHECK: br {{.*}} !prof !9 + while(e) [[unlikely]] ++e; +} + +void w_branch_elided(unsigned e){ + // CHECK-LABEL: define{{.*}}w_branch_elided + // CHECK-NOT: br {{.*}} !prof + // expected-warning@+2 {{attribute 'likely' has no effect when annotating an infinite loop}} + // expected-note@+1 {{annotating the infinite loop here}} + while(1) [[likely]] ++e; +} + +void fl(unsigned e) +{ + // CHECK-LABEL: define{{.*}}fl + // CHECK: br {{.*}} !prof !6 + for(int i = 0; i != e; ++e) [[likely]]; +} + +void fu(int e) +{ + // CHECK-LABEL: define{{.*}}fu + // CHECK: br {{.*}} !prof !9 + for(int i = 0; i != e; ++e) [[unlikely]]; +} + +void f_branch_elided() +{ + // CHECK-LABEL: define{{.*}}f_branch_elided + // CHECK-NOT: br {{.*}} !prof + for(;;) [[likely]]; +} + +void frl(int (&&e) [4]) +{ + // CHECK-LABEL: define{{.*}}frl + // CHECK: br {{.*}} !prof !6 + for(int i : e) [[likely]]; +} + +void fru(int (&&e) [4]) +{ + // CHECK-LABEL: define{{.*}}fru + // CHECK: br {{.*}} !prof !9 + for(int i : e) [[unlikely]]; +} + +// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1} +// CHECK: !9 = !{!"branch_weights", i32 1, i32 2000}