Index: clang/include/clang/Basic/AttrDocs.td =================================================================== --- clang/include/clang/Basic/AttrDocs.td +++ clang/include/clang/Basic/AttrDocs.td @@ -1738,7 +1738,8 @@ In Clang, the attributes will be ignored if they're not placed on * the ``case`` or ``default`` label of a ``switch`` statement, -* or on the substatement of an ``if`` or ``else`` statement. +* or on the substatement of an ``if`` or ``else`` statement, +* or on the substatement of an ``for``, ``do``, or ``while`` statement. The C++ Standard recommends to honor them on every statement in the path of execution, but that can be confusing: @@ -1769,8 +1770,7 @@ } -At the moment the attributes only have effect when used in an ``if``, ``else``, -or ``switch`` statement. +Below some usage examples likelihood attributes and their effect: .. code-block:: c++ @@ -1850,6 +1850,26 @@ break; } + for(int i = 0; i != size; ++i) [[likely]] { + ... // The loop the likely path of execution + } + + while(i != size) [[unlikely]] { + ... // The loop is the unlikely path of execution + } // The generated code will optimize to skip the loop body + + do [[unlikely]] { // The loop will always iterate once + ... // The likelihood attribute affects the likelihood of the + } while(i != size); // while at the end of the ``do while`` statement + + while(true) [[unlikely]] { + ... // The attribute has no effect + } // Clang elides comparison and generates an infinite loop + + do [[likely]] { + ... // The attribute has no effect + } while(0); // Clang elides comparison and generates no loop + }]; } Index: clang/lib/CodeGen/CGStmt.cpp =================================================================== --- clang/lib/CodeGen/CGStmt.cpp +++ clang/lib/CodeGen/CGStmt.cpp @@ -805,9 +805,9 @@ llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - Builder.CreateCondBr( - BoolCondVal, LoopBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -890,9 +890,9 @@ // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount; - Builder.CreateCondBr( - BoolCondVal, LoopBody, LoopExit.getBlock(), - createProfileWeightsForLoop(S.getCond(), BackedgeCount)); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), BackedgeCount, S.getBody()); + Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(), Weights); } LoopStack.pop(); @@ -960,9 +960,9 @@ // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -1041,9 +1041,9 @@ // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, - createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); + llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop( + S.getCond(), getProfileCount(S.getBody()), S.getBody()); + Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); Index: clang/lib/CodeGen/CodeGenFunction.h =================================================================== --- clang/lib/CodeGen/CodeGenFunction.h +++ clang/lib/CodeGen/CodeGenFunction.h @@ -1407,6 +1407,13 @@ llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond, uint64_t LoopCount) const; + /// Calculate the branch weight for PGO data or the likelihood attribe. + /// The function tries to get the weight of \ref createProfileWeightsForLoop. + /// If that fails it gets the weight of \ref createBranchWeights. + llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount, + const Stmt *Body) const; + public: /// Increment the profiler's counter for the given statement by \p StepV. /// If \p StepV is null, the default increment is 1. Index: clang/lib/CodeGen/CodeGenFunction.cpp =================================================================== --- clang/lib/CodeGen/CodeGenFunction.cpp +++ clang/lib/CodeGen/CodeGenFunction.cpp @@ -2573,3 +2573,12 @@ llvm::MDBuilder MDHelper(CGM.getLLVMContext()); return MDHelper.createBranchWeights(LHW->first, LHW->second); } + +llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop( + const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const { + llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount); + if (!Weights && CGM.getCodeGenOpts().OptimizationLevel) + Weights = createBranchWeights(Stmt::getLikelihood(Body)); + + return Weights; +} Index: clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp =================================================================== --- clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp +++ clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp @@ -69,6 +69,7 @@ // CHECK-NOT: br {{.*}} %if.end{{.*}} !prof if (b) + // CHECK: br {{.*}} !prof !7 while (B()) [[unlikely]] { b = false; } } @@ -83,6 +84,7 @@ // CHECK-NOT: br {{.*}} %if.end{{.*}} !prof if (b) + // CHECK: br {{.*}} !prof !7 do [[unlikely]] {} while (B()); @@ -96,6 +98,7 @@ // CHECK-NOT: br {{.*}} %if.end{{.*}} !prof if (b) + // CHECK: br {{.*}} !prof !7 for (; B();) [[unlikely]] {} } Index: clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp @@ -0,0 +1,75 @@ +// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s + +void wl(int e){ + // CHECK-LABEL: define{{.*}}wl + // CHECK: br {{.*}} !prof !6 + while(e) [[likely]] ++e; +} + +void wu(int e){ + // CHECK-LABEL: define{{.*}}wu + // CHECK: br {{.*}} !prof !9 + while(e) [[unlikely]] ++e; +} + +void w_branch_elided(unsigned e){ + // CHECK-LABEL: define{{.*}}w_branch_elided + // CHECK-NOT: br {{.*}} !prof + while(1) [[likely]] ++e; +} + +void dl(int e){ + // CHECK-LABEL: define{{.*}}dl + // CHECK: br {{.*}} !prof !6 + do [[likely]] ++e; while(e); +} + +void du(int e){ + // CHECK-LABEL: define{{.*}}du + // CHECK: br {{.*}} !prof !9 + do [[unlikely]] ++e; while(e); +} + +void d_branch_elided(unsigned e){ + // CHECK-LABEL: define{{.*}}d_branch_elided + // CHECK-NOT: br {{.*}} !prof + do [[unlikely]] ++e; while(0); +} + +void fl(unsigned e) +{ + // CHECK-LABEL: define{{.*}}fl + // CHECK: br {{.*}} !prof !6 + for(int i = 0; i != e; ++e) [[likely]]; +} + +void fu(int e) +{ + // CHECK-LABEL: define{{.*}}fu + // CHECK: br {{.*}} !prof !9 + for(int i = 0; i != e; ++e) [[unlikely]]; +} + +void f_branch_elided() +{ + // CHECK-LABEL: define{{.*}}f_branch_elided + // CHECK-NOT: br {{.*}} !prof + for(;;) [[likely]]; +} + +void frl(int (&&e) [4]) +{ + // CHECK-LABEL: define{{.*}}frl + // CHECK: br {{.*}} !prof !6 + for(int i : e) [[likely]]; +} + +void fru(int (&&e) [4]) +{ + // CHECK-LABEL: define{{.*}}fru + // CHECK: br {{.*}} !prof !9 + for(int i : e) [[unlikely]]; +} + +// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1} +// CHECK: !9 = !{!"branch_weights", i32 1, i32 2000}