Index: clang/include/clang/Basic/Attr.td =================================================================== --- clang/include/clang/Basic/Attr.td +++ clang/include/clang/Basic/Attr.td @@ -1147,6 +1147,12 @@ let Documentation = [FallthroughDocs]; } +def Likelihood : StmtAttr { + let Spellings = [CXX11<"", "likely", 201803>, Clang<"likely">, CXX11<"", "unlikely", 201803>, Clang<"unlikely">]; +// let Subjects = [Stmt, LabelStmt]; + let Documentation = [LikelihoodDocs]; +} + def FastCall : DeclOrTypeAttr { let Spellings = [GCC<"fastcall">, Keyword<"__fastcall">, Keyword<"_fastcall">]; Index: clang/include/clang/Basic/AttrDocs.td =================================================================== --- clang/include/clang/Basic/AttrDocs.td +++ clang/include/clang/Basic/AttrDocs.td @@ -1492,6 +1492,39 @@ }]; } +def LikelihoodDocs : Documentation { + let Category = DocCatStmt; + let Heading = "likely / unlikely"; + let Content = [{ + +The ``likely`` or ``unlikely`` attribute is used to annotate that a statement or label is likely or unlikely to executed + +Here is an example: + +.. code-block:: c++ + + void g(int); + int f(int n) { + if (n > 5) [[unlikely]] { // n > 5 is considered to be arbitrarily unlikely + g(0); + return n * 2 + 1; + } + + switch (n) { + case 1: + g(1); + [[fallthrough]]; + + [[likely]] case 2: // n == 2 is considered to be arbitrarily more + g(2); // likely than any other value of n + break; + } + return 3; + } + + }]; +} + def ARMInterruptDocs : Documentation { let Category = DocCatFunction; let Heading = "interrupt (ARM)"; Index: clang/include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- clang/include/clang/Basic/DiagnosticSemaKinds.td +++ clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -7339,6 +7339,8 @@ "use of the %0 attribute is a C++14 extension">, InGroup; def ext_cxx17_attr : Extension< "use of the %0 attribute is a C++17 extension">, InGroup; +def ext_cxx2a_attr : Extension< + "use of the %0 attribute is a C++2a extension">, InGroup; def warn_unused_comparison : Warning< "%select{equality|inequality|relational|three-way}0 comparison result unused">, @@ -8158,6 +8160,15 @@ "fallthrough annotation in unreachable code">, InGroup, DefaultIgnore; +def err_must_appear_after_branch : Error< + "%0 can only appear after a selection or iteration statement">; +def warn_attribute_already_present : Warning< + "was already marked %0">; +def err_mutuably_exclusive_likelihood : Error< + "%0 and %1 are mutually exclusive">; +def err_contradictory_attribute : Warning< + "%0 contradicing with previous attribute">; + def warn_unreachable_default : Warning< "default label in switch which covers all enumeration values">, InGroup, DefaultIgnore; Index: clang/lib/CodeGen/CGStmt.cpp =================================================================== --- clang/lib/CodeGen/CGStmt.cpp +++ clang/lib/CodeGen/CGStmt.cpp @@ -657,8 +657,13 @@ if (S.getElse()) ElseBlock = createBasicBlock("if.else"); + BranchHint hint = getBranchHint(S.getThen()); + if (S.getElse() && hint == NoHint) { + hint = getBranchHint(S.getElse(), true); + } + EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, - getProfileCount(S.getThen())); + getProfileCount(S.getThen()), hint); // Emit the 'then' code. EmitBlock(ThenBlock); @@ -691,6 +696,16 @@ EmitBlock(ContBlock, true); } +void CodeGenFunction::MaybeEmitLikelihoodHint(const Stmt* branch, llvm::Value *CondV) { + BranchHint hint = getBranchHint(branch); + if (hint != NoHint && CGM.getCodeGenOpts().OptimizationLevel != 0) + { + llvm::Constant* ExpectedValue = llvm::ConstantInt::get(llvm::Type::getInt1Ty(this->getLLVMContext()), hint == Taken); + llvm::Function *FnExpect = CGM.getIntrinsic(llvm::Intrinsic::expect, CondV->getType()); + Builder.CreateCall(FnExpect, {CondV, ExpectedValue}, "expval"); + } +} + void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, ArrayRef WhileAttrs) { // Emit the header for the loop, which will also become @@ -740,6 +755,7 @@ llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); + MaybeEmitLikelihoodHint(S.getBody(), BoolCondVal); Builder.CreateCondBr( BoolCondVal, LoopBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); @@ -825,6 +841,7 @@ // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount; + MaybeEmitLikelihoodHint(S.getBody(), BoolCondVal); Builder.CreateCondBr( BoolCondVal, LoopBody, LoopExit.getBlock(), createProfileWeightsForLoop(S.getCond(), BackedgeCount)); @@ -895,6 +912,7 @@ // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); + MaybeEmitLikelihoodHint(S.getBody(), BoolCondVal); Builder.CreateCondBr( BoolCondVal, ForBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); @@ -976,6 +994,7 @@ // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); + MaybeEmitLikelihoodHint(S.getBody(), BoolCondVal); Builder.CreateCondBr( BoolCondVal, ForBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); Index: clang/lib/CodeGen/CodeGenFunction.h =================================================================== --- clang/lib/CodeGen/CodeGenFunction.h +++ clang/lib/CodeGen/CodeGenFunction.h @@ -2852,6 +2852,8 @@ AggValueSlot AVS = AggValueSlot::ignored()); + void MaybeEmitLikelihoodHint(const Stmt* branch, llvm::Value *CondV); + /// EmitLabel - Emit the block for the given label. It is legal to call this /// function even if there is no current insertion point. void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt. @@ -4014,6 +4016,11 @@ /// that we can just remove the code. static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false); + /// Hint on whether the branch is expected to be taken or not + enum BranchHint {NoHint = 0, Taken = 1, NotTaken = 2}; + + static BranchHint getBranchHint(const Stmt* S, bool Invert = false); + /// containsBreak - Return true if the statement contains a break out of it. /// If the statement (recursively) contains a switch or loop with a break /// inside of it, this is fine. @@ -4041,7 +4048,7 @@ /// TrueCount should be the number of times we expect the condition to /// evaluate to true based on PGO data. void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, - llvm::BasicBlock *FalseBlock, uint64_t TrueCount); + llvm::BasicBlock *FalseBlock, uint64_t TrueCount, BranchHint hint = NoHint); /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is /// nonnull, if \p LHS is marked _Nonnull. Index: clang/lib/CodeGen/CodeGenFunction.cpp =================================================================== --- clang/lib/CodeGen/CodeGenFunction.cpp +++ clang/lib/CodeGen/CodeGenFunction.cpp @@ -1409,6 +1409,22 @@ TryMarkNoThrow(CurFn); } +CodeGenFunction::BranchHint CodeGenFunction::getBranchHint(const Stmt* S, bool Invert) { + assert(S && "expect valid pointer"); + if (isa(S)) { + auto* ThenAttr = dyn_cast(S); + auto AttrIt = std::find_if(ThenAttr->getAttrs().begin(), ThenAttr->getAttrs().end(), [](const Attr* attr){ + return isa(attr); + }); + if (AttrIt != ThenAttr->getAttrs().end()) { + if (((*AttrIt)->getSpelling()[0] == 'l') ^ Invert) + return Taken; + return NotTaken; + } + } + return NoHint; +} + /// ContainsLabel - Return true if the statement contains a label in it. If /// this statement is not executed normally, it not containing a label means /// that we can just remove the code. @@ -1531,7 +1547,8 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, - uint64_t TrueCount) { + uint64_t TrueCount, + BranchHint hint) { Cond = Cond->IgnoreParens(); if (const BinaryOperator *CondBOp = dyn_cast(Cond)) { @@ -1720,6 +1737,13 @@ ApplyDebugLocation DL(*this, Cond); CondV = EvaluateExprAsBool(Cond); } + + if (hint != NoHint && CGM.getCodeGenOpts().OptimizationLevel != 0) + { + llvm::Constant* ExpectedValue = llvm::ConstantInt::get(llvm::Type::getInt1Ty(this->getLLVMContext()), hint == Taken); + llvm::Function *FnExpect = CGM.getIntrinsic(llvm::Intrinsic::expect, CondV->getType()); + Builder.CreateCall(FnExpect, {CondV, ExpectedValue}, "expval"); + } Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable); } Index: clang/lib/Parse/ParseStmt.cpp =================================================================== --- clang/lib/Parse/ParseStmt.cpp +++ clang/lib/Parse/ParseStmt.cpp @@ -20,6 +20,7 @@ #include "clang/Sema/DeclSpec.h" #include "clang/Sema/Scope.h" #include "clang/Sema/TypoCorrection.h" +#include "clang/Sema/SemaDiagnostic.h" using namespace clang; //===----------------------------------------------------------------------===// @@ -1286,6 +1287,26 @@ /*ShouldEnter=*/ConstexprCondition && *ConstexprCondition); ElseStmt = ParseStatement(); + // diagnose likelihood attribute conflicts + if (isa(ThenStmt.get()) && isa(ElseStmt.get())) { + AttributedStmt* ThenAttrs = dyn_cast(ThenStmt.get()); + AttributedStmt* ElseAttrs = dyn_cast(ElseStmt.get()); + + auto ThenIt = std::find_if(ThenAttrs->getAttrs().begin(), ThenAttrs->getAttrs().end(), [](const Attr* attr){ + return isa(attr); + }); + if (ThenIt != ThenAttrs->getAttrs().end()) { + auto ElseIt = std::find_if(ElseAttrs->getAttrs().begin(), ElseAttrs->getAttrs().end(), [](const Attr* attr){ + return isa(attr); + }); + if (ThenIt != ThenAttrs->getAttrs().end() && ElseIt != ElseAttrs->getAttrs().end() && + (*ThenIt)->getSpelling()[0] == (*ElseIt)->getSpelling()[0]) { + Actions.Diag((*ElseIt)->getLocation(), diag::err_contradictory_attribute) << (*ElseIt)->getSpelling(); + Actions.Diag((*ThenIt)->getLocation(), diag::note_conflicting_attribute); + } + } + } + // Pop the 'else' scope if needed. InnerScope.Exit(); } else if (Tok.is(tok::code_completion)) { Index: clang/lib/Sema/SemaStmtAttr.cpp =================================================================== --- clang/lib/Sema/SemaStmtAttr.cpp +++ clang/lib/Sema/SemaStmtAttr.cpp @@ -51,6 +51,31 @@ return ::new (S.Context) auto(Attr); } +static Attr *handleLikelihoodAttr(Sema &S, Stmt *St, const ParsedAttr &A, + SourceRange Range) { + Attr* Attr = ::new (S.Context) LikelihoodAttr(A.getRange(), S.Context, + A.getAttributeSpellingListIndex()); + + if (!S.getLangOpts().CPlusPlus2a && A.isCXX11Attribute()) + S.Diag(A.getLoc(), diag::ext_cxx2a_attr) << A.getName(); + + Scope* CurScope = S.getCurScope(); + + if (CurScope) { + Scope* ControlScope = CurScope->getParent(); + if (!ControlScope || + !(ControlScope->getFlags() & Scope::ControlScope || + ControlScope->getFlags() & Scope::BreakScope) || + CurScope->getFlags() & Scope::CompoundStmtScope || + ControlScope->getFlags() & Scope::SEHExceptScope || + ControlScope->getFlags() & Scope::SEHTryScope || + ControlScope->getFlags() & Scope::TryScope || + ControlScope->getFlags() & Scope::FnTryCatchScope) + S.Diag(A.getLoc(), diag::err_must_appear_after_branch) << A.getName(); + } + return Attr; +} + static Attr *handleSuppressAttr(Sema &S, Stmt *St, const ParsedAttr &A, SourceRange Range) { if (A.getNumArgs() < 1) { @@ -201,7 +226,22 @@ } HintAttrs[] = {{nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}}; + const Attr* likelihoodAttr = nullptr; + for (const auto *I : Attrs) { + + if (isa(I)) { + if (likelihoodAttr) { + if (likelihoodAttr->getSpelling()[0] == I->getSpelling()[0]) + S.Diag(I->getLocation(), diag::warn_attribute_already_present) << I->getSpelling(); + else + S.Diag(I->getLocation(), diag::err_mutuably_exclusive_likelihood) << I->getSpelling() << likelihoodAttr->getSpelling(); + S.Diag(likelihoodAttr->getLocation(), diag::note_previous_attribute); + } else + likelihoodAttr = I; + continue; + } + const LoopHintAttr *LH = dyn_cast(I); // Skip non loop hint attributes @@ -336,6 +376,8 @@ return nullptr; case ParsedAttr::AT_FallThrough: return handleFallThroughAttr(S, St, A, Range); + case ParsedAttr::AT_Likelihood: + return handleLikelihoodAttr(S, St, A, Range); case ParsedAttr::AT_LoopHint: return handleLoopHintAttr(S, St, A, Range); case ParsedAttr::AT_OpenCLUnrollHint: Index: clang/test/AST/ast-dump-attr.cpp =================================================================== --- clang/test/AST/ast-dump-attr.cpp +++ clang/test/AST/ast-dump-attr.cpp @@ -1,5 +1,39 @@ // RUN: %clang_cc1 -triple x86_64-pc-linux -std=c++11 -Wno-deprecated-declarations -ast-dump -ast-dump-filter Test %s | FileCheck --strict-whitespace %s +int TestLikelyAttributeIf(int i) { + if (i == 1) [[likely]] { + return 0; + } else if (i == 2) [[unlikely]] + return 1; + return 2; +} +// CHECK: IfStmt +// CHECK: AttributedStmt +// CHECK-NEXT: LikelihoodAttr 0x{{[^ ]*}} likely +// CHECK: IfStmt +// CHECK: AttributedStmt +// CHECK-NEXT: LikelihoodAttr 0x{{[^ ]*}} unlikely + +int TestLikelyAttributeLoops(int i) { + while (i == 1) [[likely]] { + return 0; + } + for (;;) [[unlikely]] + do [[likely]] { + return 1; + } while (i == 2); + return 2; +} +// CHECK: WhileStmt +// CHECK: AttributedStmt +// CHECK-NEXT: LikelihoodAttr 0x{{[^ ]*}} likely +// CHECK: ForStmt +// CHECK: AttributedStmt +// CHECK-NEXT: LikelihoodAttr 0x{{[^ ]*}} unlikely +// CHECK: DoStmt +// CHECK: AttributedStmt +// CHECK-NEXT: LikelihoodAttr 0x{{[^ ]*}} likely + int TestLocation __attribute__((unused)); // CHECK: VarDecl{{.*}}TestLocation Index: clang/test/CodeGenCXX/cxx2a-likelihood-attr.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/cxx2a-likelihood-attr.cpp @@ -0,0 +1,22 @@ +// RUN: %clang_cc1 -std=c++2a -cc1 -emit-llvm -disable-llvm-passes -O3 %s -o - -triple %itanium_abi_triple | FileCheck %s + +int test(int i) { + if (i == 0) { + i = i + 1; + } else [[likely]] + return 1; + // CHECK: %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 false) + while (i == 1) [[unlikely]] { + return 2; + } + // CHECK: %[[expval:.*]] = call i1 @llvm.expect.i1(i1 %[[cmp:.*]], i1 false) + for (;i == 4;) [[unlikely]] { + return 2; + } + // CHECK: %[[expval:.*]] = call i1 @llvm.expect.i1(i1 %[[cmp:.*]], i1 false) + do [[likely]] { + return 2; + } while (i == 3); + // CHECK: %[[expval:.*]] = call i1 @llvm.expect.i1(i1 %[[cmp:.*]], i1 true) + return 0; +} Index: clang/test/SemaCXX/cxx2a-likelihood-attr.cpp =================================================================== --- /dev/null +++ clang/test/SemaCXX/cxx2a-likelihood-attr.cpp @@ -0,0 +1,51 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++2a + +int f(int i) { + if (i == 1) [[unlikely]] + { + return 0; + } + else if (i == 2) [[likely]] + return 1; + return 3; +} + +[[likely]] typedef int n1; // expected-error {{'likely' attribute cannot be applied to a declaration}} +typedef int [[likely]] n2; // expected-error {{'likely' attribute cannot be applied to types}} +typedef int n3 [[likely]]; // expected-error {{'likely' attribute cannot be applied to a declaration}} + +enum [[likely]] E { // expected-error {{'likely' attribute cannot be applied to a declaration}} + One +}; + +[[likely]] // expected-error {{'likely' attribute cannot be applied to a declaration}} + +void test(int i) { + [[likely]] // expected-error {{'likely' can only appear after a selection or iteration statement}} + if (1) [[likely, likely]] { + // expected-warning@-1 {{was already marked likely}} + // expected-note@-2 {{previous attribute is here}} + [[unlikely]] return ; // expected-error {{'unlikely' can only appear after a selection or iteration statement}} + } + else [[unlikely]] if (1) { + while (1) [[likely]] { + // switch (i) { switch support isn't implemented yet + // [[likely]] case 1: + // default: [[likely]] + // return ; + // } + } + for (;;) [[likely, unlikely]] + // expected-error@-1 {{unlikely and likely are mutually exclusive}} + // expected-note@-2 {{previous attribute is here}} + [[likely]] return ; + // expected-warning@-1 {{was already marked likely}} + // expected-note@-5 {{previous attribute is here}} + } + try { // expected-error {{cannot use 'try' with exceptions disabled}} + [[likely]]; // expected-error {{'likely' can only appear after a selection or iteration statement}} + } catch (int) { + [[likely]] test: // expected-error {{'likely' attribute cannot be applied to a declaration}} + [[unlikely]] return ; // expected-error {{'unlikely' can only appear after a selection or iteration statement}} + } +} \ No newline at end of file