Index: clang/include/clang/Basic/Builtins.def =================================================================== --- clang/include/clang/Basic/Builtins.def +++ clang/include/clang/Basic/Builtins.def @@ -566,6 +566,7 @@ BUILTIN(__builtin_unpredictable, "LiLi" , "nc") BUILTIN(__builtin_expect, "LiLiLi" , "nc") +BUILTIN(__builtin_expect_with_probability, "LiLiLid", "nc") BUILTIN(__builtin_prefetch, "vvC*.", "nc") BUILTIN(__builtin_readcyclecounter, "ULLi", "n") BUILTIN(__builtin_trap, "v", "nr") Index: clang/include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- clang/include/clang/Basic/DiagnosticSemaKinds.td +++ clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -10808,4 +10808,12 @@ "have a bit size of at least %select{2|1}0">; def err_ext_int_max_size : Error<"%select{signed|unsigned}0 _ExtInt of bit " "sizes greater than %1 not supported">; + +// errors of expect.with.probability +def err_probability_not_constant_float : Error< + "probability argument to __builtin_expect_with_probability must be constant " + "floating-point expression">; +def err_probability_out_of_range : Error< + "probability argument to __builtin_expect_with_probability is outside the " + "range [0.0, 1.0]">; } // end of sema component. Index: clang/lib/CodeGen/CGBuiltin.cpp =================================================================== --- clang/lib/CodeGen/CGBuiltin.cpp +++ clang/lib/CodeGen/CGBuiltin.cpp @@ -2189,6 +2189,34 @@ Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); return RValue::get(Result); } + case Builtin::BI__builtin_expect_with_probability: { + Value *ArgValue = EmitScalarExpr(E->getArg(0)); + llvm::Type *ArgType = ArgValue->getType(); + + Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); + llvm::APFloat Probability(0.0); + const Expr *ProbArg = E->getArg(2); + bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext()); + assert(EvalSucceed && "probability should be able to evaluate as float"); + bool loseInfo = false; + Probability.convert(llvm::APFloat::IEEEdouble(), + llvm::RoundingMode::Dynamic, &loseInfo); + (void)EvalSucceed; + llvm::Type *Ty = ConvertType(ProbArg->getType()); + Constant *Confidence = ConstantFP::get(Ty, Probability); + // Don't generate llvm.expect.with.probability on -O0 as the backend + // won't use it for anything. + // Note, we still IRGen ExpectedValue and Probability because + // it could have side-effects. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return RValue::get(ArgValue); + + Function *FnExpect = + CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType); + Value *Result = Builder.CreateCall( + FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval"); + return RValue::get(Result); + } case Builtin::BI__builtin_assume_aligned: { const Expr *Ptr = E->getArg(0); Value *PtrValue = EmitScalarExpr(Ptr); Index: clang/lib/Sema/SemaChecking.cpp =================================================================== --- clang/lib/Sema/SemaChecking.cpp +++ clang/lib/Sema/SemaChecking.cpp @@ -1796,6 +1796,36 @@ TheCall->setType(Context.IntTy); break; } + case Builtin::BI__builtin_expect_with_probability: { + // We first want to ensure we are called with 3 arguments + if (checkArgCount(*this, TheCall, 3)) + return ExprError(); + // then check probability is constant float in range [0.0, 1.0] + const Expr *ProbArg = TheCall->getArg(2); + SmallVector Notes; + Expr::EvalResult Eval; + Eval.Diag = &Notes; + if ((!ProbArg->EvaluateAsConstantExpr(Eval, Expr::EvaluateForCodeGen, + Context)) || + !Eval.Val.isFloat()) { + Diag(ProbArg->getBeginLoc(), diag::err_probability_not_constant_float) + << ProbArg->getSourceRange(); + for (const PartialDiagnosticAt &PDiag : Notes) + Diag(PDiag.first, PDiag.second); + return ExprError(); + } + llvm::APFloat Probability = Eval.Val.getFloat(); + bool loseInfo = false; + Probability.convert(llvm::APFloat::IEEEdouble(), + llvm::RoundingMode::Dynamic, &loseInfo); + if (!(Probability >= llvm::APFloat(0.0) && + Probability <= llvm::APFloat(1.0))) { + Diag(ProbArg->getBeginLoc(), diag::err_probability_out_of_range) + << ProbArg->getSourceRange(); + return ExprError(); + } + break; + } case Builtin::BI__builtin_preserve_access_index: if (SemaBuiltinPreserveAI(*this, TheCall)) return ExprError(); Index: clang/test/CodeGen/builtin-expect-with-probability-template.cpp =================================================================== --- /dev/null +++ clang/test/CodeGen/builtin-expect-with-probability-template.cpp @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -emit-llvm -o - %s -O1 | FileCheck %s +extern int global; + +struct S { + static constexpr int prob = 1; +}; + +template +int expect_taken(int x) { +// CHECK: !{{[0-9]+}} = !{!"branch_weights", i32 2147483647, i32 1} + + if (__builtin_expect_with_probability (x == 100, 1, T::prob)) { + return 0; + } + return x; +} + +void f() { + expect_taken(global); +} Index: clang/test/CodeGen/builtin-expect-with-probability.c =================================================================== --- /dev/null +++ clang/test/CodeGen/builtin-expect-with-probability.c @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -emit-llvm -o - %s -O1 | FileCheck %s + +int expect_taken(int x) { + // CHECK: !{{[0-9]+}} = !{!"branch_weights", i32 1932735283, i32 214748366} + + if (__builtin_expect_with_probability(x == 100, 1, 0.9)) { + return 0; + } + return x; +} + +int expect_taken2(int x) { + // CHECK: !{{[0-9]+}} = !{!"branch_weights", i32 107374184, i32 107374184, i32 1717986918, i32 107374184, i32 107374184} + switch (__builtin_expect_with_probability(x, 1, 0.8)) { + case 0: + x = x + 0; + case 1: + x = x + 1; + case 2: + x = x + 2; + case 5: + x = x + 5; + default: + x = x + 6; + } + return x; +} Index: clang/test/Sema/builtin-expect-with-probability-avr.cpp =================================================================== --- /dev/null +++ clang/test/Sema/builtin-expect-with-probability-avr.cpp @@ -0,0 +1,15 @@ +// RUN: %clang_cc1 -triple avr -fsyntax-only -verify %s + +void test(int x, double p) { // expected-note {{declared here}} + bool dummy = false; + dummy = __builtin_expect_with_probability(x > 0, 1, 0.9); + dummy = __builtin_expect_with_probability(x > 0, 1, 1.1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, p); // expected-error {{probability argument to __builtin_expect_with_probability must be constant floating-point expression}} expected-note {{read of non-constexpr variable 'p' is not allowed in a constant expression}} + dummy = __builtin_expect_with_probability(x > 0, 1, "aa"); // expected-error {{cannot initialize a parameter of type 'double' with an lvalue of type 'const char [3]'}} + dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_nan("")); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_inf()); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -0.0); + dummy = __builtin_expect_with_probability(x > 0, 1, 1.0 + __DBL_EPSILON__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -__DBL_DENORM_MIN__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} +} Index: clang/test/Sema/builtin-expect-with-probability.cpp =================================================================== --- /dev/null +++ clang/test/Sema/builtin-expect-with-probability.cpp @@ -0,0 +1,32 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s +extern int global; + +struct S { + static constexpr float prob = 0.7; +}; + +template +void expect_taken(int x) { + if (__builtin_expect_with_probability(x > 0, 1, T::prob)) { + global++; + } +} + +void test(int x, double p) { // expected-note {{declared here}} + bool dummy; + dummy = __builtin_expect_with_probability(x > 0, 1, 0.9); + dummy = __builtin_expect_with_probability(x > 0, 1, 1.1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -1); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, p); // expected-error {{probability argument to __builtin_expect_with_probability must be constant floating-point expression}} expected-note {{read of non-constexpr variable 'p' is not allowed in a constant expression}} + dummy = __builtin_expect_with_probability(x > 0, 1, "aa"); // expected-error {{cannot initialize a parameter of type 'double' with an lvalue of type 'const char [3]'}} + dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_nan("")); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, __builtin_inf()); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -0.0); + dummy = __builtin_expect_with_probability(x > 0, 1, 1.0 + __DBL_EPSILON__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + dummy = __builtin_expect_with_probability(x > 0, 1, -__DBL_DENORM_MIN__); // expected-error {{probability argument to __builtin_expect_with_probability is outside the range [0.0, 1.0]}} + constexpr double pd = 0.7; + dummy = __builtin_expect_with_probability(x > 0, 1, pd); + constexpr int pi = 1; + dummy = __builtin_expect_with_probability(x > 0, 1, pi); + expect_taken(x); +} Index: llvm/docs/BranchWeightMetadata.rst =================================================================== --- llvm/docs/BranchWeightMetadata.rst +++ llvm/docs/BranchWeightMetadata.rst @@ -15,7 +15,7 @@ "branch_weights". Number of operators depends on the terminator type. Branch weights might be fetch from the profiling file, or generated based on -`__builtin_expect`_ instruction. +`__builtin_expect`_ and `__builtin_expect_with_probability`_ instruction. All weights are represented as an unsigned 32-bit values, where higher value indicates greater chance to be taken. @@ -144,6 +144,47 @@ case 5: // This case is likely to be taken. } +.. _\__builtin_expect_with_probability: + +Built-in ``expect.with.probability`` Instruction +================================================ + +``__builtin_expect_with_probability(long exp, long c, double probability)`` has +the same semantics as ``__builtin_expect``, but the caller provides the +probability that ``exp == c``. The last argument ``probability`` must be +constant floating-point expression and be in the range [0.0, 1.0] inclusive. +The usage is also similar as ``__builtin_expect``, for example: + +``if`` statement +^^^^^^^^^^^^^^^^ + +If the expect comparison value ``c`` is equal to 1(true), and probability +value ``probability`` is set to 0.8, that means the probability of condition +to be true is 80% while that of false is 20%. + +.. code-block:: c++ + + if (__builtin_expect_with_probability(x > 0, 1, 0.8)) { + // This block is likely to be taken with probability 80%. + } + +``switch`` statement +^^^^^^^^^^^^^^^^^^^^ + +This is basically the same as ``switch`` statement in ``__builtin_expect``. +The probability that ``exp`` is equal to the expect value is given in +the third argument ``probability``, while the probability of other value is +the average of remaining probability(``1.0 - probability``). For example: + +.. code-block:: c++ + + switch (__builtin_expect_with_probability(x, 5, 0.7)) { + default: break; // Take this case with probability 10% + case 0: // Take this case with probability 10% + case 3: // Take this case with probability 10% + case 5: // This case is likely to be taken with probability 70% + } + CFG Modifications ================= Index: llvm/include/llvm/IR/Intrinsics.td =================================================================== --- llvm/include/llvm/IR/Intrinsics.td +++ llvm/include/llvm/IR/Intrinsics.td @@ -837,6 +837,10 @@ def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrWillReturn]>; +def int_expect_with_probability : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_double_ty], + [IntrNoMem, IntrWillReturn]>; + //===-------------------- Bit Manipulation Intrinsics ---------------------===// // Index: llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -55,13 +55,32 @@ "unlikely-branch-weight", cl::Hidden, cl::init(1), cl::desc("Weight of the branch unlikely to be taken (default = 1)")); +std::pair setBranchWeight(Intrinsic::ID IntrinsicID, + CallInst *CI, int BranchCount) { + if (IntrinsicID == Intrinsic::expect) { + // __builtin_expect + return {LikelyBranchWeight, UnlikelyBranchWeight}; + } else { + // __builtin_expect_with_probability + assert(CI->getNumOperands() >= 3 && + "expect with probability must have 3 arguments"); + ConstantFP *Confidence = dyn_cast(CI->getArgOperand(2)); + double TrueProb = Confidence->getValueAPF().convertToDouble(); + double FalseProb = (1.0 - TrueProb) / (BranchCount - 1); + uint32_t LikelyBW = ceil((TrueProb * (double)(INT32_MAX - 1)) + 1.0); + uint32_t UnlikelyBW = ceil((FalseProb * (double)(INT32_MAX - 1)) + 1.0); + return {LikelyBW, UnlikelyBW}; + } +} + static bool handleSwitchExpect(SwitchInst &SI) { CallInst *CI = dyn_cast(SI.getCondition()); if (!CI) return false; Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) + if (!Fn || (Fn->getIntrinsicID() != Intrinsic::expect && + Fn->getIntrinsicID() != Intrinsic::expect_with_probability)) return false; Value *ArgValue = CI->getArgOperand(0); @@ -71,15 +90,20 @@ SwitchInst::CaseHandle Case = *SI.findCaseValue(ExpectedValue); unsigned n = SI.getNumCases(); // +1 for default case. - SmallVector Weights(n + 1, UnlikelyBranchWeight); + std::pair WeightNums = + setBranchWeight(Fn->getIntrinsicID(), CI, n + 1); + uint32_t LikelyBranchWeightVal = WeightNums.first; + uint32_t UnlikelyBranchWeightVal = WeightNums.second; + + SmallVector Weights(n + 1, UnlikelyBranchWeightVal); uint64_t Index = (Case == *SI.case_default()) ? 0 : Case.getCaseIndex() + 1; - Weights[Index] = LikelyBranchWeight; + Weights[Index] = LikelyBranchWeightVal; - SI.setMetadata( - LLVMContext::MD_misexpect, - MDBuilder(CI->getContext()) - .createMisExpect(Index, LikelyBranchWeight, UnlikelyBranchWeight)); + SI.setMetadata(LLVMContext::MD_misexpect, + MDBuilder(CI->getContext()) + .createMisExpect(Index, LikelyBranchWeightVal, + UnlikelyBranchWeightVal)); SI.setCondition(ArgValue); misexpect::checkFrontendInstrumentation(SI); @@ -223,15 +247,19 @@ return true; return false; }; + std::pair WeightNums = setBranchWeight( + Expect->getCalledFunction()->getIntrinsicID(), Expect, 2); + uint32_t LikelyBranchWeightVal = WeightNums.first; + uint32_t UnlikelyBranchWeightVal = WeightNums.second; if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) - BI->setMetadata( - LLVMContext::MD_prof, - MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight)); + BI->setMetadata(LLVMContext::MD_prof, + MDB.createBranchWeights(LikelyBranchWeightVal, + UnlikelyBranchWeightVal)); else if (IsOpndComingFromSuccessor(BI->getSuccessor(0))) - BI->setMetadata( - LLVMContext::MD_prof, - MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight)); + BI->setMetadata(LLVMContext::MD_prof, + MDB.createBranchWeights(UnlikelyBranchWeightVal, + LikelyBranchWeightVal)); } } @@ -277,7 +305,8 @@ } Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) + if (!Fn || (Fn->getIntrinsicID() != Intrinsic::expect && + Fn->getIntrinsicID() != Intrinsic::expect_with_probability)) return false; Value *ArgValue = CI->getArgOperand(0); @@ -289,13 +318,22 @@ MDNode *Node; MDNode *ExpNode; + std::pair WeightNums = + setBranchWeight(Fn->getIntrinsicID(), CI, 2); + uint32_t LikelyBranchWeightVal = WeightNums.first; + uint32_t UnlikelyBranchWeightVal = WeightNums.second; + if ((ExpectedValue->getZExtValue() == ValueComparedTo) == (Predicate == CmpInst::ICMP_EQ)) { - Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); - ExpNode = MDB.createMisExpect(0, LikelyBranchWeight, UnlikelyBranchWeight); + Node = + MDB.createBranchWeights(LikelyBranchWeightVal, UnlikelyBranchWeightVal); + ExpNode = + MDB.createMisExpect(0, LikelyBranchWeightVal, UnlikelyBranchWeightVal); } else { - Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); - ExpNode = MDB.createMisExpect(1, LikelyBranchWeight, UnlikelyBranchWeight); + Node = + MDB.createBranchWeights(UnlikelyBranchWeightVal, LikelyBranchWeightVal); + ExpNode = + MDB.createMisExpect(1, LikelyBranchWeightVal, UnlikelyBranchWeightVal); } BSI.setMetadata(LLVMContext::MD_misexpect, ExpNode); @@ -347,7 +385,8 @@ } Function *Fn = CI->getCalledFunction(); - if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { + if (Fn && (Fn->getIntrinsicID() == Intrinsic::expect || + Fn->getIntrinsicID() == Intrinsic::expect_with_probability)) { // Before erasing the llvm.expect, walk backward to find // phi that define llvm.expect's first arg, and // infer branch probability: Index: llvm/test/Transforms/LowerExpectIntrinsic/basic.ll =================================================================== --- llvm/test/Transforms/LowerExpectIntrinsic/basic.ll +++ llvm/test/Transforms/LowerExpectIntrinsic/basic.ll @@ -285,6 +285,29 @@ declare i1 @llvm.expect.i1(i1, i1) nounwind readnone +; CHECK-LABEL: @test11( +define dso_local void @test11(i32 %0) noinline nounwind optnone uwtable { + %2 = alloca i32, align 4 + store i32 %0, i32* %2, align 4 + %3 = load i32, i32* %2, align 4 + %4 = icmp sgt i32 %3, 0 + %5 = zext i1 %4 to i32 + %6 = sext i32 %5 to i64 + %7 = call i64 @llvm.expect.with.probability.i64(i64 %6, i64 1, double 8.000000e-01) + %8 = icmp ne i64 %7, 0 + br i1 %8, label %9, label %10 + +9: ; preds = %1 + call void (...) @ff() + br label %10 + +10: ; preds = %9, %1 + ret void +} + +declare i64 @llvm.expect.with.probability.i64(i64, i64, double) nounwind readnone willreturn +declare dso_local void @ff(...) + ; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1} ; CHECK: !1 = !{!"misexpect", i64 0, i64 2000, i64 1} ; CHECK: !2 = !{!"branch_weights", i32 1, i32 2000} @@ -292,3 +315,5 @@ ; CHECK: !4 = !{!"branch_weights", i32 1, i32 1, i32 2000} ; CHECK: !5 = !{!"misexpect", i64 2, i64 2000, i64 1} ; CHECK: !6 = !{!"branch_weights", i32 2000, i32 1, i32 1} +; CHECK: !7 = !{!"branch_weights", i32 1717986918, i32 429496731} +; CHECK: !8 = !{!"misexpect", i64 0, i64 1717986918, i64 429496731}