diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp --- a/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-if-branch-weights.cpp @@ -1,5 +1,5 @@ // RUN: %clang_cc1 -O1 -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck -DLIKELY=2000 -DUNLIKELY=1 %s -// RUN: %clang_cc1 -O1 -emit-llvm %s -triple=x86_64-linux-gnu -mllvm -likely-branch-weight=99 -mllvm -unlikely-branch-weight=42 -o - | FileCheck -DLIKELY=99 -DUNLIKELY=42 %s +// RUN: %clang_cc1 -O1 -emit-llvm %s -triple=x86_64-linux-gnu -mllvm -likely-branch-weight=99 -o - | FileCheck -DLIKELY=99 %s extern volatile bool b; extern volatile int i; @@ -144,5 +144,5 @@ } } -// CHECK: ![[PROF_LIKELY]] = !{!"branch_weights", i32 [[UNLIKELY]], i32 [[LIKELY]]} -// CHECK: ![[PROF_UNLIKELY]] = !{!"branch_weights", i32 [[LIKELY]], i32 [[UNLIKELY]]} +// CHECK: ![[PROF_LIKELY]] = !{!"branch_weights", i32 1, i32 [[LIKELY]]} +// CHECK: ![[PROF_UNLIKELY]] = !{!"branch_weights", i32 [[LIKELY]], i32 1 diff --git a/llvm/docs/MisExpect.rst b/llvm/docs/MisExpect.rst --- a/llvm/docs/MisExpect.rst +++ b/llvm/docs/MisExpect.rst @@ -34,12 +34,12 @@ We calculate the threshold for emitting MisExpect related diagnostics based on the values the compiler assigns to ``llvm.expect`` intrinsics, -which can be set through the ``-likely-branch-weight`` and -``-unlikely-branch-weight`` LLVM options. During verification, if the -profile weights mismatch the calculated threshold, then we will emit a -remark or warning detailing a potential performance regression. The -diagnostic also reports the percentage of the time the annotation was -correct during profiling to help developers reason about how to proceed. +which can be set through the ``-likely-branch-weight`` LLVM option. +During verification, if the profile weights mismatch the calculated threshold, +then we will emit a remark or warning detailing a potential performance +regression. The diagnostic also reports the percentage of the time the +annotation was correct during profiling to help developers reason about how +to proceed. The diagnostics are also available in the form of optimization remarks, which can be serialized and processed through the ``opt-viewer.py`` diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -99,5 +99,12 @@ /// metadata was found. bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); +/// Return the weight for a "likely branch". A 1/LikelyBranchWeight ratio +/// corresponds to the probability of `llvm.expect` matching the expected +/// value. +/// Note: There is also `TargetTransformInfo::getPredictableBranchThreshold` +/// for a value matching the current target. +uint32_t getLikelyBranchWeight(); + } // namespace llvm #endif diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -23,6 +23,18 @@ using namespace llvm; +// These default values are chosen to represent an extremely skewed outcome for +// a condition, but they leave some room for interpretation by later passes. +// +// If the documentation for __builtin_expect() was made explicit that it should +// only be used in extreme cases, we could make this ratio higher. As it stands, +// programmers may be using __builtin_expect() / llvm.expect to annotate that a +// branch is likely or unlikely to be taken. + +static cl::opt LikelyBranchWeight( + "likely-branch-weight", cl::Hidden, cl::init(2000), + cl::desc("Weight of the branch likely to be taken (default = 2000)")); + namespace { // MD_prof nodes have the following layout @@ -184,4 +196,6 @@ return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal); } +uint32_t getLikelyBranchWeight() { return LikelyBranchWeight; } + } // namespace llvm diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp --- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/ProfDataUtils.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" @@ -36,31 +37,11 @@ STATISTIC(ExpectIntrinsicsHandled, "Number of 'expect' intrinsic instructions handled"); -// These default values are chosen to represent an extremely skewed outcome for -// a condition, but they leave some room for interpretation by later passes. -// -// If the documentation for __builtin_expect() was made explicit that it should -// only be used in extreme cases, we could make this ratio higher. As it stands, -// programmers may be using __builtin_expect() / llvm.expect to annotate that a -// branch is likely or unlikely to be taken. - -// WARNING: these values are internal implementation detail of the pass. -// They should not be exposed to the outside of the pass, front-end codegen -// should emit @llvm.expect intrinsics instead of using these weights directly. -// Transforms should use TargetTransformInfo's getPredictableBranchThreshold(). -static cl::opt LikelyBranchWeight( - "likely-branch-weight", cl::Hidden, cl::init(2000), - cl::desc("Weight of the branch likely to be taken (default = 2000)")); -static cl::opt UnlikelyBranchWeight( - "unlikely-branch-weight", cl::Hidden, cl::init(1), - cl::desc("Weight of the branch unlikely to be taken (default = 1)")); - static std::tuple getBranchWeight(Intrinsic::ID IntrinsicID, CallInst *CI, int BranchCount) { if (IntrinsicID == Intrinsic::expect) { // __builtin_expect - return std::make_tuple(LikelyBranchWeight.getValue(), - UnlikelyBranchWeight.getValue()); + return std::make_tuple(getLikelyBranchWeight(), 1); } else { // __builtin_expect_with_probability assert(CI->getNumOperands() >= 3 && diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='function(lower-expect),strip-dead-prototypes' -likely-branch-weight=2147483647 -unlikely-branch-weight=1 < %s | FileCheck %s +; RUN: opt -S -passes='function(lower-expect),strip-dead-prototypes' -likely-branch-weight=2147483647 < %s | FileCheck %s ; The C case ; if (__builtin_expect_with_probability(((a0 == 1) || (a1 == 1) || (a2 == 1)), 1, 0)) diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll --- a/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll +++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll @@ -2,7 +2,7 @@ ; This can happen when the sum of all counters exceeds the max size of uint32_t ; RUN: llvm-profdata merge %S/Inputs/misexpect-branch-overflow.proftext -o %t.profdata -; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -unlikely-branch-weight=2147483648 -likely-branch-weight=2147483648 -S 2>&1 | FileCheck %s --check-prefix=OVERFLOW +; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=OVERFLOW ; OVERFLOW-NOT: warning: misexpect-branch.c:22:0: 50.00% ; OVERFLOW-NOT: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 50.00% (2147483648 / 4294967296) of profiled executions. @@ -32,9 +32,8 @@ %lnot1 = xor i1 %lnot, true, !dbg !15 %lnot.ext = zext i1 %lnot1 to i32, !dbg !15 %conv = sext i32 %lnot.ext to i64, !dbg !15 - %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1), !dbg !15 - %tobool = icmp ne i64 %expval, 0, !dbg !15 - br i1 %tobool, label %if.then, label %if.else, !dbg !15 + %tobool = icmp ne i64 %conv, 0, !dbg !15 + br i1 %tobool, label %if.then, label %if.else, !dbg !15, !prof !21 if.then: ; preds = %entry %1 = load i32, ptr %rando, align 4, !dbg !16, !tbaa !10 @@ -100,3 +99,4 @@ !18 = !DILocation(line: 25, scope: !6) !19 = !DILocation(line: 27, scope: !6) !20 = !DILocation(line: 28, scope: !6) +!21 = !{!"branch_weights", i32 2147483648, i32 2147483648}