diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -847,6 +847,16 @@ Weight = getEstimatedEdgeWeight(Edge); + if (isLoopEnteringEdge(Edge) && SuccLoopBB.getLoop() && + SuccLoopBB.getLoop()->isRotatedForm() && + // Avoid adjustment of ZERO weight since it should remain unchanged. + Weight != static_cast(BlockExecWeight::ZERO)) { + // Scale up loop entering weight by trip count, + // this is symmetrical with loop exiting edge. + Weight = + Weight.getValueOr(static_cast(BlockExecWeight::DEFAULT)) * + TC; + } if (isLoopExitingEdge(Edge) && // Avoid adjustment of ZERO weight since it should remain unchanged. Weight != static_cast(BlockExecWeight::ZERO)) { diff --git a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll @@ -143,8 +143,8 @@ entry: %cond1 = icmp eq i32 %a, 42 br i1 %cond1, label %header, label %exit -; CHECK: edge entry -> header probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge entry -> exit probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge entry -> header probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge entry -> exit probability is 0x04000000 / 0x80000000 = 3.12% header: br label %body diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll @@ -56,8 +56,8 @@ %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc5, %for.end ] call void @g1() br i1 %cmp27, label %for.body3, label %for.end -; CHECK: edge for.body -> for.body3 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge for.body -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body -> for.body3 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge for.body -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body3: %j.08 = phi i32 [ %inc, %for.body3 ], [ 0, %for.body ] @@ -92,8 +92,8 @@ %0 = load i32, i32* %c, align 4 %cmp = icmp slt i32 %0, 42 br i1 %cmp, label %do.body1, label %if.end -; CHECK: edge do.body -> do.body1 probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge do.body -> if.end probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge do.body -> do.body1 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge do.body -> if.end probability is 0x04000000 / 0x80000000 = 3.12% do.body1: %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ] @@ -128,8 +128,8 @@ %0 = load i32, i32* %c, align 4 %cmp = icmp slt i32 %0, 42 br i1 %cmp, label %return, label %do.body1 -; CHECK: edge do.body -> return probability is 0x04000000 / 0x80000000 = 3.12% -; CHECK: edge do.body -> do.body1 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge do.body -> return probability is 0x00220ff7 / 0x80000000 = 0.10% +; CHECK: edge do.body -> do.body1 probability is 0x7fddf009 / 0x80000000 = 99.90% [HOT edge] do.body1: %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ] @@ -268,8 +268,8 @@ if.end: call void @g1() br i1 %cmp38, label %for.body4, label %for.end -; CHECK: edge if.end -> for.body4 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge if.end -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge if.end -> for.body4 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge if.end -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body4: %j.09 = phi i32 [ %inc, %for.body4 ], [ 0, %if.end ] @@ -315,8 +315,8 @@ %i.019 = phi i32 [ 0, %for.body.lr.ph ], [ %inc14, %for.end ] call void @g1() br i1 %cmp216, label %for.body3, label %for.end -; CHECK: edge for.body -> for.body3 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge for.body -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body -> for.body3 probability is 0x7c000000 / 0x80000000 = 96.88% +; CHECK: edge for.body -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body3: %j.017 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ] @@ -524,8 +524,8 @@ ; If loop has single exit and it leads to 'cold' block then edge leading to loop enter ; should be considered 'cold' as well. define void @test13() { -; CHECK: edge entry -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge entry -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge entry -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge entry -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] ; CHECK: edge loop -> cold probability is 0x0041edfd / 0x80000000 = 0.20% ; CHECK: edge cold -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] @@ -577,8 +577,8 @@ ; If loop has multiple low probability exits then edge leading to loop enter ; should be considered low probable as well. define void @test15() { -; CHECK: edge entry -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge entry -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge entry -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge entry -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ; CHECK: edge cont -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -611,8 +611,8 @@ ; This is the same case as test15 but with additional loop 'preheader' block. define void @test16() { ; CHECK: edge entry -> preheader probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge preheader -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge preheader -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge preheader -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge preheader -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ; CHECK: edge cont -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -651,8 +651,8 @@ ; Check that exit to 'cold' and 'noreturn' has lower probability than 'normal' exit. define void @test17() { ; CHECK: edge entry -> preheader probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge preheader -> loop probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge preheader -> exit probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge preheader -> loop probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge preheader -> exit probability is 0x04000000 / 0x80000000 = 3.12% ; CHECK: edge loop -> cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> noreturn probability is 0x00000800 / 0x80000000 = 0.00% ; CHECK: edge cont -> cont2 probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -694,14 +694,14 @@ ; low probable exit what encreases robability to take exit in the top level loop. define void @test18() { ; CHECK: edge entry -> top.loop probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge top.loop -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% -; CHECK: edge top.loop -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% +; CHECK: edge top.loop -> loop probability is 0x7de75ca9 / 0x80000000 = 98.36% [HOT edge] +; CHECK: edge top.loop -> exit probability is 0x0218a357 / 0x80000000 = 1.64% ; CHECK: edge loop -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] ; CHECK: edge loop -> cold probability is 0x0041edfd / 0x80000000 = 0.20% ; CHECK: edge cold -> top.loop probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] entry: - br label %top.loop + br label %top.loop top.loop: %j.0 = phi i32 [ 0, %entry ], [ %j.inc, %cold ] diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll @@ -84,8 +84,8 @@ entry: %cond1 = icmp eq i32 %a, 42 br i1 %cond1, label %header, label %exit -; CHECK: edge entry -> header probability is 0x00000800 / 0x80000000 = 0.00% -; CHECK: edge entry -> exit probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge entry -> header probability is 0x0000f7fe / 0x80000000 = 0.00% +; CHECK: edge entry -> exit probability is 0x7fff0802 / 0x80000000 = 100.00% [HOT edge] header: br label %body diff --git a/llvm/test/ThinLTO/X86/function_entry_count.ll b/llvm/test/ThinLTO/X86/function_entry_count.ll --- a/llvm/test/ThinLTO/X86/function_entry_count.ll +++ b/llvm/test/ThinLTO/X86/function_entry_count.ll @@ -18,7 +18,7 @@ ; CHECK: define void @f(i32{{.*}}) [[ATTR:#[0-9]+]] !prof ![[PROF1:[0-9]+]] ; CHECK: define available_externally void @g() !prof ![[PROF2]] ; CHECK-DAG: ![[PROF1]] = !{!"synthetic_function_entry_count", i64 10} -; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 198} +; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 308} ; CHECK-DAG: attributes [[ATTR]] = { norecurse nounwind } target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll --- a/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll +++ b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll @@ -46,5 +46,5 @@ !1 = !{!"branch_weights", i32 1, i32 99} !2 = !{!"branch_weights", i32 1, i32 1} ; CHECK: ![[COUNT1]] = !{!"synthetic_function_entry_count", i64 10} -; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 520} -; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 260} +; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 988} +; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 494}