diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -847,6 +847,15 @@ Weight = getEstimatedEdgeWeight(Edge); + if (isLoopEnteringEdge(Edge) && + // Avoid adjustment of ZERO weight since it should remain unchanged. + Weight != static_cast(BlockExecWeight::ZERO)) { + // Scale up loop entering weight by trip count, + // this is symmetrical with loop exiting edge. + Weight = + Weight.getValueOr(static_cast(BlockExecWeight::DEFAULT)) * + TC; + } if (isLoopExitingEdge(Edge) && // Avoid adjustment of ZERO weight since it should remain unchanged. Weight != static_cast(BlockExecWeight::ZERO)) { diff --git a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll @@ -143,8 +143,8 @@ entry: %cond1 = icmp eq i32 %a, 42 br i1 %cond1, label %header, label %exit -; CHECK: edge entry -> header probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge entry -> exit probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge entry -> header probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge entry -> exit probability is 0x04000000 / 0x80000000 = 3.12% header: br label %body diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll @@ -56,8 +56,8 @@ %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc5, %for.end ] call void @g1() br i1 %cmp27, label %for.body3, label %for.end -; CHECK: edge for.body -> for.body3 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge for.body -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body -> for.body3 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge for.body -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body3: %j.08 = phi i32 [ %inc, %for.body3 ], [ 0, %for.body ] @@ -92,8 +92,8 @@ %0 = load i32, i32* %c, align 4 %cmp = icmp slt i32 %0, 42 br i1 %cmp, label %do.body1, label %if.end -; CHECK: edge do.body -> do.body1 probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge do.body -> if.end probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge do.body -> do.body1 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge do.body -> if.end probability is 0x04000000 / 0x80000000 = 3.12% do.body1: %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ] @@ -128,8 +128,8 @@ %0 = load i32, i32* %c, align 4 %cmp = icmp slt i32 %0, 42 br i1 %cmp, label %return, label %do.body1 -; CHECK: edge do.body -> return probability is 0x04000000 / 0x80000000 = 3.12% -; CHECK: edge do.body -> do.body1 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge do.body -> return probability is 0x00220ff7 / 0x80000000 = 0.10% +; CHECK: edge do.body -> do.body1 probability is 0x7fddf009 / 0x80000000 = 99.90% [HOT edge] do.body1: %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ] @@ -268,8 +268,8 @@ if.end: call void @g1() br i1 %cmp38, label %for.body4, label %for.end -; CHECK: edge if.end -> for.body4 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge if.end -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge if.end -> for.body4 probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge if.end -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body4: %j.09 = phi i32 [ %inc, %for.body4 ], [ 0, %if.end ] @@ -315,8 +315,8 @@ %i.019 = phi i32 [ 0, %for.body.lr.ph ], [ %inc14, %for.end ] call void @g1() br i1 %cmp216, label %for.body3, label %for.end -; CHECK: edge for.body -> for.body3 probability is 0x50000000 / 0x80000000 = 62.50% -; CHECK: edge for.body -> for.end probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body -> for.body3 probability is 0x7c000000 / 0x80000000 = 96.88% +; CHECK: edge for.body -> for.end probability is 0x04000000 / 0x80000000 = 3.12% for.body3: %j.017 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ] @@ -370,8 +370,8 @@ entry: %tobool = icmp eq i32 %c, 0 br i1 %tobool, label %if.end, label %midloop -; CHECK: edge entry -> if.end probability is 0x30000000 / 0x80000000 = 37.50% -; CHECK: edge entry -> midloop probability is 0x50000000 / 0x80000000 = 62.50% +; CHECK: edge entry -> if.end probability is 0x04000000 / 0x80000000 = 3.12% +; CHECK: edge entry -> midloop probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] if.end: br label %for.cond @@ -524,8 +524,8 @@ ; If loop has single exit and it leads to 'cold' block then edge leading to loop enter ; should be considered 'cold' as well. define void @test13() { -; CHECK: edge entry -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge entry -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge entry -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge entry -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] ; CHECK: edge loop -> cold probability is 0x0041edfd / 0x80000000 = 0.20% ; CHECK: edge cold -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] @@ -577,8 +577,8 @@ ; If loop has multiple low probability exits then edge leading to loop enter ; should be considered low probable as well. define void @test15() { -; CHECK: edge entry -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge entry -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge entry -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge entry -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ; CHECK: edge cont -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -611,8 +611,8 @@ ; This is the same case as test15 but with additional loop 'preheader' block. define void @test16() { ; CHECK: edge entry -> preheader probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge preheader -> loop probability is 0x078780e3 / 0x80000000 = 5.88% -; CHECK: edge preheader -> exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] +; CHECK: edge preheader -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% +; CHECK: edge preheader -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% ; CHECK: edge loop -> cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ; CHECK: edge cont -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -651,8 +651,8 @@ ; Check that exit to 'cold' and 'noreturn' has lower probability than 'normal' exit. define void @test17() { ; CHECK: edge entry -> preheader probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge preheader -> loop probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge preheader -> exit probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge preheader -> loop probability is 0x7c000000 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge preheader -> exit probability is 0x04000000 / 0x80000000 = 3.12% ; CHECK: edge loop -> cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] ; CHECK: edge loop -> noreturn probability is 0x00000800 / 0x80000000 = 0.00% ; CHECK: edge cont -> cont2 probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] @@ -694,14 +694,14 @@ ; low probable exit what encreases robability to take exit in the top level loop. define void @test18() { ; CHECK: edge entry -> top.loop probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge top.loop -> loop probability is 0x546cd4b7 / 0x80000000 = 65.96% -; CHECK: edge top.loop -> exit probability is 0x2b932b49 / 0x80000000 = 34.04% +; CHECK: edge top.loop -> loop probability is 0x7de75ca9 / 0x80000000 = 98.36% [HOT edge] +; CHECK: edge top.loop -> exit probability is 0x0218a357 / 0x80000000 = 1.64% ; CHECK: edge loop -> loop probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] ; CHECK: edge loop -> cold probability is 0x0041edfd / 0x80000000 = 0.20% ; CHECK: edge cold -> top.loop probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] entry: - br label %top.loop + br label %top.loop top.loop: %j.0 = phi i32 [ 0, %entry ], [ %j.inc, %cold ] diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll @@ -84,8 +84,8 @@ entry: %cond1 = icmp eq i32 %a, 42 br i1 %cond1, label %header, label %exit -; CHECK: edge entry -> header probability is 0x00000800 / 0x80000000 = 0.00% -; CHECK: edge entry -> exit probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge entry -> header probability is 0x0000f7fe / 0x80000000 = 0.00% +; CHECK: edge entry -> exit probability is 0x7fff0802 / 0x80000000 = 100.00% [HOT edge] header: br label %body diff --git a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll --- a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll +++ b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll @@ -1386,29 +1386,28 @@ ; ENABLE-NEXT: jmp LBB16_1 ; ENABLE-NEXT: LBB16_2: ## %split ; ENABLE-NEXT: movq _irreducibleCFGb@GOTPCREL(%rip), %rax +; ENABLE-NEXT: xorl %ebx, %ebx ; ENABLE-NEXT: cmpl $0, (%rax) -; ENABLE-NEXT: je LBB16_3 -; ENABLE-NEXT: ## %bb.4: ## %for.body4.i -; ENABLE-NEXT: movq _irreducibleCFGa@GOTPCREL(%rip), %rax -; ENABLE-NEXT: movl (%rax), %edi -; ENABLE-NEXT: xorl %ebx, %ebx -; ENABLE-NEXT: xorl %eax, %eax -; ENABLE-NEXT: callq _something -; ENABLE-NEXT: jmp LBB16_5 -; ENABLE-NEXT: LBB16_3: -; ENABLE-NEXT: xorl %ebx, %ebx +; ENABLE-NEXT: jne LBB16_3 ; ENABLE-NEXT: .p2align 4, 0x90 -; ENABLE-NEXT: LBB16_5: ## %for.inc +; ENABLE-NEXT: LBB16_4: ## %for.inc ; ENABLE-NEXT: ## =>This Inner Loop Header: Depth=1 ; ENABLE-NEXT: incl %ebx ; ENABLE-NEXT: cmpl $7, %ebx -; ENABLE-NEXT: jl LBB16_5 -; ENABLE-NEXT: ## %bb.6: ## %fn1.exit +; ENABLE-NEXT: jl LBB16_4 +; ENABLE-NEXT: ## %bb.5: ## %fn1.exit ; ENABLE-NEXT: xorl %eax, %eax ; ENABLE-NEXT: addq $8, %rsp ; ENABLE-NEXT: popq %rbx ; ENABLE-NEXT: popq %rbp ; ENABLE-NEXT: retq +; ENABLE-NEXT: LBB16_3: ## %for.body4.i +; ENABLE-NEXT: movq _irreducibleCFGa@GOTPCREL(%rip), %rax +; ENABLE-NEXT: movl (%rax), %edi +; ENABLE-NEXT: xorl %ebx, %ebx +; ENABLE-NEXT: xorl %eax, %eax +; ENABLE-NEXT: callq _something +; ENABLE-NEXT: jmp LBB16_4 ; ; DISABLE-LABEL: irreducibleCFG: ; DISABLE: ## %bb.0: ## %entry @@ -1429,29 +1428,28 @@ ; DISABLE-NEXT: jmp LBB16_1 ; DISABLE-NEXT: LBB16_2: ## %split ; DISABLE-NEXT: movq _irreducibleCFGb@GOTPCREL(%rip), %rax +; DISABLE-NEXT: xorl %ebx, %ebx ; DISABLE-NEXT: cmpl $0, (%rax) -; DISABLE-NEXT: je LBB16_3 -; DISABLE-NEXT: ## %bb.4: ## %for.body4.i -; DISABLE-NEXT: movq _irreducibleCFGa@GOTPCREL(%rip), %rax -; DISABLE-NEXT: movl (%rax), %edi -; DISABLE-NEXT: xorl %ebx, %ebx -; DISABLE-NEXT: xorl %eax, %eax -; DISABLE-NEXT: callq _something -; DISABLE-NEXT: jmp LBB16_5 -; DISABLE-NEXT: LBB16_3: -; DISABLE-NEXT: xorl %ebx, %ebx +; DISABLE-NEXT: jne LBB16_3 ; DISABLE-NEXT: .p2align 4, 0x90 -; DISABLE-NEXT: LBB16_5: ## %for.inc +; DISABLE-NEXT: LBB16_4: ## %for.inc ; DISABLE-NEXT: ## =>This Inner Loop Header: Depth=1 ; DISABLE-NEXT: incl %ebx ; DISABLE-NEXT: cmpl $7, %ebx -; DISABLE-NEXT: jl LBB16_5 -; DISABLE-NEXT: ## %bb.6: ## %fn1.exit +; DISABLE-NEXT: jl LBB16_4 +; DISABLE-NEXT: ## %bb.5: ## %fn1.exit ; DISABLE-NEXT: xorl %eax, %eax ; DISABLE-NEXT: addq $8, %rsp ; DISABLE-NEXT: popq %rbx ; DISABLE-NEXT: popq %rbp ; DISABLE-NEXT: retq +; DISABLE-NEXT: LBB16_3: ## %for.body4.i +; DISABLE-NEXT: movq _irreducibleCFGa@GOTPCREL(%rip), %rax +; DISABLE-NEXT: movl (%rax), %edi +; DISABLE-NEXT: xorl %ebx, %ebx +; DISABLE-NEXT: xorl %eax, %eax +; DISABLE-NEXT: callq _something +; DISABLE-NEXT: jmp LBB16_4 entry: %i0 = load i32, i32* @irreducibleCFGa, align 4 %.pr = load i8, i8* @irreducibleCFGf, align 1 diff --git a/llvm/test/ThinLTO/X86/function_entry_count.ll b/llvm/test/ThinLTO/X86/function_entry_count.ll --- a/llvm/test/ThinLTO/X86/function_entry_count.ll +++ b/llvm/test/ThinLTO/X86/function_entry_count.ll @@ -18,7 +18,7 @@ ; CHECK: define void @f(i32{{.*}}) [[ATTR:#[0-9]+]] !prof ![[PROF1:[0-9]+]] ; CHECK: define available_externally void @g() !prof ![[PROF2]] ; CHECK-DAG: ![[PROF1]] = !{!"synthetic_function_entry_count", i64 10} -; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 198} +; CHECK-DAG: ![[PROF2]] = !{!"synthetic_function_entry_count", i64 308} ; CHECK-DAG: attributes [[ATTR]] = { norecurse nounwind } target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll --- a/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll +++ b/llvm/test/Transforms/ConstantHoisting/AArch64/consthoist-unreachable.ll @@ -10,14 +10,13 @@ define void @c() { ; CHECK-LABEL: @c( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[CONST:%.*]] = bitcast i32 1232131 to i32 ; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i16 0, 0 ; CHECK-NEXT: br i1 undef, label [[LBL1_US:%.*]], label [[ENTRY_ENTRY_SPLIT_CRIT_EDGE:%.*]] ; CHECK: entry.entry.split_crit_edge: -; CHECK-NEXT: [[CONST:%.*]] = bitcast i32 1232131 to i32 ; CHECK-NEXT: br label [[LBL1:%.*]] ; CHECK: lbl1.us: -; CHECK-NEXT: [[CONST1:%.*]] = bitcast i32 1232131 to i32 -; CHECK-NEXT: store i32 [[CONST1]], i32* @c.a, align 1 +; CHECK-NEXT: store i32 [[CONST]], i32* @c.a, align 1 ; CHECK-NEXT: br label [[FOR_COND4:%.*]] ; CHECK: lbl1: ; CHECK-NEXT: store i32 [[CONST]], i32* @c.a, align 1 diff --git a/llvm/test/Transforms/PGOProfile/bfi_verification.ll b/llvm/test/Transforms/PGOProfile/bfi_verification.ll --- a/llvm/test/Transforms/PGOProfile/bfi_verification.ll +++ b/llvm/test/Transforms/PGOProfile/bfi_verification.ll @@ -98,10 +98,10 @@ ret void } ; THRESHOLD-CHECK: remark: :0:0: BB do.body Count=39637749 BFI_Count=40801304 -; THRESHOLD-CHECK: remark: :0:0: BB while.cond Count=80655628 BFI_Count=83956530 -; THRESHOLD-CHECK: remark: :0:0: BB while.body Count=41017879 BFI_Count=42370585 -; THRESHOLD-CHECK: remark: :0:0: BB while.cond3 Count=71254487 BFI_Count=73756204 -; THRESHOLD-CHECK: remark: :0:0: BB while.body7 Count=31616738 BFI_Count=32954900 +; THRESHOLD-CHECK: remark: :0:0: BB while.cond Count=71254487 BFI_Count=73756204 +; THRESHOLD-CHECK: remark: :0:0: BB while.body Count=31616738 BFI_Count=32954900 +; THRESHOLD-CHECK: remark: :0:0: BB while.cond3 Count=80655628 BFI_Count=83956530 +; THRESHOLD-CHECK: remark: :0:0: BB while.body7 Count=41017879 BFI_Count=42370585 ; THRESHOLD-CHECK: remark: :0:0: BB while.end8 Count=39637749 BFI_Count=40801304 ; THRESHOLD-CHECK: remark: :0:0: BB if.then Count=32743703 BFI_Count=33739540 ; THRESHOLD-CHECK: remark: :0:0: BB if.end Count=39637749 BFI_Count=40801304 diff --git a/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll --- a/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll +++ b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll @@ -46,5 +46,5 @@ !1 = !{!"branch_weights", i32 1, i32 99} !2 = !{!"branch_weights", i32 1, i32 1} ; CHECK: ![[COUNT1]] = !{!"synthetic_function_entry_count", i64 10} -; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 520} -; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 260} +; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 988} +; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 494}