Index: lib/CodeGen/MachineBlockPlacement.cpp =================================================================== --- lib/CodeGen/MachineBlockPlacement.cpp +++ lib/CodeGen/MachineBlockPlacement.cpp @@ -363,22 +363,36 @@ uint32_t BestWeight = 0; uint32_t WeightScale = 0; uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale); - DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n"); + + // Adjust sum of weights by excluding weights on edges pointing to blocks that + // cannot be BB's successor in the chain. + uint32_t AdjustedSumWeight = SumWeight; + SmallVector Successors; for (MachineBasicBlock *Succ : BB->successors()) { - if (BlockFilter && !BlockFilter->count(Succ)) - continue; - BlockChain &SuccChain = *BlockToChain[Succ]; - if (&SuccChain == &Chain) { - DEBUG(dbgs() << " " << getBlockName(Succ) << " -> Already merged!\n"); - continue; - } - if (Succ != *SuccChain.begin()) { - DEBUG(dbgs() << " " << getBlockName(Succ) << " -> Mid chain!\n"); - continue; + bool SkipSucc = false; + if (BlockFilter && !BlockFilter->count(Succ)) { + SkipSucc = true; + } else { + BlockChain *SuccChain = BlockToChain[Succ]; + if (SuccChain == &Chain) { + DEBUG(dbgs() << " " << getBlockName(Succ) + << " -> Already merged!\n"); + SkipSucc = true; + } else if (Succ != *SuccChain->begin()) { + DEBUG(dbgs() << " " << getBlockName(Succ) << " -> Mid chain!\n"); + SkipSucc = true; + } } + if (SkipSucc) + AdjustedSumWeight -= MBPI->getEdgeWeight(BB, Succ) / WeightScale; + else + Successors.push_back(Succ); + } + DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n"); + for (MachineBasicBlock *Succ : Successors) { uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ); - BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight); + BranchProbability SuccProb(SuccWeight / WeightScale, AdjustedSumWeight); // If we outline optional branches, look whether Succ is unavoidable, i.e. // dominates all terminators of the MachineFunction. If it does, other @@ -406,6 +420,7 @@ // Only consider successors which are either "hot", or wouldn't violate // any CFG constraints. + BlockChain &SuccChain = *BlockToChain[Succ]; if (SuccChain.LoopPredecessors != 0) { if (SuccProb < HotProb) { DEBUG(dbgs() << " " << getBlockName(Succ) << " -> " << SuccProb @@ -415,8 +430,9 @@ // Make sure that a hot successor doesn't have a globally more // important predecessor. + BranchProbability RealSuccProb(SuccWeight / WeightScale, SumWeight); BlockFrequency CandidateEdgeFreq = - MBFI->getBlockFreq(BB) * SuccProb * HotProb.getCompl(); + MBFI->getBlockFreq(BB) * RealSuccProb * HotProb.getCompl(); bool BadCFGConflict = false; for (MachineBasicBlock *Pred : Succ->predecessors()) { if (Pred == Succ || (BlockFilter && !BlockFilter->count(Pred)) || Index: test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll @@ -0,0 +1,81 @@ +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s | FileCheck %s -check-prefix=CHECK + +define void @foo() { +; Test that when determining the edge probability from a node in an inner loop +; to a node in an outer loop, the weights on edges in the inner loop should be +; ignored. +; +; CHECK-LABEL: foo: +; CHECK: callq c +; CHECK: callq b + +entry: + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !1 + +if.then: + %call1 = call zeroext i1 @a() + br i1 %call1, label %while.body, label %if.end.1, !prof !1 + +while.body: + %call2 = call zeroext i1 @a() + br i1 %call2, label %if.then.1, label %while.cond + +if.then.1: + call void @d() + br label %while.cond + +while.cond: + %call3 = call zeroext i1 @a() + br i1 %call3, label %while.body, label %if.end + +if.end.1: + call void @d() + br label %if.end + +if.else: + call void @b() + br label %if.end + +if.end: + call void @c() + ret void +} + +define void @bar() { +; Test that when determining the edge probability from a node in a loop to a +; node in its peer loop, the weights on edges in the inner loop should be +; ignored. +; +; CHECK-LABEL: bar: +; CHECK: callq c +; CHECK: callq b + +entry: + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !1 + +if.then: + %call1 = call zeroext i1 @a() + br i1 %call1, label %if.then, label %while.body, !prof !2 + +while.body: + %call2 = call zeroext i1 @a() + br i1 %call2, label %while.body, label %if.end, !prof !2 + +if.else: + call void @b() + br label %if.end + +if.end: + call void @c() + ret void +} + +declare zeroext i1 @a() +declare void @b() +declare void @c() +declare void @d() + +!1 = !{!"branch_weights", i32 10, i32 1} +!2 = !{!"branch_weights", i32 100, i32 1}