diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -315,7 +315,7 @@ bool calcColdCallHeuristics(const BasicBlock *BB); bool calcPointerHeuristics(const BasicBlock *BB); bool calcLoopBranchHeuristics(const BasicBlock *BB, const LoopInfo &LI); - bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI); + bool calcIntegerHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI); bool calcFloatingPointHeuristics(const BasicBlock *BB); bool calcInvokeHeuristics(const BasicBlock *BB); }; diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -856,7 +856,7 @@ return true; } -bool BranchProbabilityInfo::calcZeroHeuristics(const BasicBlock *BB, +bool BranchProbabilityInfo::calcIntegerHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI) { const BranchInst *BI = dyn_cast(BB->getTerminator()); if (!BI || !BI->isConditional()) @@ -873,10 +873,24 @@ return dyn_cast(V); }; + BranchProbability TakenProb(ZH_TAKEN_WEIGHT, + ZH_TAKEN_WEIGHT + ZH_NONTAKEN_WEIGHT); + BranchProbability UntakenProb(ZH_NONTAKEN_WEIGHT, + ZH_TAKEN_WEIGHT + ZH_NONTAKEN_WEIGHT); + bool isProb; Value *RHS = CI->getOperand(1); ConstantInt *CV = GetConstantInt(RHS); - if (!CV) - return false; + if (!CV) { + // X == Y -> Unlikely + // X != Y -> Likely + isProb = !CI->isTrueWhenEqual(); + + if (!isProb) + std::swap(TakenProb, UntakenProb); + setEdgeProbability( + BB, SmallVector({TakenProb, UntakenProb})); + return true; + } // If the LHS is the result of AND'ing a value with a single bit bitmask, // we don't have information about probabilities. @@ -893,7 +907,6 @@ if (Function *CalledFn = Call->getCalledFunction()) TLI->getLibFunc(*CalledFn, Func); - bool isProb; if (Func == LibFunc_strcasecmp || Func == LibFunc_strcmp || Func == LibFunc_strncasecmp || @@ -964,10 +977,6 @@ return false; } - BranchProbability TakenProb(ZH_TAKEN_WEIGHT, - ZH_TAKEN_WEIGHT + ZH_NONTAKEN_WEIGHT); - BranchProbability UntakenProb(ZH_NONTAKEN_WEIGHT, - ZH_TAKEN_WEIGHT + ZH_NONTAKEN_WEIGHT); if (!isProb) std::swap(TakenProb, UntakenProb); @@ -1221,7 +1230,7 @@ continue; if (calcPointerHeuristics(BB)) continue; - if (calcZeroHeuristics(BB, TLI)) + if (calcIntegerHeuristics(BB, TLI)) continue; if (calcFloatingPointHeuristics(BB)) continue; diff --git a/llvm/test/Analysis/BranchProbabilityInfo/integer_heuristics.ll b/llvm/test/Analysis/BranchProbabilityInfo/integer_heuristics.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/integer_heuristics.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/integer_heuristics.ll @@ -101,3 +101,36 @@ exit: ret void } + +declare void @foo() + +define i32 @foo1(i32 %x, i32 %y, i8 signext %z, i8 signext %w) { +entry: + %c = icmp eq i32 %x, %y + br i1 %c, label %then, label %else +; CHECK: edge entry -> then probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge entry -> else probability is 0x50000000 / 0x80000000 = 62.50% +then: + tail call void @foo() + br label %else +; CHECK: edge then -> else probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +else: + %v = phi i8 [ %z, %then ], [ %w, %entry ] + %r = sext i8 %v to i32 + ret i32 %r +} + +define i32 @foo2(i32 %x, i32 %y, i8 signext %z, i8 signext %w) { +entry: + %c = icmp ne i32 %x, %y + br i1 %c, label %then, label %else +; CHECK: edge entry -> then probability is 0x50000000 / 0x80000000 = 62.50% +; CHECK: edge entry -> else probability is 0x30000000 / 0x80000000 = 37.50% +then: + br label %else +; CHECK: edge then -> else probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +else: + %v = phi i8 [ %z, %then ], [ %w, %entry ] + %r = sext i8 %v to i32 + ret i32 %r +} diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll --- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll @@ -263,8 +263,8 @@ %0 = load i32, i32* %c, align 4 %cmp1 = icmp eq i32 %0, %i.011 br i1 %cmp1, label %for.inc5, label %if.end -; CHECK: edge for.body -> for.inc5 probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge for.body -> if.end probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge for.body -> for.inc5 probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body -> if.end probability is 0x50000000 / 0x80000000 = 62.50% if.end: call void @g1() @@ -324,22 +324,22 @@ %0 = load i32, i32* %c, align 4 %cmp4 = icmp eq i32 %0, %j.017 br i1 %cmp4, label %for.inc, label %if.end -; CHECK: edge for.body3 -> for.inc probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge for.body3 -> if.end probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge for.body3 -> for.inc probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge for.body3 -> if.end probability is 0x50000000 / 0x80000000 = 62.50% if.end: %1 = load i32, i32* %arrayidx5, align 4 %cmp6 = icmp eq i32 %1, %j.017 br i1 %cmp6, label %for.inc, label %if.end8 -; CHECK: edge if.end -> for.inc probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge if.end -> if.end8 probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge if.end -> for.inc probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge if.end -> if.end8 probability is 0x50000000 / 0x80000000 = 62.50% if.end8: %2 = load i32, i32* %arrayidx9, align 4 %cmp10 = icmp eq i32 %2, %j.017 br i1 %cmp10, label %for.inc, label %if.end12 -; CHECK: edge if.end8 -> for.inc probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge if.end8 -> if.end12 probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge if.end8 -> for.inc probability is 0x30000000 / 0x80000000 = 37.50% +; CHECK: edge if.end8 -> if.end12 probability is 0x50000000 / 0x80000000 = 62.50% if.end12: call void @g2()