Index: llvm/lib/Analysis/BranchProbabilityInfo.cpp =================================================================== --- llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -74,6 +74,70 @@ char BranchProbabilityInfoWrapperPass::ID = 0; +/// Heuristics and lookup tables for non-loop branches: + +#define Unlikely false +#define Likely true + +/// Integer compares with 0: +/// X == 0 -> Unlikely +/// X != 0 -> Likely +/// X < 0 -> Unlikely +/// X > 0 -> Likely +static std::map ICmpWithZeroTable { + { CmpInst::ICMP_EQ, Unlikely }, + { CmpInst::ICMP_NE, Likely }, + { CmpInst::ICMP_SLT, Unlikely }, + { CmpInst::ICMP_SGT, Likely }, +}; + +/// Integer compares with -1: +/// X == -1 -> Unlikely +/// X != -1 -> Likely +/// X >= 0 -> Likely (InstCombine canonicalizes X >= 0 into X > -1) +static std::map ICmpWithMinusOneTable { + { CmpInst::ICMP_EQ, Unlikely }, + { CmpInst::ICMP_NE, Likely }, + { CmpInst::ICMP_SGT, Likely }, +}; + +/// Integer compares with 1: +/// X <= 0 -> Unlikely +/// (InstCombine canonicalizes X <= 0 into X < 1) +static std::map ICmpWithOneTable { + { CmpInst::ICMP_SLT, Unlikely }, +}; + +/// strcmp and similar functions return zero, negative, or positive, if the +/// first string is equal, less, or greater than the second. We consider it +/// likely that the strings are not equal, so a comparison with zero is +/// probably false, but also a comparison with any other number is also +/// probably false given that what exactly is returned for nonzero values is +/// not specified. Any kind of comparison other than equality we know +/// nothing about. +static std::map ICmpWithLibCallTable { + { CmpInst::ICMP_EQ, Unlikely }, + { CmpInst::ICMP_NE, Likely }, +}; + +/// Floating-Point compares: +/// !isnan -> Likely +/// isnan -> Unlikely +static std::map FCmpTable { + { FCmpInst::FCMP_ORD, Likely }, + { FCmpInst::FCMP_UNO, Unlikely }, +}; + +/// Pointer comparisons: +/// p != 0 -> Likely +/// p == 0 -> Unlikely +/// p != q -> Likely +/// p == q -> Unlikely +static std::map PointerTable { + { ICmpInst::ICMP_NE, Unlikely }, + { ICmpInst::ICMP_EQ, Likely }, +}; + // Weights are for internal use only. They are used by heuristics to help to // estimate edges' probability. Example: // @@ -104,12 +168,15 @@ /// All reachable probability will proportionally share the remaining part. static const BranchProbability UR_TAKEN_PROB = BranchProbability::getRaw(1); +// Pointer Heuristics (PH) static const uint32_t PH_TAKEN_WEIGHT = 20; static const uint32_t PH_NONTAKEN_WEIGHT = 12; +// Zero Heuristics (ZH) static const uint32_t ZH_TAKEN_WEIGHT = 20; static const uint32_t ZH_NONTAKEN_WEIGHT = 12; +// Floating-Point Heuristics (FPH) static const uint32_t FPH_TAKEN_WEIGHT = 20; static const uint32_t FPH_NONTAKEN_WEIGHT = 12; @@ -473,12 +540,11 @@ BranchProbability UntakenProb(PH_NONTAKEN_WEIGHT, PH_TAKEN_WEIGHT + PH_NONTAKEN_WEIGHT); - // p != 0 -> isProb = true - // p == 0 -> isProb = false - // p != q -> isProb = true - // p == q -> isProb = false; - bool isProb = CI->getPredicate() == ICmpInst::ICMP_NE; - if (!isProb) + auto Search = PointerTable.find(CI->getPredicate()); + if (Search == PointerTable.end()) + return false; + + if (!Search->second) std::swap(TakenProb, UntakenProb); setEdgeProbability( @@ -956,66 +1022,25 @@ Func == LibFunc_strncmp || Func == LibFunc_memcmp || Func == LibFunc_bcmp) { - // strcmp and similar functions return zero, negative, or positive, if the - // first string is equal, less, or greater than the second. We consider it - // likely that the strings are not equal, so a comparison with zero is - // probably false, but also a comparison with any other number is also - // probably false given that what exactly is returned for nonzero values is - // not specified. Any kind of comparison other than equality we know - // nothing about. - switch (CI->getPredicate()) { - case CmpInst::ICMP_EQ: - isProb = false; - break; - case CmpInst::ICMP_NE: - isProb = true; - break; - default: + auto Search = ICmpWithLibCallTable.find(CI->getPredicate()); + if (Search == ICmpWithLibCallTable.end()) return false; - } + isProb = Search->second; } else if (CV->isZero()) { - switch (CI->getPredicate()) { - case CmpInst::ICMP_EQ: - // X == 0 -> Unlikely - isProb = false; - break; - case CmpInst::ICMP_NE: - // X != 0 -> Likely - isProb = true; - break; - case CmpInst::ICMP_SLT: - // X < 0 -> Unlikely - isProb = false; - break; - case CmpInst::ICMP_SGT: - // X > 0 -> Likely - isProb = true; - break; - default: + auto Search = ICmpWithZeroTable.find(CI->getPredicate()); + if (Search == ICmpWithZeroTable.end()) return false; - } - } else if (CV->isOne() && CI->getPredicate() == CmpInst::ICMP_SLT) { - // InstCombine canonicalizes X <= 0 into X < 1. - // X <= 0 -> Unlikely - isProb = false; + isProb = Search->second; + } else if (CV->isOne()) { + auto Search = ICmpWithOneTable.find(CI->getPredicate()); + if (Search == ICmpWithOneTable.end()) + return false; + isProb = Search->second; } else if (CV->isMinusOne()) { - switch (CI->getPredicate()) { - case CmpInst::ICMP_EQ: - // X == -1 -> Unlikely - isProb = false; - break; - case CmpInst::ICMP_NE: - // X != -1 -> Likely - isProb = true; - break; - case CmpInst::ICMP_SGT: - // InstCombine canonicalizes X >= 0 into X > -1. - // X >= 0 -> Likely - isProb = true; - break; - default: + auto Search = ICmpWithMinusOneTable.find(CI->getPredicate()); + if (Search == ICmpWithMinusOneTable.end()) return false; - } + isProb = Search->second; } else { return false; } @@ -1049,18 +1074,13 @@ // f1 == f2 -> Unlikely // f1 != f2 -> Likely isProb = !FCmp->isTrueWhenEqual(); - } else if (FCmp->getPredicate() == FCmpInst::FCMP_ORD) { - // !isnan -> Likely - isProb = true; - TakenWeight = FPH_ORD_WEIGHT; - NontakenWeight = FPH_UNO_WEIGHT; - } else if (FCmp->getPredicate() == FCmpInst::FCMP_UNO) { - // isnan -> Unlikely - isProb = false; + } else { + auto Search = FCmpTable.find(FCmp->getPredicate()); + if (Search == FCmpTable.end()) + return false; + isProb = Search->second; TakenWeight = FPH_ORD_WEIGHT; NontakenWeight = FPH_UNO_WEIGHT; - } else { - return false; } BranchProbability TakenProb(TakenWeight, TakenWeight + NontakenWeight);