diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h --- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h +++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h @@ -76,6 +76,7 @@ } // end namespace afdo_detail extern cl::opt SampleProfileUseProfi; +extern cl::opt SampleProfileInferEntryCount; template class SampleProfileLoaderBaseImpl { public: @@ -920,7 +921,9 @@ // Samples->getHeadSamples() + 1 to avoid functions with zero count. if (SampleProfileUseProfi) { const BasicBlockT *EntryBB = getEntryBB(&F); - if (BlockWeights[EntryBB] > 0) { + ErrorOr EntryWeight = getBlockWeight(EntryBB); + if (BlockWeights[EntryBB] > 0 && + (SampleProfileInferEntryCount || !EntryWeight)) { getFunction(F).setEntryCount( ProfileCount(BlockWeights[EntryBB], Function::PCT_Real), &InlinedGUIDs); diff --git a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp --- a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp +++ b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp @@ -15,6 +15,7 @@ #include "llvm/Transforms/Utils/SampleProfileInference.h" #include "llvm/ADT/BitVector.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include #include @@ -24,6 +25,29 @@ namespace { +static cl::opt SampleProfileProfiCostInc( + "sample-profile-profi-cost-inc", cl::init(10), cl::Hidden, cl::ZeroOrMore, + cl::desc("A cost of increasing a block's count by one.")); + +static cl::opt SampleProfileProfiCostDec( + "sample-profile-profi-cost-dec", cl::init(20), cl::Hidden, cl::ZeroOrMore, + cl::desc("A cost of decreasing a block's count by one.")); + +static cl::opt SampleProfileProfiCostIncZero( + "sample-profile-profi-cost-inc-zero", cl::init(11), cl::Hidden, + cl::ZeroOrMore, + cl::desc("A cost of increasing a count of zero-weight block by one.")); + +static cl::opt SampleProfileProfiCostIncEntry( + "sample-profile-profi-cost-inc-entry", cl::init(40), cl::Hidden, + cl::ZeroOrMore, + cl::desc("A cost of increasing the entry block's count by one.")); + +static cl::opt SampleProfileProfiCostDecEntry( + "sample-profile-profi-cost-dec-entry", cl::init(10), cl::Hidden, + cl::ZeroOrMore, + cl::desc("A cost of decreasing the entry block's count by one.")); + /// A value indicating an infinite flow/capacity/weight of a block/edge. /// Not using numeric_limits::max(), as the values can be summed up /// during the execution. @@ -134,16 +158,6 @@ return Flow; } - /// A cost of increasing a block's count by one. - static constexpr int64_t AuxCostInc = 10; - /// A cost of decreasing a block's count by one. - static constexpr int64_t AuxCostDec = 20; - /// A cost of increasing a count of zero-weight block by one. - static constexpr int64_t AuxCostIncZero = 11; - /// A cost of increasing the entry block's count by one. - static constexpr int64_t AuxCostIncEntry = 40; - /// A cost of decreasing the entry block's count by one. - static constexpr int64_t AuxCostDecEntry = 10; /// A cost of taking an unlikely jump. static constexpr int64_t AuxCostUnlikely = ((int64_t)1) << 30; @@ -778,8 +792,8 @@ // We assume that decreasing block counts is more expensive than increasing, // and thus, setting separate costs here. In the future we may want to tune // the relative costs so as to maximize the quality of generated profiles. - int64_t AuxCostInc = MinCostMaxFlow::AuxCostInc; - int64_t AuxCostDec = MinCostMaxFlow::AuxCostDec; + int64_t AuxCostInc = SampleProfileProfiCostInc; + int64_t AuxCostDec = SampleProfileProfiCostDec; if (Block.UnknownWeight) { // Do not penalize changing weights of blocks w/o known profile count AuxCostInc = 0; @@ -788,12 +802,12 @@ // Increasing the count for "cold" blocks with zero initial count is more // expensive than for "hot" ones if (Block.Weight == 0) { - AuxCostInc = MinCostMaxFlow::AuxCostIncZero; + AuxCostInc = SampleProfileProfiCostIncZero; } // Modifying the count of the entry block is expensive if (Block.isEntry()) { - AuxCostInc = MinCostMaxFlow::AuxCostIncEntry; - AuxCostDec = MinCostMaxFlow::AuxCostDecEntry; + AuxCostInc = SampleProfileProfiCostIncEntry; + AuxCostDec = SampleProfileProfiCostDecEntry; } } // For blocks with self-edges, do not penalize a reduction of the count, diff --git a/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp b/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp --- a/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp +++ b/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp @@ -38,6 +38,10 @@ "sample-profile-use-profi", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Use profi to infer block and edge counts.")); +cl::opt SampleProfileInferEntryCount( + "sample-profile-infer-entry-count", cl::init(false), cl::Hidden, + cl::ZeroOrMore, cl::desc("Use profi to infer function entry count.")); + namespace sampleprofutil { /// Return true if the given callsite is hot wrt to hot cutoff threshold. diff --git a/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll b/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll --- a/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll +++ b/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-file=%S/Inputs/profile-inference-islands.prof -S -o %t +; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-infer-entry-count -sample-profile-file=%S/Inputs/profile-inference-islands.prof -S -o %t ; RUN: FileCheck %s < %t -check-prefix=CHECK-ENTRY-COUNT ; RUN: opt < %t -passes='print' -disable-output 2>&1 | FileCheck %s