Index: include/llvm/Analysis/IndirectCallPromotionAnalysis.h =================================================================== --- include/llvm/Analysis/IndirectCallPromotionAnalysis.h +++ include/llvm/Analysis/IndirectCallPromotionAnalysis.h @@ -27,10 +27,12 @@ // Allocate space to read the profile annotation. std::unique_ptr ValueDataArray; - // Count is the call count for the direct-call target and - // TotalCount is the call count for the indirect-call callsite. + // Count is the call count for the direct-call target. + // TotalCount is the total call count for the indirect-call callsite. + // RemainingCount is the TotalCount minus promoted-direct-call count. // Return true we should promote this indirect-call target. - bool isPromotionProfitable(uint64_t Count, uint64_t TotalCount); + bool isPromotionProfitable(uint64_t Count, uint64_t TotalCount, + uint64_t RemainingCount); // Returns the number of profitable candidates to promote for the // current ValueDataArray and the given \p Inst. Index: lib/Analysis/IndirectCallPromotionAnalysis.cpp =================================================================== --- lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -41,11 +41,19 @@ "for the promotion")); // The percent threshold for the direct-call target (this call site vs the +// remaining call count) for it to be considered as the promotion target. +static cl::opt ICPRemainingPercentThreshold( + "icp-remaining-percent-threshold", cl::init(30), cl::Hidden, cl::ZeroOrMore, + cl::desc("The percentage threshold against remaining unpromoted indirect " + "call count for the promotion")); + +// The percent threshold for the direct-call target (this call site vs the // total call count) for it to be considered as the promotion target. static cl::opt - ICPPercentThreshold("icp-percent-threshold", cl::init(30), cl::Hidden, - cl::ZeroOrMore, - cl::desc("The percentage threshold for the promotion")); + ICPTotalPercentThreshold("icp-total-percent-threshold", cl::init(5), + cl::Hidden, cl::ZeroOrMore, + cl::desc("The percentage threshold against total " + "count for the promotion")); // Set the maximum number of targets to promote for a single indirect-call // callsite. @@ -59,12 +67,11 @@ } bool ICallPromotionAnalysis::isPromotionProfitable(uint64_t Count, - uint64_t TotalCount) { - if (Count < ICPCountThreshold) - return false; - - unsigned Percentage = (Count * 100) / TotalCount; - return (Percentage >= ICPPercentThreshold); + uint64_t TotalCount, + uint64_t RemainingCount) { + return Count >= ICPCountThreshold && + Count * 100 >= ICPRemainingPercentThreshold * RemainingCount && + Count * 100 >= ICPTotalPercentThreshold * TotalCount; } // Indirect-call promotion heuristic. The direct targets are sorted based on @@ -78,17 +85,18 @@ << "\n"); uint32_t I = 0; + uint64_t RemainingCount = TotalCount; for (; I < MaxNumPromotions && I < NumVals; I++) { uint64_t Count = ValueDataRef[I].Count; - assert(Count <= TotalCount); + assert(Count <= RemainingCount); DEBUG(dbgs() << " Candidate " << I << " Count=" << Count << " Target_func: " << ValueDataRef[I].Value << "\n"); - if (!isPromotionProfitable(Count, TotalCount)) { + if (!isPromotionProfitable(Count, TotalCount, RemainingCount)) { DEBUG(dbgs() << " Not promote: Cold target.\n"); return I; } - TotalCount -= Count; + RemainingCount -= Count; } return I; } Index: test/Transforms/PGOProfile/indirect_call_promotion.ll =================================================================== --- test/Transforms/PGOProfile/indirect_call_promotion.ll +++ test/Transforms/PGOProfile/indirect_call_promotion.ll @@ -2,13 +2,20 @@ ; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM ; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM-SAMPLEPGO ; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK -; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=20 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS2-REMARK + ; PASS-REMARK: remark: :0:0: Promote indirect call to func4 with count 1030 out of 1600 ; PASS-REMARK: remark: :0:0: Promote indirect call to func2 with count 410 out of 570 ; PASS-REMARK: remark: :0:0: Promote indirect call to func3 with count 150 out of 160 ; PASS-REMARK: remark: :0:0: Promote indirect call to func1 with count 10 out of 10 +; PASS2-REMARK: remark: :0:0: Promote indirect call to func4 with count 1030 out of 1600 +; PASS2-REMARK: remark: :0:0: Promote indirect call to func2 with count 410 out of 570 +; PASS2-REMARK-NOT: remark: :0:0: Promote indirect call to func3 +; PASS2-REMARK-NOT: remark: :0:0: Promote indirect call to func1 + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"