Index: lib/Analysis/IndirectCallPromotionAnalysis.cpp =================================================================== --- lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -32,14 +32,6 @@ #define DEBUG_TYPE "pgo-icall-prom-analysis" -// The minimum call count for the direct-call target to be considered as the -// promotion candidate. -static cl::opt - ICPCountThreshold("icp-count-threshold", cl::Hidden, cl::ZeroOrMore, - cl::init(1000), - cl::desc("The minimum count to the direct call target " - "for the promotion")); - // The percent threshold for the direct-call target (this call site vs the // remaining call count) for it to be considered as the promotion target. static cl::opt ICPRemainingPercentThreshold( @@ -69,8 +61,7 @@ bool ICallPromotionAnalysis::isPromotionProfitable(uint64_t Count, uint64_t TotalCount, uint64_t RemainingCount) { - return Count >= ICPCountThreshold && - Count * 100 >= ICPRemainingPercentThreshold * RemainingCount && + return Count * 100 >= ICPRemainingPercentThreshold * RemainingCount && Count * 100 >= ICPTotalPercentThreshold * TotalCount; } Index: lib/Transforms/Instrumentation/IndirectCallPromotion.cpp =================================================================== --- lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -22,6 +22,7 @@ #include "llvm/Analysis/IndirectCallPromotionAnalysis.h" #include "llvm/Analysis/IndirectCallSiteVisitor.h" #include "llvm/Analysis/OptimizationDiagnosticInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DerivedTypes.h" @@ -121,6 +122,10 @@ *PassRegistry::getPassRegistry()); } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + } + StringRef getPassName() const override { return "PGOIndirectCallPromotion"; } private: @@ -194,7 +199,7 @@ bool SamplePGO, OptimizationRemarkEmitter &ORE) : F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {} - bool processFunction(); + bool processFunction(ProfileSummaryInfo *PSI); }; } // end anonymous namespace @@ -613,7 +618,7 @@ // Traverse all the indirect-call callsite and get the value profile // annotation to perform indirect-call promotion. -bool ICallPromotionFunc::processFunction() { +bool ICallPromotionFunc::processFunction(ProfileSummaryInfo *PSI) { bool Changed = false; ICallPromotionAnalysis ICallAnalysis; for (auto &I : findIndirectCallSites(F)) { @@ -621,7 +626,8 @@ uint64_t TotalCount; auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction( I, NumVals, TotalCount, NumCandidates); - if (!NumCandidates) + if (!NumCandidates || + (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount))) continue; auto PromotionCandidates = getPromotionCandidatesForCallSite( I, ICallProfDataRef, TotalCount, NumCandidates); @@ -643,7 +649,8 @@ } // A wrapper function that does the actual work. -static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO, +static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO, + bool SamplePGO, ModuleAnalysisManager *AM = nullptr) { if (DisableICP) return false; @@ -673,7 +680,7 @@ } ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO, *ORE); - bool FuncChanged = ICallPromotion.processFunction(); + bool FuncChanged = ICallPromotion.processFunction(PSI); if (ICPDUMPAFTER && FuncChanged) { DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs())); DEBUG(dbgs() << "\n"); @@ -688,14 +695,19 @@ } bool PGOIndirectCallPromotionLegacyPass::runOnModule(Module &M) { + ProfileSummaryInfo *PSI = + getAnalysis().getPSI(); + // Command-line option has the priority for InLTO. - return promoteIndirectCalls(M, InLTO | ICPLTOMode, + return promoteIndirectCalls(M, PSI, InLTO | ICPLTOMode, SamplePGO | ICPSamplePGOMode); } PreservedAnalyses PGOIndirectCallPromotion::run(Module &M, ModuleAnalysisManager &AM) { - if (!promoteIndirectCalls(M, InLTO | ICPLTOMode, SamplePGO | ICPSamplePGOMode, + ProfileSummaryInfo *PSI = &AM.getResult(M); + + if (!promoteIndirectCalls(M, PSI, InLTO | ICPLTOMode, SamplePGO | ICPSamplePGOMode, &AM)) return PreservedAnalyses::all(); Index: test/Other/new-pm-lto-defaults.ll =================================================================== --- test/Other/new-pm-lto-defaults.ll +++ test/Other/new-pm-lto-defaults.ll @@ -30,6 +30,7 @@ ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O2-NEXT: PGOIndirectCallPromotion +; CHECK-O2-NEXT: Running analysis: ProfileSummaryAnalysis ; CHECK-O2-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Function ; CHECK-O2-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis ; CHECK-O2-NEXT: Running pass: IPSCCPPass Index: test/Other/new-pm-thinlto-defaults.ll =================================================================== --- test/Other/new-pm-thinlto-defaults.ll +++ test/Other/new-pm-thinlto-defaults.ll @@ -54,6 +54,7 @@ ; CHECK-DIS-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}> ; CHECK-DIS-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-POSTLINK-O-NEXT: Running pass: PGOIndirectCallPromotion +; CHECK-POSTLINK-O-NEXT: Running analysis: ProfileSummaryAnalysis ; CHECK-POSTLINK-O-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Function ; CHECK-POSTLINK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis ; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}> @@ -86,7 +87,7 @@ ; CHECK-O-NEXT: Running analysis: GlobalsAA ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis -; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis +; CHECK-PRELINK-O-NEXT: Running analysis: ProfileSummaryAnalysis ; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}> ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis Index: test/Transforms/PGOProfile/icp_invoke.ll =================================================================== --- test/Transforms/PGOProfile/icp_invoke.ll +++ test/Transforms/PGOProfile/icp_invoke.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -icp-lto -pgo-icall-prom -S -icp-count-threshold=0 | FileCheck %s --check-prefix=ICP -; RUN: opt < %s -icp-lto -passes=pgo-icall-prom -S -icp-count-threshold=0 | FileCheck %s --check-prefix=ICP +; RUN: opt < %s -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICP +; RUN: opt < %s -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICP target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: test/Transforms/PGOProfile/indirect_call_promotion.ll =================================================================== --- test/Transforms/PGOProfile/indirect_call_promotion.ll +++ test/Transforms/PGOProfile/indirect_call_promotion.ll @@ -1,10 +1,9 @@ -; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM-SAMPLEPGO -; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK -; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK -; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=20 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS2-REMARK +; RUN: opt < %s -pgo-icall-prom -S -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt < %s -passes=pgo-icall-prom -S -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=20 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS2-REMARK ; PASS-REMARK: remark: :0:0: Promote indirect call to func4 with count 1030 out of 1600 ; PASS-REMARK: remark: :0:0: Promote indirect call to func2 with count 410 out of 570 Index: test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll =================================================================== --- test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll +++ test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll @@ -7,8 +7,8 @@ ; IMPORTS-DAG: Import a ; IMPORTS-DAG: Import c -; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -icp-count-threshold=1 | FileCheck %s --check-prefix=ICALL-PROM -; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=1 2>&1 | FileCheck %s --check-prefix=PASS-REMARK +; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK ; PASS-REMARK: Promote indirect call to a with count 1 out of 1 ; PASS-REMARK: Promote indirect call to c.llvm.0 with count 1 out of 1 Index: test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll =================================================================== --- test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll +++ test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll @@ -7,7 +7,7 @@ ; by ICP. Note that the GUID in the profile is from the oroginal name. ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS ; IMPORTS: Import _ZL3foov.llvm.0 -; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -icp-count-threshold=1 | FileCheck %s --check-prefix=ICALL-PROM +; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"