diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -120,6 +120,11 @@ bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI); + /// Returns true if \p F contains cold code with regard to a given cold + /// percentile cutoff value. + bool isFunctionColdInCallGraphNthPercentile(int PercentileCutoff, + const Function *F, + BlockFrequencyInfo &BFI); /// Returns true if count \p C is considered hot. bool isHotCount(uint64_t C); /// Returns true if count \p C is considered cold. @@ -127,6 +132,9 @@ /// Returns true if count \p C is considered hot with regard to a given /// hot percentile cutoff value. bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C); + /// Returns true if count \p C is considered cold with regard to a given + /// cold percentile cutoff value. + bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C); /// Returns true if BasicBlock \p BB is considered hot. bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered cold. @@ -135,6 +143,10 @@ /// hot percentile cutoff value. bool isHotBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered cold with regard to a given + /// cold percentile cutoff value. + bool isColdBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if CallSite \p CS is considered hot. bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// Returns true if Callsite \p CS is considered cold. diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -220,6 +220,30 @@ return false; } +bool ProfileSummaryInfo::isFunctionColdInCallGraphNthPercentile( + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { + if (!F || !computeSummary()) + return false; + if (auto FunctionCount = F->getEntryCount()) + if (!isColdCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) + return false; + + if (hasSampleProfile()) { + uint64_t TotalCallCount = 0; + for (const auto &BB : *F) + for (const auto &I : BB) + if (isa(I) || isa(I)) + if (auto CallCount = getProfileCount(&I, nullptr)) + TotalCallCount += CallCount.getValue(); + if (!isColdCountNthPercentile(PercentileCutoff, TotalCallCount)) + return false; + } + for (const auto &BB : *F) + if (!isColdBlockNthPercentile(PercentileCutoff, &BB, &BFI)) + return false; + return true; +} + /// Returns true if the function's entry is a cold. If it returns false, it /// either means it is not cold or it is unknown whether it is cold or not (for /// example, no profile data is available). @@ -304,6 +328,11 @@ return CountThreshold && C >= CountThreshold.getValue(); } +bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, uint64_t C) { + auto CountThreshold = computeThreshold(PercentileCutoff); + return CountThreshold && C <= CountThreshold.getValue(); +} + uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() { if (!HotCountThreshold) computeThresholds(); @@ -334,6 +363,13 @@ return Count && isHotCountNthPercentile(PercentileCutoff, *Count); } +bool ProfileSummaryInfo::isColdBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, + BlockFrequencyInfo *BFI) { + auto Count = BFI->getBlockProfileCount(BB); + return Count && isColdCountNthPercentile(PercentileCutoff, *Count); +} + bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI) { auto C = getProfileCount(CS.getInstruction(), BFI); diff --git a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp --- a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp +++ b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp @@ -65,6 +65,20 @@ " %y2 = phi i32 [0, %bb1], [1, %bb2] \n" " ret i32 %y2\n" "}\n" + "define i32 @l(i32 %x) {{\n" + "bb0:\n" + " %y1 = icmp eq i32 %x, 0 \n" + " br i1 %y1, label %bb1, label %bb2, !prof !23 \n" + "bb1:\n" + " %z1 = call i32 @g(i32 %x)\n" + " br label %bb3\n" + "bb2:\n" + " %z2 = call i32 @h(i32 %x)\n" + " br label %bb3\n" + "bb3:\n" + " %y2 = phi i32 [0, %bb1], [1, %bb2] \n" + " ret i32 %y2\n" + "}\n" "!20 = !{{!\"function_entry_count\", i64 400}\n" "!21 = !{{!\"function_entry_count\", i64 1}\n" "!22 = !{{!\"function_entry_count\", i64 100}\n" @@ -141,14 +155,26 @@ EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 100)); EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 2)); + EXPECT_FALSE(PSI.isColdCountNthPercentile(990000, 400)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(990000, 100)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(990000, 2)); + EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 400)); EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 100)); EXPECT_FALSE(PSI.isHotCountNthPercentile(999999, 2)); + EXPECT_FALSE(PSI.isColdCountNthPercentile(999999, 400)); + EXPECT_FALSE(PSI.isColdCountNthPercentile(999999, 100)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(999999, 2)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 400)); EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 100)); EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 2)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(10000, 400)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(10000, 100)); + EXPECT_TRUE(PSI.isColdCountNthPercentile(10000, 2)); + EXPECT_TRUE(PSI.isFunctionEntryHot(F)); EXPECT_FALSE(PSI.isFunctionEntryHot(G)); EXPECT_FALSE(PSI.isFunctionEntryHot(H)); @@ -177,16 +203,31 @@ EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB1, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB3, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, &BB0, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB1, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB2, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB3, &BFI)); + CallSite CS1(BB1->getFirstNonPHI()); auto *CI2 = BB2->getFirstNonPHI(); CallSite CS2(CI2); @@ -201,6 +242,31 @@ EXPECT_FALSE(PSI.isHotCallSite(CS2, &BFI)); } +TEST_F(ProfileSummaryInfoTest, InstrProfNoFuncEntryCount) { + auto M = makeLLVMModule("InstrProf"); + Function *F = M->getFunction("l"); + ProfileSummaryInfo PSI = buildPSI(M.get()); + EXPECT_TRUE(PSI.hasProfileSummary()); + EXPECT_TRUE(PSI.hasInstrumentationProfile()); + + BasicBlock &BB0 = F->getEntryBlock(); + BasicBlock *BB1 = BB0.getTerminator()->getSuccessor(0); + BasicBlock *BB2 = BB0.getTerminator()->getSuccessor(1); + BasicBlock *BB3 = BB1->getSingleSuccessor(); + + BlockFrequencyInfo BFI = buildBFI(*F); + + // Without the entry count, all should return false. + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB3, &BFI)); +} + TEST_F(ProfileSummaryInfoTest, SampleProf) { auto M = makeLLVMModule("SampleProfile"); Function *F = M->getFunction("f"); @@ -224,16 +290,31 @@ EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB1, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(999900, BB3, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI)); EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, &BB0, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB1, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB2, &BFI)); + EXPECT_TRUE(PSI.isColdBlockNthPercentile(10000, BB3, &BFI)); + CallSite CS1(BB1->getFirstNonPHI()); auto *CI2 = BB2->getFirstNonPHI(); // Manually attach branch weights metadata to the call instruction. @@ -250,6 +331,51 @@ // weights that exceed the hot count threshold. CI2->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights({400})); EXPECT_TRUE(PSI.isHotCallSite(CS2, &BFI)); + + { + Function *F = M->getFunction("l"); + BlockFrequencyInfo BFI = buildBFI(*F); + BasicBlock &BB0 = F->getEntryBlock(); + BasicBlock *BB1 = BB0.getTerminator()->getSuccessor(0); + BasicBlock *BB2 = BB0.getTerminator()->getSuccessor(1); + BasicBlock *BB3 = BB1->getSingleSuccessor(); + + // Without the entry count, all should return false. + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB3, &BFI)); + } +} + +TEST_F(ProfileSummaryInfoTest, SampleProfNoFuncEntryCount) { + auto M = makeLLVMModule("SampleProfile"); + Function *F = M->getFunction("l"); + ProfileSummaryInfo PSI = buildPSI(M.get()); + EXPECT_TRUE(PSI.hasProfileSummary()); + EXPECT_TRUE(PSI.hasSampleProfile()); + + BasicBlock &BB0 = F->getEntryBlock(); + BasicBlock *BB1 = BB0.getTerminator()->getSuccessor(0); + BasicBlock *BB2 = BB0.getTerminator()->getSuccessor(1); + BasicBlock *BB3 = BB1->getSingleSuccessor(); + + BlockFrequencyInfo BFI = buildBFI(*F); + + // Without the entry count, all should return false. + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_FALSE(PSI.isColdBlockNthPercentile(990000, BB3, &BFI)); } } // end anonymous namespace