diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -58,9 +58,13 @@ /// basic blocks and is enabled with -fbasic-block-sections. MachineFunctionPass *createBasicBlockSectionsPass(); - /// createMachineFunctionSplitterPass - This pass splits machine functions - /// using profile information. - MachineFunctionPass *createMachineFunctionSplitterPass(); + /// createMachineFunctionSplitterPass - This pass splits machine + /// functions using profile information. Set + /// "HasAccurateSampleProfile" to true when using FSAFDO or CSSPGO, + /// Set this to false when IRPGO (traditional instrumented FDO) + /// profile is used. + MachineFunctionPass * + createMachineFunctionSplitterPass(bool HasAccurateSampleProfile = false); /// MachineFunctionPrinter pass - This pass prints out the machine function to /// the given stream as a debugging tool. diff --git a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp --- a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp +++ b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp @@ -24,7 +24,10 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/EHUtils.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/BasicBlockSectionUtils.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -69,7 +72,9 @@ class MachineFunctionSplitter : public MachineFunctionPass { public: static char ID; - MachineFunctionSplitter() : MachineFunctionPass(ID) { + MachineFunctionSplitter(bool HasAccurateSampleProfile) + : MachineFunctionPass(ID), + HasAccurateSampleProfile(HasAccurateSampleProfile) { initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); } @@ -80,6 +85,10 @@ void getAnalysisUsage(AnalysisUsage &AU) const override; bool runOnMachineFunction(MachineFunction &F) override; + +private: + // Whether this pass is using FSAFDO profile or not. + bool HasAccurateSampleProfile = false; }; } // end anonymous namespace @@ -96,8 +105,13 @@ static bool isColdBlock(const MachineBasicBlock &MBB, const MachineBlockFrequencyInfo *MBFI, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, + bool HasAccurateSampleProfile) { std::optional Count = MBFI->getBlockProfileCount(&MBB); + if (HasAccurateSampleProfile) { + return Count.has_value() && Count.value() < ColdCountThreshold; + } + if (!Count) return true; @@ -140,9 +154,18 @@ MachineBlockFrequencyInfo *MBFI = nullptr; ProfileSummaryInfo *PSI = nullptr; + bool CandidateForSplit = true; if (UseProfileData) { MBFI = &getAnalysis(); PSI = &getAnalysis().getPSI(); + if (HasAccurateSampleProfile) { + const LoopInfo &LI = getAnalysis().getLoopInfo(); + const BranchProbabilityInfo *BPI = + new BranchProbabilityInfo(MF.getFunction(), LI); + BlockFrequencyInfo BFI(MF.getFunction(), *BPI, LI); + CandidateForSplit = + (PSI && PSI->isFunctionHotInCallGraph(&MF.getFunction(), BFI)); + } } SmallVector LandingPads; @@ -152,7 +175,9 @@ if (MBB.isEHPad()) LandingPads.push_back(&MBB); - else if (UseProfileData && isColdBlock(MBB, MBFI, PSI) && !SplitAllEHCode) + else if (UseProfileData && CandidateForSplit && + isColdBlock(MBB, MBFI, PSI, HasAccurateSampleProfile) && + !SplitAllEHCode) MBB.setSectionID(MBBSectionID::ColdSectionID); } @@ -163,7 +188,8 @@ else { bool HasHotLandingPads = false; for (const MachineBasicBlock *LP : LandingPads) { - if (!isColdBlock(*LP, MBFI, PSI)) + if (!(CandidateForSplit && + isColdBlock(*LP, MBFI, PSI, HasAccurateSampleProfile))) HasHotLandingPads = true; } if (!HasHotLandingPads) { @@ -180,6 +206,7 @@ } void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addRequired(); @@ -190,6 +217,10 @@ "Split machine functions using profile information", false, false) -MachineFunctionPass *llvm::createMachineFunctionSplitterPass() { - return new MachineFunctionSplitter(); +// Set "HasAccurateSampleProfile" to true when we FSAFDO or CSSPGO +// profiles are used. Set this to false when IRPGO (traditional +// instrumented FDO) profile is used. +MachineFunctionPass * +llvm::createMachineFunctionSplitterPass(bool HasAccurateSampleProfile) { + return new MachineFunctionSplitter(HasAccurateSampleProfile); }