[PGO][PGSO] Prep for enabling non-cold code size opts under non-partial-profile sample PGO.

[hjyamauchi]

Summary

• Distinguish between partial-profile and non-partial-profile sample PGO.
• Add a flag for partial-profile sample PGO.
• Tune the sample PGO cutoff.
• No default behavior change (yet).

[davidxl]

llvm/include/llvm/Transforms/Utils/SizeOpts.h

43

make the function name consistent with options: PGSOCodeCodeOnly(..)?

[hjyamauchi]

Address comment.

llvm/include/llvm/Transforms/Utils/SizeOpts.h

43

Done. Renamed to isPGSOColdCodeOnly (PGSOColdCodeOnly would conflict with the flag name.)

[davidxl]

lgtm

llvm/include/llvm/Analysis/ProfileSummaryInfo.h

public:
   bool hasProfileSummary() { return computeSummary(); }
 
   /// Returns true if module \c M has sample profile.
   bool hasSampleProfile() {
     return hasProfileSummary() &&
            Summary->getKind() == ProfileSummary::PSK_Sample;
   }
 
+  /// Returns true if module \c M has partial-profile sample profile.
+  bool hasPartialSampleProfile() {
+    return hasProfileSummary() &&
+           Summary->getKind() == ProfileSummary::PSK_Sample &&
+           Summary->isPartialProfile();
+  }
+
   /// Returns true if module \c M has instrumentation profile.
   bool hasInstrumentationProfile() {
     return hasProfileSummary() &&
            Summary->getKind() == ProfileSummary::PSK_Instr;
   }
 
   /// Returns true if module \c M has context sensitive instrumentation profile.
   bool hasCSInstrumentationProfile() {

llvm/include/llvm/Transforms/Utils/SizeOpts.h

 #include "llvm/Support/CommandLine.h"
 
 extern llvm::cl::opt<bool> EnablePGSO;
 extern llvm::cl::opt<bool> PGSOLargeWorkingSetSizeOnly;
 extern llvm::cl::opt<bool> PGSOIRPassOrTestOnly;
 extern llvm::cl::opt<bool> PGSOColdCodeOnly;
 extern llvm::cl::opt<bool> PGSOColdCodeOnlyForInstrPGO;
 extern llvm::cl::opt<bool> PGSOColdCodeOnlyForSamplePGO;
+extern llvm::cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO;
 extern llvm::cl::opt<bool> ForcePGSO;
 extern llvm::cl::opt<int> PgsoCutoffInstrProf;
 extern llvm::cl::opt<int> PgsoCutoffSampleProf;
 
 namespace llvm {
 
 class BasicBlock;
 class BlockFrequencyInfo;
 class Function;
 
 enum class PGSOQueryType {
   IRPass, // A query call from an IR-level transform pass.
   Test,   // A query call from a unit test.
   Other,  // Others.
 };
 
+static inline bool isPGSOColdCodeOnly(ProfileSummaryInfo *PSI) {
+  return PGSOColdCodeOnly ||
+         (PSI->hasInstrumentationProfile() && PGSOColdCodeOnlyForInstrPGO) ||
+         (PSI->hasSampleProfile() &&
+          ((!PSI->hasPartialSampleProfile() && PGSOColdCodeOnlyForSamplePGO) ||
+           (PSI->hasPartialSampleProfile() &&
+            PGSOColdCodeOnlyForPartialSamplePGO))) ||
+         (PGSOLargeWorkingSetSizeOnly && !PSI->hasLargeWorkingSetSize());
+}
+
 template<typename AdapterT, typename FuncT, typename BFIT>
 bool shouldFuncOptimizeForSizeImpl(const FuncT *F, ProfileSummaryInfo *PSI,
                                    BFIT *BFI, PGSOQueryType QueryType) {
   assert(F);
   if (!PSI || !BFI || !PSI->hasProfileSummary())
     return false;
   if (ForcePGSO)
     return true;
   if (!EnablePGSO)
     return false;
   // Temporarily enable size optimizations only for the IR pass or test query
   // sites for gradual commit/rollout. This is to be removed later.
   if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass ||
                                 QueryType == PGSOQueryType::Test))
     return false;
-  if (PGSOColdCodeOnly ||
-      (PSI->hasInstrumentationProfile() && PGSOColdCodeOnlyForInstrPGO) ||
-      (PSI->hasSampleProfile() && PGSOColdCodeOnlyForSamplePGO) ||
-      (PGSOLargeWorkingSetSizeOnly && !PSI->hasLargeWorkingSetSize())) {
-    // Even if the working set size isn't large, size-optimize cold code.
+  if (isPGSOColdCodeOnly(PSI))
     return AdapterT::isFunctionColdInCallGraph(F, PSI, *BFI);
-  }
   if (PSI->hasSampleProfile())
     // The "isCold" check seems to work better for Sample PGO as it could have
     // many profile-unannotated functions.
     return AdapterT::isFunctionColdInCallGraphNthPercentile(
         PgsoCutoffSampleProf, F, PSI, *BFI);
   return !AdapterT::isFunctionHotInCallGraphNthPercentile(PgsoCutoffInstrProf,
                                                           F, PSI, *BFI);
 }
 
 template<typename AdapterT, typename BlockTOrBlockFreq, typename BFIT>
 bool shouldOptimizeForSizeImpl(BlockTOrBlockFreq BBOrBlockFreq, ProfileSummaryInfo *PSI,
                                BFIT *BFI, PGSOQueryType QueryType) {
   if (!PSI || !BFI || !PSI->hasProfileSummary())
     return false;
   if (ForcePGSO)
     return true;
   if (!EnablePGSO)
     return false;
   // Temporarily enable size optimizations only for the IR pass or test query
   // sites for gradual commit/rollout. This is to be removed later.
   if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass ||
                                 QueryType == PGSOQueryType::Test))
     return false;
-  if (PGSOColdCodeOnly ||
-      (PSI->hasInstrumentationProfile() && PGSOColdCodeOnlyForInstrPGO) ||
-      (PSI->hasSampleProfile() && PGSOColdCodeOnlyForSamplePGO) ||
-      (PGSOLargeWorkingSetSizeOnly && !PSI->hasLargeWorkingSetSize())) {
-    // Even if the working set size isn't large, size-optimize cold code.
+  if (isPGSOColdCodeOnly(PSI))
     return AdapterT::isColdBlock(BBOrBlockFreq, PSI, BFI);
-  }
   if (PSI->hasSampleProfile())
     // The "isCold" check seems to work better for Sample PGO as it could have
     // many profile-unannotated functions.
     return AdapterT::isColdBlockNthPercentile(PgsoCutoffSampleProf,
                                               BBOrBlockFreq, PSI, BFI);
   return !AdapterT::isHotBlockNthPercentile(PgsoCutoffInstrProf, BBOrBlockFreq,
                                             PSI, BFI);
 }

llvm/lib/Transforms/Utils/SizeOpts.cpp

cl::opt<bool> PGSOColdCodeOnlyForInstrPGO(
     cl::desc("Apply the profile guided size optimizations only "
              "to cold code under instrumentation PGO."));
 
 cl::opt<bool> PGSOColdCodeOnlyForSamplePGO(
     "pgso-cold-code-only-for-sample-pgo", cl::Hidden, cl::init(true),
     cl::desc("Apply the profile guided size optimizations only "
              "to cold code under sample PGO."));
 
+cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO(
+    "pgso-cold-code-only-for-partial-sample-pgo", cl::Hidden, cl::init(true),
+    cl::desc("Apply the profile guided size optimizations only "
+             "to cold code under partial-profile sample PGO."));
+
 cl::opt<bool> PGSOIRPassOrTestOnly(
     "pgso-ir-pass-or-test-only", cl::Hidden, cl::init(false),
     cl::desc("Apply the profile guided size optimizations only"
              "to the IR passes or tests."));
 
 cl::opt<bool> ForcePGSO(
     "force-pgso", cl::Hidden, cl::init(false),
     cl::desc("Force the (profiled-guided) size optimizations. "));
 
 cl::opt<int> PgsoCutoffInstrProf(
     "pgso-cutoff-instr-prof", cl::Hidden, cl::init(950000), cl::ZeroOrMore,
     cl::desc("The profile guided size optimization profile summary cutoff "
              "for instrumentation profile."));
 
 cl::opt<int> PgsoCutoffSampleProf(
-    "pgso-cutoff-sample-prof", cl::Hidden, cl::init(800000), cl::ZeroOrMore,
+    "pgso-cutoff-sample-prof", cl::Hidden, cl::init(990000), cl::ZeroOrMore,
     cl::desc("The profile guided size optimization profile summary cutoff "
              "for sample profile."));
 
 namespace {
 struct BasicBlockBFIAdapter {
   static bool isFunctionColdInCallGraph(const Function *F,
                                         ProfileSummaryInfo *PSI,
                                         BlockFrequencyInfo &BFI) {