[ProfileSummary] Add the PartialProfileRatio field in ProfileSummary metadata.

[hjyamauchi]

Summary

PartialProfileRatio approximately represents the ratio of the number of profile
counters of the program being built to the number of profile counters in the
partial sample profile. It is used to scale the working set size under the
partial sample profile to reflect the size of the program being built and to
improve the working set size heuristics.

This is a split from D79831.

[wmi]

llvm/include/llvm/IR/ProfileSummary.h

95–96

Can we add an assertion that Partial is true for getter and setter of PartialProfileRatio?

[davidxl]

llvm/lib/IR/ProfileSummary.cpp

77

Document the new parameter in the comment.

195

can you commit the style changes as NFC first?

[hjyamauchi]

Address comments.

llvm/include/llvm/IR/ProfileSummary.h

95–96

Done for the setter, but it does not work for the getter as getMD will query the field value along with all the fields and just even to write a false value for it, even when it's not partial. Just querying seems okay.

llvm/lib/IR/ProfileSummary.cpp

195

Sure, will do.

[hjyamauchi]

llvm/lib/IR/ProfileSummary.cpp

195

Split it into D80230.

[hjyamauchi]

Rebase past D80230.

[hjyamauchi]

PTAL.

[wmi]

llvm/include/llvm/IR/ProfileSummary.h

97

Ok, better have some message in the assert.

[wmi]

LGTM.

[hjyamauchi]

Address comments.

llvm/include/llvm/IR/ProfileSummary.h

97

Done.

llvm/include/llvm/IR/ProfileSummary.h

private:
   uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount;
   uint32_t NumCounts, NumFunctions;
   /// If 'Partial' is false, it means the profile being used to optimize
   /// a target is collected from the same target.
   /// If 'Partial' is true, it means the profile is for common/shared
   /// code. The common profile is usually merged from profiles collected
   /// from running other targets.
   bool Partial = false;
+  /// This approximately represents the ratio of the number of profile counters
+  /// of the program being built to the number of profile counters in the
+  /// partial sample profile. When 'Partial' is false, it is undefined.
+  double PartialProfileRatio = 0;
   /// Return detailed summary as metadata.
   Metadata *getDetailedSummaryMD(LLVMContext &Context);
 
 public:
   static const int Scale = 1000000;
 
   ProfileSummary(Kind K, SummaryEntryVector DetailedSummary,
                  uint64_t TotalCount, uint64_t MaxCount,
                  uint64_t MaxInternalCount, uint64_t MaxFunctionCount,
                  uint32_t NumCounts, uint32_t NumFunctions,
-                 bool Partial = false)
+                 bool Partial = false, double PartialProfileRatio = 0)
       : PSK(K), DetailedSummary(std::move(DetailedSummary)),
         TotalCount(TotalCount), MaxCount(MaxCount),
         MaxInternalCount(MaxInternalCount), MaxFunctionCount(MaxFunctionCount),
-        NumCounts(NumCounts), NumFunctions(NumFunctions), Partial(Partial) {}
+        NumCounts(NumCounts), NumFunctions(NumFunctions), Partial(Partial),
+        PartialProfileRatio(PartialProfileRatio) {}
 
   Kind getKind() const { return PSK; }
   /// Return summary information as metadata.
-  Metadata *getMD(LLVMContext &Context, bool AddPartialField = true);
+  Metadata *getMD(LLVMContext &Context, bool AddPartialField = true,
+                  bool AddPartialProfileRatioField = true);
   /// Construct profile summary from metdata.
   static ProfileSummary *getFromMD(Metadata *MD);
   SummaryEntryVector &getDetailedSummary() { return DetailedSummary; }
   uint32_t getNumFunctions() { return NumFunctions; }
   uint64_t getMaxFunctionCount() { return MaxFunctionCount; }
   uint32_t getNumCounts() { return NumCounts; }
   uint64_t getTotalCount() { return TotalCount; }
   uint64_t getMaxCount() { return MaxCount; }
   uint64_t getMaxInternalCount() { return MaxInternalCount; }
   void setPartialProfile(bool PP) { Partial = PP; }
   bool isPartialProfile() { return Partial; }
+  double getPartialProfileRatio() { return PartialProfileRatio; }
+  void setPartialProfileRatio(double R) { PartialProfileRatio = R; }
   void printSummary(raw_ostream &OS);
   void printDetailedSummary(raw_ostream &OS);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_IR_PROFILESUMMARY_H

llvm/lib/IR/ProfileSummary.cpp

 static Metadata *getKeyValMD(LLVMContext &Context, const char *Key,
                              uint64_t Val) {
   Type *Int64Ty = Type::getInt64Ty(Context);
   Metadata *Ops[2] = {MDString::get(Context, Key),
                       ConstantAsMetadata::get(ConstantInt::get(Int64Ty, Val))};
   return MDTuple::get(Context, Ops);
 }
 
+static Metadata *getKeyFPValMD(LLVMContext &Context, const char *Key,
+                               double Val) {
+  Type *DoubleTy = Type::getDoubleTy(Context);
+  Metadata *Ops[2] = {MDString::get(Context, Key),
+                      ConstantAsMetadata::get(ConstantFP::get(DoubleTy, Val))};
+  return MDTuple::get(Context, Ops);
+}
+
 // Return an MDTuple with two elements. The first element is a string Key and
 // the second is a string Value.
 static Metadata *getKeyValMD(LLVMContext &Context, const char *Key,
                              const char *Val) {
   Metadata *Ops[2] = {MDString::get(Context, Key), MDString::get(Context, Val)};
   return MDTuple::get(Context, Ops);
 }
 

 }
 
 // This returns an MDTuple representing this ProfileSummary object. The first
 // entry of this tuple is another MDTuple of two elements: a string
 // "ProfileFormat" and a string representing the format ("InstrProf" or
 // "SampleProfile"). The rest of the elements of the outer MDTuple are specific
 // to the kind of profile summary as returned by getFormatSpecificMD.
 // IsPartialProfile is an optional field and \p AddPartialField will decide
 // whether to add a field for it.
-Metadata *ProfileSummary::getMD(LLVMContext &Context, bool AddPartialField) {
+Metadata *ProfileSummary::getMD(LLVMContext &Context, bool AddPartialField,
+                                bool AddPartialProfileRatioField) {
   const char *KindStr[3] = {"InstrProf", "CSInstrProf", "SampleProfile"};
   SmallVector<Metadata *, 16> Components;
   Components.push_back(getKeyValMD(Context, "ProfileFormat", KindStr[PSK]));
   Components.push_back(getKeyValMD(Context, "TotalCount", getTotalCount()));
   Components.push_back(getKeyValMD(Context, "MaxCount", getMaxCount()));
   Components.push_back(
       getKeyValMD(Context, "MaxInternalCount", getMaxInternalCount()));
   Components.push_back(
       getKeyValMD(Context, "MaxFunctionCount", getMaxFunctionCount()));
   Components.push_back(getKeyValMD(Context, "NumCounts", getNumCounts()));
   Components.push_back(getKeyValMD(Context, "NumFunctions", getNumFunctions()));
   if (AddPartialField)
     Components.push_back(
         getKeyValMD(Context, "IsPartialProfile", isPartialProfile()));
+  if (AddPartialProfileRatioField)
+    Components.push_back(getKeyFPValMD(Context, "PartialProfileRatio",
+                                       getPartialProfileRatio()));
   Components.push_back(getDetailedSummaryMD(Context));
   return MDTuple::get(Context, Components);
 }
 
-// Parse an MDTuple representing (Key, Val) pair.
-static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val) {
+// Get the value metadata for the input MD/Key.
+static ConstantAsMetadata *getValMD(MDTuple *MD, const char *Key) {
   if (!MD)
-    return false;
+    return nullptr;
   if (MD->getNumOperands() != 2)
-    return false;
+    return nullptr;
   MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0));
   ConstantAsMetadata *ValMD = dyn_cast<ConstantAsMetadata>(MD->getOperand(1));
   if (!KeyMD || !ValMD)
-    return false;
+    return nullptr;
   if (!KeyMD->getString().equals(Key))
-    return false;
+    return nullptr;
+  return ValMD;
+}
+
+// Parse an MDTuple representing (Key, Val) pair.
+static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val) {
+  if (auto *ValMD = getValMD(MD, Key)) {
     Val = cast<ConstantInt>(ValMD->getValue())->getZExtValue();
     return true;
   }
+  return false;
+}
+
+static bool getVal(MDTuple *MD, const char *Key, double &Val) {
+  if (auto *ValMD = getValMD(MD, Key)) {
+    Val = cast<ConstantFP>(ValMD->getValue())->getValueAPF().convertToDouble();
+    return true;
+  }
+  return false;
+}
 
 // Check if an MDTuple represents a (Key, Val) pair.
 static bool isKeyValuePair(MDTuple *MD, const char *Key, const char *Val) {
   if (!MD || MD->getNumOperands() != 2)
     return false;
   MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0));
   MDString *ValMD = dyn_cast<MDString>(MD->getOperand(1));
   if (!KeyMD || !ValMD)
if (!Op0 || !Op1 || !Op2)
       return false;
     Summary.emplace_back(cast<ConstantInt>(Op0->getValue())->getZExtValue(),
                          cast<ConstantInt>(Op1->getValue())->getZExtValue(),
                          cast<ConstantInt>(Op2->getValue())->getZExtValue());
   }
   return true;
 }
 
+template <typename ValueType>
+static bool getOptionalVal(MDTuple *Tuple, unsigned &Idx, const char *Key,
+                           ValueType &Value) {
+  if (getVal(dyn_cast<MDTuple>(Tuple->getOperand(Idx)), Key, Value)) {
+    Idx++;
+    // Need to make sure when the key is present, we won't step over the bound
+    // of Tuple operand array. Since (non-optional) DetailedSummary always comes
+    // last, the next entry in the tuple operand array must exist.
+    return Idx < Tuple->getNumOperands();
+  }
+  // It was absent, keep going.
+  return true;
+}
+
 ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) {
   MDTuple *Tuple = dyn_cast_or_null<MDTuple>(MD);
-  if (!Tuple && (Tuple->getNumOperands() < 8 || Tuple->getNumOperands() > 9))
+  if (!Tuple || Tuple->getNumOperands() < 8 || Tuple->getNumOperands() > 10)
     return nullptr;
 
-  int i = 0;
-  auto &FormatMD = Tuple->getOperand(i++);
+  unsigned I = 0;
+  auto &FormatMD = Tuple->getOperand(I++);
   ProfileSummary::Kind SummaryKind;
   if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat",
                      "SampleProfile"))
     SummaryKind = PSK_Sample;
   else if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat",
                           "InstrProf"))
     SummaryKind = PSK_Instr;
   else if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat",
                           "CSInstrProf"))
     SummaryKind = PSK_CSInstr;
   else
     return nullptr;
 
   uint64_t NumCounts, TotalCount, NumFunctions, MaxFunctionCount, MaxCount,
       MaxInternalCount;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "TotalCount",
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "TotalCount",
               TotalCount))
     return nullptr;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "MaxCount", MaxCount))
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxCount", MaxCount))
     return nullptr;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "MaxInternalCount",
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxInternalCount",
               MaxInternalCount))
     return nullptr;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "MaxFunctionCount",
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxFunctionCount",
               MaxFunctionCount))
     return nullptr;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "NumCounts",
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "NumCounts",
               NumCounts))
     return nullptr;
-  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(i++)), "NumFunctions",
+  if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "NumFunctions",
               NumFunctions))
     return nullptr;
-  // Initialize IsPartialProfile because the field is optional.
-  uint64_t IsPartialProfile = 0;
 
-  // IsPartialProfile is optional so it doesn't matter even if the next val
-  // is not IsPartialProfile.
-  if (getVal(dyn_cast<MDTuple>(Tuple->getOperand(i)), "IsPartialProfile",
-             IsPartialProfile)) {
-    // Need to make sure when IsPartialProfile is presented, we won't step
-    // over the bound of Tuple operand array.
-    if (Tuple->getNumOperands() < 9)
-      return nullptr;
-    i++;
-  }
+  // Optional fields. Initialize these variables because the fields are
+  // optional.
+  uint64_t IsPartialProfile = 0;
+  double PartialProfileRatio = 0;
+  if (!getOptionalVal(Tuple, I, "IsPartialProfile", IsPartialProfile))
+    return nullptr;
+  if (!getOptionalVal(Tuple, I, "PartialProfileRatio", PartialProfileRatio))
 
   SummaryEntryVector Summary;
-  if (!getSummaryFromMD(dyn_cast<MDTuple>(Tuple->getOperand(i++)), Summary))
+  if (!getSummaryFromMD(dyn_cast<MDTuple>(Tuple->getOperand(I++)), Summary))
     return nullptr;
   return new ProfileSummary(SummaryKind, std::move(Summary), TotalCount,
                             MaxCount, MaxInternalCount, MaxFunctionCount,
-                            NumCounts, NumFunctions, IsPartialProfile);
+                            NumCounts, NumFunctions, IsPartialProfile,
+                            PartialProfileRatio);
 }
 
 void ProfileSummary::printSummary(raw_ostream &OS) {
   OS << "Total functions: " << NumFunctions << "\n";
   OS << "Maximum function count: " << MaxFunctionCount << "\n";
   OS << "Maximum block count: " << MaxCount << "\n";
   OS << "Total number of blocks: " << NumCounts << "\n";
   OS << "Total count: " << TotalCount << "\n";

llvm/test/Transforms/PGOProfile/unreachable_bb.ll

entry:
   unreachable
 return:
   ret void
 }
 
 declare void @bar()
 
 ;USE: !0 = !{i32 1, !"ProfileSummary", !1}
-;USE: !1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
+;USE: !1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11}
 ;USE: !2 = !{!"ProfileFormat", !"InstrProf"}
 ;USE: !3 = !{!"TotalCount", i64 0}
 
 

llvm/unittests/ProfileData/SampleProfTest.cpp

void createRemapFile(SmallVectorImpl<char> &RemapPath, StringRef &RemapFile) {
     )";
     OS->close();
   }
 
   // Verify profile summary is consistent in the roundtrip to and from
   // Metadata. \p AddPartialField is to choose whether the Metadata
   // contains the IsPartialProfile field which is optional.
   void verifyProfileSummary(ProfileSummary &Summary, Module &M,
-                            const bool AddPartialField) {
+                            const bool AddPartialField,
+                            const bool AddPartialProfileRatioField) {
     LLVMContext &Context = M.getContext();
     const bool IsPartialProfile = Summary.isPartialProfile();
-    auto VerifySummary = [IsPartialProfile](ProfileSummary &Summary) mutable {
+    const double PartialProfileRatio = Summary.getPartialProfileRatio();
+    auto VerifySummary = [IsPartialProfile, PartialProfileRatio](
+                             ProfileSummary &Summary) mutable {
       ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind());
       ASSERT_EQ(137392u, Summary.getTotalCount());
       ASSERT_EQ(8u, Summary.getNumCounts());
       ASSERT_EQ(4u, Summary.getNumFunctions());
       ASSERT_EQ(1437u, Summary.getMaxFunctionCount());
       ASSERT_EQ(60351u, Summary.getMaxCount());
       ASSERT_EQ(IsPartialProfile, Summary.isPartialProfile());
+      ASSERT_EQ(PartialProfileRatio, Summary.getPartialProfileRatio());
 
       uint32_t Cutoff = 800000;
       auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
         return PE.Cutoff == Cutoff;
       };
       std::vector<ProfileSummaryEntry> &Details = Summary.getDetailedSummary();
       auto EightyPerc = find_if(Details, Predicate);
       Cutoff = 900000;
       auto NinetyPerc = find_if(Details, Predicate);
       Cutoff = 950000;
       auto NinetyFivePerc = find_if(Details, Predicate);
       Cutoff = 990000;
       auto NinetyNinePerc = find_if(Details, Predicate);
       ASSERT_EQ(60000u, EightyPerc->MinCount);
       ASSERT_EQ(12557u, NinetyPerc->MinCount);
       ASSERT_EQ(12557u, NinetyFivePerc->MinCount);
       ASSERT_EQ(610u, NinetyNinePerc->MinCount);
     };
     VerifySummary(Summary);
 
     // Test that conversion of summary to and from Metadata works.
-    Metadata *MD = Summary.getMD(Context, AddPartialField);
+    Metadata *MD =
+        Summary.getMD(Context, AddPartialField, AddPartialProfileRatioField);
     ASSERT_TRUE(MD);
     ProfileSummary *PS = ProfileSummary::getFromMD(MD);
     ASSERT_TRUE(PS);
     VerifySummary(*PS);
     delete PS;
 
     // Test that summary can be attached to and read back from module.
     M.eraseNamedMetadata(M.getOrInsertModuleFlagsMetadata());
void testRoundTrip(SampleProfileFormat Format, bool Remap, bool UseMD5) {
     StringRef StringviewRep =
         getRepInFormat(StringviewName, UseMD5, StringviewGUID);
     ASSERT_EQ(1000u, CTMap.get()[MconstructRep]);
     ASSERT_EQ(437u, CTMap.get()[StringviewRep]);
 
 
     ProfileSummary &Summary = Reader->getSummary();
     Summary.setPartialProfile(true);
-    verifyProfileSummary(Summary, M, true);
+    verifyProfileSummary(Summary, M, true, false);
 
     Summary.setPartialProfile(false);
-    verifyProfileSummary(Summary, M, true);
+    verifyProfileSummary(Summary, M, true, false);
+
+    Summary.setPartialProfileRatio(0.5);
+    verifyProfileSummary(Summary, M, false, true);
+
+    Summary.setPartialProfileRatio(0);
+    verifyProfileSummary(Summary, M, false, true);
 
     Summary.setPartialProfile(false);
-    verifyProfileSummary(Summary, M, false);
+    Summary.setPartialProfileRatio(0);
+    verifyProfileSummary(Summary, M, false, false);
+
+    Summary.setPartialProfile(true);
+    Summary.setPartialProfileRatio(0.5);
+    verifyProfileSummary(Summary, M, true, true);
   }
 
   void addFunctionSamples(StringMap<FunctionSamples> *Smap, const char *Fname,
                           uint64_t TotalSamples, uint64_t HeadSamples) {
     StringRef Name(Fname);
     FunctionSamples FcnSamples;
     FcnSamples.setName(Name);
     FcnSamples.addTotalSamples(TotalSamples);