diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -288,6 +288,8 @@ // numrefs, numrefs x valueid, // n x (valueid, offset)] FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS = 23, + // The total number of basic blocks in the module. + FS_BLOCK_COUNT = 24, }; enum MetadataCodes { diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h --- a/llvm/include/llvm/IR/Module.h +++ b/llvm/include/llvm/IR/Module.h @@ -493,10 +493,12 @@ void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, Constant *Val); void addModuleFlag(ModFlagBehavior Behavior, StringRef Key, uint32_t Val); void addModuleFlag(MDNode *Node); + /// Like addModuleFlag but replaces the old module flag if it already exists. + void setModuleFlag(ModFlagBehavior Behavior, StringRef Key, Metadata *Val); -/// @} -/// @name Materialization -/// @{ + /// @} + /// @name Materialization + /// @{ /// Sets the GVMaterializer to GVM. This module must not yet have a /// Materializer. To reset the materializer for a module that already has one, diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -1007,6 +1007,9 @@ StringSaver Saver; BumpPtrAllocator Alloc; + // The total number of basic blocks in the module. + uint64_t BlockCount; + // YAML I/O support. friend yaml::MappingTraits; @@ -1019,15 +1022,15 @@ public: // See HaveGVs variable comment. ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit = false) - : HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit), Saver(Alloc) { - } + : HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit), Saver(Alloc), + BlockCount(0) {} // Current version for the module summary in bitcode files. // The BitcodeSummaryVersion should be bumped whenever we introduce changes // in the way some record are interpreted, like flags for instance. // Note that incrementing this may require changes in both BitcodeReader.cpp // and BitcodeWriter.cpp. - static constexpr uint64_t BitcodeSummaryVersion = 8; + static constexpr uint64_t BitcodeSummaryVersion = 9; // Regular LTO module name for ASM writer static constexpr const char *getRegularLTOModuleName() { @@ -1039,6 +1042,12 @@ uint64_t getFlags() const; void setFlags(uint64_t Flags); + uint64_t getBlockCount() const { return BlockCount; } + void addBlockCount(uint64_t C) { BlockCount += C; } + void setBlockCount(uint64_t C) { BlockCount = C; } + + void setPartialSampleProfileRatio(Module &M) const; + gvsummary_iterator begin() { return GlobalValueMap.begin(); } const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); } gvsummary_iterator end() { return GlobalValueMap.end(); } diff --git a/llvm/include/llvm/IR/ProfileSummary.h b/llvm/include/llvm/IR/ProfileSummary.h --- a/llvm/include/llvm/IR/ProfileSummary.h +++ b/llvm/include/llvm/IR/ProfileSummary.h @@ -55,6 +55,10 @@ /// code. The common profile is usually merged from profiles collected /// from running other targets. bool Partial = false; + /// This approximately represents the ratio of the number of profile counters + /// of the program being built to the number of profile counters in the + /// partial sample profile. When 'Partial' is false, it is undefined. + double PartialProfileRatio; /// Return detailed summary as metadata. Metadata *getDetailedSummaryMD(LLVMContext &Context); @@ -65,15 +69,17 @@ uint64_t TotalCount, uint64_t MaxCount, uint64_t MaxInternalCount, uint64_t MaxFunctionCount, uint32_t NumCounts, uint32_t NumFunctions, - bool Partial = false) + bool Partial = false, double PartialProfileRatio = 0) : PSK(K), DetailedSummary(std::move(DetailedSummary)), TotalCount(TotalCount), MaxCount(MaxCount), MaxInternalCount(MaxInternalCount), MaxFunctionCount(MaxFunctionCount), - NumCounts(NumCounts), NumFunctions(NumFunctions), Partial(Partial) {} + NumCounts(NumCounts), NumFunctions(NumFunctions), Partial(Partial), + PartialProfileRatio(PartialProfileRatio) {} Kind getKind() const { return PSK; } /// Return summary information as metadata. - Metadata *getMD(LLVMContext &Context, bool AddPartialField = true); + Metadata *getMD(LLVMContext &Context, bool AddPartialField = true, + bool AddPartialProfileRatioField = true); /// Construct profile summary from metdata. static ProfileSummary *getFromMD(Metadata *MD); SummaryEntryVector &getDetailedSummary() { return DetailedSummary; } @@ -85,6 +91,8 @@ uint64_t getMaxInternalCount() { return MaxInternalCount; } void setPartialProfile(bool PP) { Partial = PP; } bool isPartialProfile() { return Partial; } + double getPartialProfileRatio() { return PartialProfileRatio; } + void setPartialProfileRatio(double R) { PartialProfileRatio = R; } }; } // end namespace llvm diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -249,6 +249,7 @@ assert(F.hasName()); unsigned NumInsts = 0; + uint64_t NumBlocks = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector CallGraphEdges; @@ -268,7 +269,8 @@ std::vector NonVolatileStores; bool HasInlineAsmMaybeReferencingInternal = false; - for (const BasicBlock &BB : F) + for (const BasicBlock &BB : F) { + ++NumBlocks; for (const Instruction &I : BB) { if (isa(I)) continue; @@ -390,6 +392,8 @@ .updateHotness(getHotness(Candidate.Count, PSI)); } } + } + Index.addBlockCount(NumBlocks); std::vector Refs; if (IsThinLTO) { diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -66,6 +66,19 @@ cl::desc("A fixed cold count that overrides the count derived from" " profile-summary-cutoff-cold")); +static cl::opt ScalePartialSampleProfileWorkingSetSize( + "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true), + cl::desc( + "If true, scale the working set size of the partial sample profile " + "by the partial profile ratio to reflect the size of the program " + "being compiled.")); + +static cl::opt PartialSampleProfileWorkingSetSizeScaleFactor( + "partial-sample-profile-working-set-size-scale-factor", cl::Hidden, + cl::init(0.008), + cl::desc("The scale factor used to scale the working set size of the " + "partial sample profile along with the partial profile ratio.")); + // Find the summary entry for a desired percentile of counts. static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS, uint64_t Percentile) { @@ -273,10 +286,21 @@ ColdCountThreshold = ProfileSummaryColdCount; assert(ColdCountThreshold <= HotCountThreshold && "Cold count threshold cannot exceed hot count threshold!"); - HasHugeWorkingSetSize = - HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; - HasLargeWorkingSetSize = - HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; + if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) { + HasHugeWorkingSetSize = + HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; + HasLargeWorkingSetSize = + HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; + } else { + double PartialProfileRatio = Summary->getPartialProfileRatio(); + uint64_t ScaledHotEntryNumCounts = + static_cast(HotEntry.NumCounts * PartialProfileRatio * + PartialSampleProfileWorkingSetSizeScaleFactor); + HasHugeWorkingSetSize = + ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; + HasLargeWorkingSetSize = + ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; + } } Optional ProfileSummaryInfo::computeThreshold(int PercentileCutoff) { diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -739,6 +739,7 @@ KEYWORD(name); KEYWORD(summaries); KEYWORD(flags); + KEYWORD(blockcount); KEYWORD(linkage); KEYWORD(notEligibleToImport); KEYWORD(live); diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h --- a/llvm/lib/AsmParser/LLParser.h +++ b/llvm/lib/AsmParser/LLParser.h @@ -347,6 +347,7 @@ bool ParseModuleReference(StringRef &ModulePath); bool ParseGVReference(ValueInfo &VI, unsigned &GVId); bool ParseSummaryIndexFlags(); + bool ParseBlockCount(); bool ParseGVEntry(unsigned ID); bool ParseFunctionSummary(std::string Name, GlobalValue::GUID, unsigned ID); bool ParseVariableSummary(std::string Name, GlobalValue::GUID, unsigned ID); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -855,6 +855,9 @@ case lltok::kw_flags: result = ParseSummaryIndexFlags(); break; + case lltok::kw_blockcount: + result = ParseBlockCount(); + break; default: result = Error(Lex.getLoc(), "unexpected summary kind"); break; @@ -8067,6 +8070,21 @@ return false; } +/// ParseBlockCount +/// ::= 'blockcount' ':' UInt64 +bool LLParser::ParseBlockCount() { + assert(Lex.getKind() == lltok::kw_blockcount); + Lex.Lex(); + + if (ParseToken(lltok::colon, "expected ':' here")) + return true; + uint64_t BlockCount; + if (ParseUInt64(BlockCount)) + return true; + Index->setBlockCount(BlockCount); + return false; +} + /// ParseGVEntry /// ::= 'gv' ':' '(' ('name' ':' STRINGCONSTANT | 'guid' ':' UInt64) /// [',' 'summaries' ':' Summary[',' Summary]* ]? ')' diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h --- a/llvm/lib/AsmParser/LLToken.h +++ b/llvm/lib/AsmParser/LLToken.h @@ -372,6 +372,7 @@ kw_name, kw_summaries, kw_flags, + kw_blockcount, kw_linkage, kw_notEligibleToImport, kw_live, diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -305,6 +305,7 @@ STRINGIFY_CODE(FS, CFI_FUNCTION_DECLS) STRINGIFY_CODE(FS, TYPE_ID) STRINGIFY_CODE(FS, TYPE_ID_METADATA) + STRINGIFY_CODE(FS, BLOCK_COUNT) } case bitc::METADATA_ATTACHMENT_ID: switch (CodeID) { @@ -984,4 +985,3 @@ return Skipped.takeError(); } } - diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -6197,6 +6197,10 @@ case bitc::FS_TYPE_ID_METADATA: parseTypeIdCompatibleVtableSummaryRecord(Record); break; + + case bitc::FS_BLOCK_COUNT: + // TheIndex.setBlockCount(Record[0]); + TheIndex.addBlockCount(Record[0]); } } llvm_unreachable("Exit infinite loop"); diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3901,6 +3901,9 @@ NameVals.clear(); } + Stream.EmitRecord(bitc::FS_BLOCK_COUNT, + ArrayRef{Index->getBlockCount()}); + Stream.ExitBlock(); } @@ -4184,6 +4187,9 @@ } } + Stream.EmitRecord(bitc::FS_BLOCK_COUNT, + ArrayRef{Index.getBlockCount()}); + Stream.ExitBlock(); } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -2789,8 +2789,13 @@ } // Don't emit flags when it's not really needed (value is zero by default). - if (TheIndex->getFlags()) + if (TheIndex->getFlags()) { Out << "^" << NumSlots << " = flags: " << TheIndex->getFlags() << "\n"; + ++NumSlots; + } + + Out << "^" << NumSlots << " = blockcount: " << TheIndex->getBlockCount() + << "\n"; } static const char * diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp --- a/llvm/lib/IR/Module.cpp +++ b/llvm/lib/IR/Module.cpp @@ -358,6 +358,24 @@ getOrInsertModuleFlagsMetadata()->addOperand(Node); } +void Module::setModuleFlag(ModFlagBehavior Behavior, StringRef Key, + Metadata *Val) { + NamedMDNode *ModFlags = getOrInsertModuleFlagsMetadata(); + // Replace the flag if it already exists. + for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { + MDNode *Flag = ModFlags->getOperand(I); + ModFlagBehavior MFB; + if (Flag->getNumOperands() == 3 && + isValidModFlagBehavior(Flag->getOperand(0), MFB) && MFB == Behavior && + dyn_cast_or_null(Flag->getOperand(1)) && + cast(Flag->getOperand(1))->getString() == Key) { + Flag->replaceOperandWith(2, Val); + return; + } + } + addModuleFlag(Behavior, Key, Val); +} + void Module::setDataLayout(StringRef Desc) { DL.reset(Desc); } @@ -547,9 +565,9 @@ void Module::setProfileSummary(Metadata *M, ProfileSummary::Kind Kind) { if (Kind == ProfileSummary::PSK_CSInstr) - addModuleFlag(ModFlagBehavior::Error, "CSProfileSummary", M); + setModuleFlag(ModFlagBehavior::Error, "CSProfileSummary", M); else - addModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M); + setModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M); } Metadata *Module::getProfileSummary(bool IsCS) { diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -601,3 +601,23 @@ OS << "}"; } + +void ModuleSummaryIndex::setPartialSampleProfileRatio(Module &M) const { + // Set the PartialProfileRatio field for partial profile sample + // ProfileSummary. + if (auto *SummaryMD = M.getProfileSummary(/*IsCS*/ false)) { + if (auto *ProfileSummary = ProfileSummary::getFromMD(SummaryMD)) { + if (ProfileSummary->getKind() != ProfileSummary::PSK_Sample || + !ProfileSummary->isPartialProfile()) + return; + uint64_t BlockCount = getBlockCount(); + uint32_t NumCounts = ProfileSummary->getNumCounts(); + if (!NumCounts) + return; + double Ratio = (double)BlockCount / NumCounts; + ProfileSummary->setPartialProfileRatio(Ratio); + M.setProfileSummary(ProfileSummary->getMD(M.getContext()), + ProfileSummary::PSK_Sample); + } + } +} diff --git a/llvm/lib/IR/ProfileSummary.cpp b/llvm/lib/IR/ProfileSummary.cpp --- a/llvm/lib/IR/ProfileSummary.cpp +++ b/llvm/lib/IR/ProfileSummary.cpp @@ -31,6 +31,14 @@ return MDTuple::get(Context, Ops); } +static Metadata *getKeyFPValMD(LLVMContext &Context, const char *Key, + double Val) { + Type *DoubleTy = Type::getDoubleTy(Context); + Metadata *Ops[2] = {MDString::get(Context, Key), + ConstantAsMetadata::get(ConstantFP::get(DoubleTy, Val))}; + return MDTuple::get(Context, Ops); +} + // Return an MDTuple with two elements. The first element is a string Key and // the second is a string Value. static Metadata *getKeyValMD(LLVMContext &Context, const char *Key, @@ -67,7 +75,8 @@ // to the kind of profile summary as returned by getFormatSpecificMD. // IsPartialProfile is an optional field and \p AddPartialField will decide // whether to add a field for it. -Metadata *ProfileSummary::getMD(LLVMContext &Context, bool AddPartialField) { +Metadata *ProfileSummary::getMD(LLVMContext &Context, bool AddPartialField, + bool AddPartialProfileRatioField) { const char *KindStr[3] = {"InstrProf", "CSInstrProf", "SampleProfile"}; SmallVector Components; Components.push_back(getKeyValMD(Context, "ProfileFormat", KindStr[PSK])); @@ -82,6 +91,9 @@ if (AddPartialField) Components.push_back( getKeyValMD(Context, "IsPartialProfile", isPartialProfile())); + if (AddPartialProfileRatioField) + Components.push_back(getKeyFPValMD(Context, "PartialProfileRatio", + getPartialProfileRatio())); Components.push_back(getDetailedSummaryMD(Context)); return MDTuple::get(Context, Components); } @@ -102,6 +114,21 @@ return true; } +static bool getVal(MDTuple *MD, const char *Key, double &Val) { + if (!MD) + return false; + if (MD->getNumOperands() != 2) + return false; + MDString *KeyMD = dyn_cast(MD->getOperand(0)); + ConstantAsMetadata *ValMD = dyn_cast(MD->getOperand(1)); + if (!KeyMD || !ValMD) + return false; + if (!KeyMD->getString().equals(Key)) + return false; + Val = cast(ValMD->getValue())->getValueAPF().convertToDouble(); + return true; +} + // Check if an MDTuple represents a (Key, Val) pair. static bool isKeyValuePair(MDTuple *MD, const char *Key, const char *Val) { if (!MD || MD->getNumOperands() != 2) @@ -145,12 +172,26 @@ return true; } +template +static bool getOptionalVal(MDTuple *Tuple, unsigned &Idx, const char *Key, + ValueType &Value) { + if (getVal(dyn_cast(Tuple->getOperand(Idx)), Key, Value)) { + Idx++; + // Need to make sure when the key is present, we won't step over the bound + // of Tuple operand array. Since (non-optional) DetailedSummary always comes + // last, the next entry in the tuple operand array must exist. + return Idx < Tuple->getNumOperands(); + } + // It was absent, keep going. + return true; +} + ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) { MDTuple *Tuple = dyn_cast_or_null(MD); - if (!Tuple && (Tuple->getNumOperands() < 8 || Tuple->getNumOperands() > 9)) + if (!Tuple || Tuple->getNumOperands() < 8 || Tuple->getNumOperands() > 10) return nullptr; - int i = 0; + unsigned i = 0; auto &FormatMD = Tuple->getOperand(i++); ProfileSummary::Kind SummaryKind; if (isKeyValuePair(dyn_cast_or_null(FormatMD), "ProfileFormat", @@ -184,24 +225,21 @@ if (!getVal(dyn_cast(Tuple->getOperand(i++)), "NumFunctions", NumFunctions)) return nullptr; - // Initialize IsPartialProfile because the field is optional. - uint64_t IsPartialProfile = 0; - // IsPartialProfile is optional so it doesn't matter even if the next val - // is not IsPartialProfile. - if (getVal(dyn_cast(Tuple->getOperand(i)), "IsPartialProfile", - IsPartialProfile)) { - // Need to make sure when IsPartialProfile is presented, we won't step - // over the bound of Tuple operand array. - if (Tuple->getNumOperands() < 9) - return nullptr; - i++; - } + // Optional fields. Initialize these variables because the fields are + // optional. + uint64_t IsPartialProfile = 0; + double PartialProfileRatio = 0; + if (!getOptionalVal(Tuple, i, "IsPartialProfile", IsPartialProfile)) + return nullptr; + if (!getOptionalVal(Tuple, i, "PartialProfileRatio", PartialProfileRatio)) + return nullptr; SummaryEntryVector Summary; if (!getSummaryFromMD(dyn_cast(Tuple->getOperand(i++)), Summary)) return nullptr; return new ProfileSummary(SummaryKind, std::move(Summary), TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount, - NumCounts, NumFunctions, IsPartialProfile); + NumCounts, NumFunctions, IsPartialProfile, + PartialProfileRatio); } diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -541,6 +541,8 @@ return DiagFileOrErr.takeError(); auto DiagnosticOutputFile = std::move(*DiagFileOrErr); + CombinedIndex.setPartialSampleProfileRatio(Mod); + if (Conf.CodeGenOnly) { codegen(Conf, TM.get(), AddStream, Task, Mod); return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile)); diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -1232,6 +1232,8 @@ // have loaded all the required metadata! UpgradeDebugInfo(*SrcModule); + Index.setPartialSampleProfileRatio(*SrcModule); + // Link in the specified functions. if (renameModuleForThinLTO(*SrcModule, Index, ClearDSOLocalOnDeclarations, &GlobalsToImport)) diff --git a/llvm/test/Assembler/thinlto-vtable-summary.ll b/llvm/test/Assembler/thinlto-vtable-summary.ll --- a/llvm/test/Assembler/thinlto-vtable-summary.ll +++ b/llvm/test/Assembler/thinlto-vtable-summary.ll @@ -36,3 +36,4 @@ ^6 = typeidCompatibleVTable: (name: "_ZTS1A", summary: ((offset: 16, ^2), (offset: 16, ^4))) ; guid = 7004155349499253778 ^7 = typeidCompatibleVTable: (name: "_ZTS1B", summary: ((offset: 16, ^2))) ; guid = 6203814149063363976 ^8 = typeidCompatibleVTable: (name: "_ZTS1C", summary: ((offset: 16, ^4))) ; guid = 1884921850105019584 +^9 = blockcount: 0 diff --git a/llvm/test/Bitcode/summary_version.ll b/llvm/test/Bitcode/summary_version.ll --- a/llvm/test/Bitcode/summary_version.ll +++ b/llvm/test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: diff --git a/llvm/test/Bitcode/thinlto-alias.ll b/llvm/test/Bitcode/thinlto-alias.ll --- a/llvm/test/Bitcode/thinlto-alias.ll +++ b/llvm/test/Bitcode/thinlto-alias.ll @@ -22,6 +22,7 @@ ; See if the call to func is registered. ; The value id 1 matches the second FUNCTION record above. ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: ; COMBINED-NEXT: ; COMBINED-NEXT: ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; ModuleID = 'thinlto-alias2.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph-cast.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph-cast.ll --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph-cast.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph-cast.ll @@ -12,6 +12,7 @@ ; CHECK-NEXT: ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph-cast.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll @@ -19,6 +19,7 @@ ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll @@ -50,6 +50,7 @@ ; CHECK-NEXT: ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123 ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: -; COMBINED_NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll @@ -15,6 +15,7 @@ ; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: ; op4=none1 op6=hot1 op8=cold1 op10=none2 op12=hot2 op14=cold2 op16=none3 op18=hot3 op20=cold3 op22=123 ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: -; COMBINED_NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph.ll --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph.ll @@ -20,6 +20,7 @@ ; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' diff --git a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll --- a/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-originalnames.ll @@ -15,6 +15,7 @@ ; COMBINED-DAG: ; COMBINED-DAG: +; COMBINED-NEXT: ; COMBINED-NEXT: source_filename = "/path/to/source.c" diff --git a/llvm/test/Bitcode/thinlto-function-summary.ll b/llvm/test/Bitcode/thinlto-function-summary.ll --- a/llvm/test/Bitcode/thinlto-function-summary.ll +++ b/llvm/test/Bitcode/thinlto-function-summary.ll @@ -25,6 +25,7 @@ ; BC-NEXT: ; BC-NEXT: ; BACKEND1-NEXT: ; BACKEND2-NEXT: ; COMBINED-NEXT: makeLLVMModule(const char *ProfKind = nullptr) { + std::unique_ptr makeLLVMModule(const char *ProfKind = nullptr, + uint64_t NumCounts = 3, + uint64_t IsPartialProfile = 0, + double PartialProfileRatio = 0.0, + uint64_t HotNumCounts = 3, + uint64_t ColdNumCounts = 10) { const char *ModuleString = "define i32 @g(i32 %x) !prof !21 {{\n" " ret i32 0\n" @@ -83,27 +88,32 @@ "!22 = !{{!\"function_entry_count\", i64 100}\n" "!23 = !{{!\"branch_weights\", i32 64, i32 4}\n" "{0}"; - const char *SummaryString = "!llvm.module.flags = !{{!1}" - "!1 = !{{i32 1, !\"ProfileSummary\", !2}" - "!2 = !{{!3, !4, !5, !6, !7, !8, !9, !10}" - "!3 = !{{!\"ProfileFormat\", !\"{0}\"}" - "!4 = !{{!\"TotalCount\", i64 10000}" - "!5 = !{{!\"MaxCount\", i64 10}" - "!6 = !{{!\"MaxInternalCount\", i64 1}" - "!7 = !{{!\"MaxFunctionCount\", i64 1000}" - "!8 = !{{!\"NumCounts\", i64 3}" - "!9 = !{{!\"NumFunctions\", i64 3}" - "!10 = !{{!\"DetailedSummary\", !11}" - "!11 = !{{!12, !13, !14}" - "!12 = !{{i32 10000, i64 1000, i32 1}" - "!13 = !{{i32 999000, i64 300, i32 3}" - "!14 = !{{i32 999999, i64 5, i32 10}"; + const char *SummaryString = + "!llvm.module.flags = !{{!1}\n" + "!1 = !{{i32 1, !\"ProfileSummary\", !2}\n" + "!2 = !{{!3, !4, !5, !6, !7, !8, !9, !10, !11, !12}\n" + "!3 = !{{!\"ProfileFormat\", !\"{0}\"}\n" + "!4 = !{{!\"TotalCount\", i64 10000}\n" + "!5 = !{{!\"MaxCount\", i64 10}\n" + "!6 = !{{!\"MaxInternalCount\", i64 1}\n" + "!7 = !{{!\"MaxFunctionCount\", i64 1000}\n" + "!8 = !{{!\"NumCounts\", i64 {1}}\n" + "!9 = !{{!\"NumFunctions\", i64 3}\n" + "!10 = !{{!\"IsPartialProfile\", i64 {2}}\n" + "!11 = !{{!\"PartialProfileRatio\", double {3}}\n" + "!12 = !{{!\"DetailedSummary\", !13}\n" + "!13 = !{{!14, !15, !16}\n" + "!14 = !{{i32 10000, i64 1000, i32 1}\n" + "!15 = !{{i32 990000, i64 300, i32 {4}}\n" + "!16 = !{{i32 999999, i64 5, i32 {5}}\n"; SMDiagnostic Err; - if (ProfKind) - return parseAssemblyString( - formatv(ModuleString, formatv(SummaryString, ProfKind).str()).str(), - Err, C); - else + if (ProfKind) { + auto Summary = + formatv(SummaryString, ProfKind, NumCounts, IsPartialProfile, + PartialProfileRatio, HotNumCounts, ColdNumCounts) + .str(); + return parseAssemblyString(formatv(ModuleString, Summary).str(), Err, C); + } else return parseAssemblyString(formatv(ModuleString, "").str(), Err, C); } }; @@ -280,6 +290,7 @@ ProfileSummaryInfo PSI = buildPSI(M.get()); EXPECT_TRUE(PSI.hasProfileSummary()); EXPECT_TRUE(PSI.hasSampleProfile()); + EXPECT_FALSE(PSI.hasPartialSampleProfile()); BasicBlock &BB0 = F->getEntryBlock(); BasicBlock *BB1 = BB0.getTerminator()->getSuccessor(0); @@ -373,5 +384,45 @@ EXPECT_FALSE(PSI.isFunctionColdInCallGraphNthPercentile(990000, F, BFI)); } +TEST_F(ProfileSummaryInfoTest, PartialSampleProfWorkingSetSize) { + // With PartialProfileRatio unset (zero.) + auto M1 = makeLLVMModule("SampleProfile", /*NumCounts*/ 3, + /*IsPartialProfile*/ 1, + /*PartialProfileRatio*/ 0.0, + /*HotNumCounts*/ 3, /*ColdNumCounts*/ 10); + ProfileSummaryInfo PSI1 = buildPSI(M1.get()); + EXPECT_TRUE(PSI1.hasProfileSummary()); + EXPECT_TRUE(PSI1.hasSampleProfile()); + EXPECT_TRUE(PSI1.hasPartialSampleProfile()); + EXPECT_FALSE(PSI1.hasHugeWorkingSetSize()); + EXPECT_FALSE(PSI1.hasLargeWorkingSetSize()); + + // With PartialProfileRatio set (non-zero) and a small working set size. + auto M2 = makeLLVMModule("SampleProfile", /*NumCounts*/ 27493235, + /*IsPartialProfile*/ 1, + /*PartialProfileRatio*/ 0.00000012, + /*HotNumCounts*/ 3102082, + /*ColdNumCounts*/ 18306149); + ProfileSummaryInfo PSI2 = buildPSI(M2.get()); + EXPECT_TRUE(PSI2.hasProfileSummary()); + EXPECT_TRUE(PSI2.hasSampleProfile()); + EXPECT_TRUE(PSI2.hasPartialSampleProfile()); + EXPECT_FALSE(PSI2.hasHugeWorkingSetSize()); + EXPECT_FALSE(PSI2.hasLargeWorkingSetSize()); + + // With PartialProfileRatio is set (non-zero) and a large working set size. + auto M3 = makeLLVMModule("SampleProfile", /*NumCounts*/ 27493235, + /*IsPartialProfile*/ 1, + /*PartialProfileRatio*/ 0.9, + /*HotNumCounts*/ 3102082, + /*ColdNumCounts*/ 18306149); + ProfileSummaryInfo PSI3 = buildPSI(M3.get()); + EXPECT_TRUE(PSI3.hasProfileSummary()); + EXPECT_TRUE(PSI3.hasSampleProfile()); + EXPECT_TRUE(PSI3.hasPartialSampleProfile()); + EXPECT_TRUE(PSI3.hasHugeWorkingSetSize()); + EXPECT_TRUE(PSI3.hasLargeWorkingSetSize()); +} + } // end anonymous namespace } // end namespace llvm diff --git a/llvm/unittests/IR/CMakeLists.txt b/llvm/unittests/IR/CMakeLists.txt --- a/llvm/unittests/IR/CMakeLists.txt +++ b/llvm/unittests/IR/CMakeLists.txt @@ -28,6 +28,7 @@ MDBuilderTest.cpp ManglerTest.cpp MetadataTest.cpp + ModuleSummaryIndexTest.cpp ModuleTest.cpp PassManagerTest.cpp PatternMatch.cpp diff --git a/llvm/unittests/IR/ModuleSummaryIndexTest.cpp b/llvm/unittests/IR/ModuleSummaryIndexTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/IR/ModuleSummaryIndexTest.cpp @@ -0,0 +1,54 @@ +//===- unittests/IR/ModuleSummaryIndexTest.cpp ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/AsmParser/Parser.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +const char *IRString = R"IR( + !llvm.module.flags = !{!0} + + !0 = !{i32 1, !"ProfileSummary", !1} + !1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11} + !2 = !{!"ProfileFormat", !"SampleProfile"} + !3 = !{!"TotalCount", i64 10000} + !4 = !{!"MaxCount", i64 10} + !5 = !{!"MaxInternalCount", i64 1} + !6 = !{!"MaxFunctionCount", i64 1000} + !7 = !{!"NumCounts", i64 200} + !8 = !{!"NumFunctions", i64 3} + !9 = !{!"IsPartialProfile", i64 1} + !10 = !{!"PartialProfileRatio", double 0.0} + !11 = !{!"DetailedSummary", !12} + !12 = !{!13, !14, !15} + !13 = !{i32 10000, i64 1000, i32 1} + !14 = !{i32 990000, i64 300, i32 10} + !15 = !{i32 999999, i64 5, i32 100} +)IR"; + +TEST(ModuleSummaryIndexTest, setPartialSampleProfileRatio) { + SMDiagnostic Err; + LLVMContext Context; + std::unique_ptr M = parseAssemblyString(IRString, Err, Context); + ModuleSummaryIndex Index(/*HaveGVs*/ false); + const unsigned BlockCount = 100; + const unsigned NumCounts = 200; + Index.setBlockCount(BlockCount); + Index.setPartialSampleProfileRatio(*M); + double Ratio = (double)BlockCount / NumCounts; + auto *ProfileSummary = + ProfileSummary::getFromMD(M->getProfileSummary(/*IsCS*/ false)); + EXPECT_EQ(Ratio, ProfileSummary->getPartialProfileRatio()); + delete ProfileSummary; +} + +} // end namespace diff --git a/llvm/unittests/IR/ModuleTest.cpp b/llvm/unittests/IR/ModuleTest.cpp --- a/llvm/unittests/IR/ModuleTest.cpp +++ b/llvm/unittests/IR/ModuleTest.cpp @@ -72,4 +72,17 @@ RandomStreams[1].begin())); } +TEST(ModuleTest, setModuleFlag) { + LLVMContext Context; + Module M("M", Context); + StringRef Key = "Key"; + Metadata *Val1 = MDString::get(Context, "Val1"); + Metadata *Val2 = MDString::get(Context, "Val2"); + EXPECT_EQ(nullptr, M.getModuleFlag(Key)); + M.setModuleFlag(Module::ModFlagBehavior::Error, Key, Val1); + EXPECT_EQ(Val1, M.getModuleFlag(Key)); + M.setModuleFlag(Module::ModFlagBehavior::Error, Key, Val2); + EXPECT_EQ(Val2, M.getModuleFlag(Key)); +} + } // end namespace diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -81,10 +81,13 @@ // Metadata. \p AddPartialField is to choose whether the Metadata // contains the IsPartialProfile field which is optional. void verifyProfileSummary(ProfileSummary &Summary, Module &M, - const bool AddPartialField) { + const bool AddPartialField, + const bool AddPartialProfileRatioField) { LLVMContext &Context = M.getContext(); const bool IsPartialProfile = Summary.isPartialProfile(); - auto VerifySummary = [IsPartialProfile](ProfileSummary &Summary) mutable { + const double PartialProfileRatio = Summary.getPartialProfileRatio(); + auto VerifySummary = [IsPartialProfile, PartialProfileRatio]( + ProfileSummary &Summary) mutable { ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind()); ASSERT_EQ(137392u, Summary.getTotalCount()); ASSERT_EQ(8u, Summary.getNumCounts()); @@ -92,6 +95,7 @@ ASSERT_EQ(1437u, Summary.getMaxFunctionCount()); ASSERT_EQ(60351u, Summary.getMaxCount()); ASSERT_EQ(IsPartialProfile, Summary.isPartialProfile()); + ASSERT_EQ(PartialProfileRatio, Summary.getPartialProfileRatio()); uint32_t Cutoff = 800000; auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) { @@ -113,7 +117,8 @@ VerifySummary(Summary); // Test that conversion of summary to and from Metadata works. - Metadata *MD = Summary.getMD(Context, AddPartialField); + Metadata *MD = + Summary.getMD(Context, AddPartialField, AddPartialProfileRatioField); ASSERT_TRUE(MD); ProfileSummary *PS = ProfileSummary::getFromMD(MD); ASSERT_TRUE(PS); @@ -271,13 +276,24 @@ ProfileSummary &Summary = Reader->getSummary(); Summary.setPartialProfile(true); - verifyProfileSummary(Summary, M, true); + verifyProfileSummary(Summary, M, true, false); Summary.setPartialProfile(false); - verifyProfileSummary(Summary, M, true); + verifyProfileSummary(Summary, M, true, false); + + Summary.setPartialProfileRatio(0.5); + verifyProfileSummary(Summary, M, false, true); + + Summary.setPartialProfileRatio(0); + verifyProfileSummary(Summary, M, false, true); Summary.setPartialProfile(false); - verifyProfileSummary(Summary, M, false); + Summary.setPartialProfileRatio(0); + verifyProfileSummary(Summary, M, false, false); + + Summary.setPartialProfile(true); + Summary.setPartialProfileRatio(0.5); + verifyProfileSummary(Summary, M, true, true); } void addFunctionSamples(StringMap *Smap, const char *Fname,