diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -200,9 +200,9 @@ /// SecFlagFSDiscriminator means this profile uses flow-sensitive /// discriminators. SecFlagFSDiscriminator = (1 << 2), - /// SecFlagIsCSNested means this is context-sensitive nested profile for - /// CSSPGO - SecFlagIsCSNested = (1 << 4), + /// SecFlagIsPreInlined means this profile contains ShouldBeInlined + /// contexts thus this is CS preinliner computed. + SecFlagIsPreInlined = (1 << 4), }; enum class SecFuncMetadataFlags : uint32_t { @@ -830,7 +830,7 @@ /// Return the sample count of the first instruction of the function. /// The function can be either a standalone symbol or an inlined function. uint64_t getEntrySamples() const { - if (FunctionSamples::ProfileIsCSFlat && getHeadSamples()) { + if (FunctionSamples::ProfileIsCS && getHeadSamples()) { // For CS profile, if we already have more accurate head samples // counted by branch sample from caller, use them as entry samples. return getHeadSamples(); @@ -1047,9 +1047,9 @@ static bool ProfileIsProbeBased; - static bool ProfileIsCSFlat; + static bool ProfileIsCS; - static bool ProfileIsCSNested; + static bool ProfileIsPreInlined; SampleContext &getContext() const { return Context; } diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -470,11 +470,11 @@ /// Whether input profile is based on pseudo probes. bool profileIsProbeBased() const { return ProfileIsProbeBased; } - /// Whether input profile is fully context-sensitive and flat. - bool profileIsCSFlat() const { return ProfileIsCSFlat; } + /// Whether input profile is fully context-sensitive. + bool profileIsCS() const { return ProfileIsCS; } - /// Whether input profile is fully context-sensitive and nested. - bool profileIsCSNested() const { return ProfileIsCSNested; } + /// Whether input profile contains ShouldBeInlined contexts. + bool profileIsPreInlined() const { return ProfileIsPreInlined; } virtual std::unique_ptr getProfileSymbolList() { return nullptr; @@ -534,10 +534,10 @@ bool ProfileIsProbeBased = false; /// Whether function profiles are context-sensitive flat profiles. - bool ProfileIsCSFlat = false; + bool ProfileIsCS = false; - /// Whether function profiles are context-sensitive nested profiles. - bool ProfileIsCSNested = false; + /// Whether function profile contains ShouldBeInlined contexts. + bool ProfileIsPreInlined = false; /// Number of context-sensitive profiles. uint32_t CSProfileCount = 0; diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h --- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h +++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h @@ -68,7 +68,7 @@ // Constructor for non-CS profile. ProfiledCallGraph(SampleProfileMap &ProfileMap) { - assert(!FunctionSamples::ProfileIsCSFlat && + assert(!FunctionSamples::ProfileIsCS && "CS flat profile is not handled here"); for (const auto &Samples : ProfileMap) { addProfiledCalls(Samples.second); diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -196,7 +196,7 @@ // more function profiles each with lower counts, which in turn leads to lower // hot thresholds. To compensate for that, by default we merge context // profiles before computing profile summary. - if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCSFlat && + if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && !UseContextLessSummary.getNumOccurrences())) { for (const auto &I : Profiles) { ContextLessProfiles[I.second.getName()].merge(I.second); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -42,8 +42,8 @@ namespace llvm { namespace sampleprof { bool FunctionSamples::ProfileIsProbeBased = false; -bool FunctionSamples::ProfileIsCSFlat = false; -bool FunctionSamples::ProfileIsCSNested = false; +bool FunctionSamples::ProfileIsCS = false; +bool FunctionSamples::ProfileIsPreInlined = false; bool FunctionSamples::UseMD5 = false; bool FunctionSamples::HasUniqSuffix = true; bool FunctionSamples::ProfileIsFS = false; @@ -535,11 +535,6 @@ ContextDuplicatedIntoBase); } - // Contexts coming with a `ContextShouldBeInlined` attribute indicate this - // is a preinliner-computed profile. - if (OrigChildContext.hasAttribute(ContextShouldBeInlined)) - FunctionSamples::ProfileIsCSNested = true; - // Remove the original child profile. ProfileMap.erase(OrigChildContext); } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -348,7 +348,7 @@ } FProfile.getContext().setAllAttributes(Attributes); if (Attributes & (uint32_t)ContextShouldBeInlined) - ProfileIsCSNested = true; + ProfileIsPreInlined = true; DepthMetadata = Depth; break; } @@ -358,14 +358,14 @@ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile"); - ProfileIsCSFlat = (CSProfileCount > 0); + ProfileIsCS = (CSProfileCount > 0); assert((TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles"); ProfileIsProbeBased = (TopLevelProbeProfileCount > 0); FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; - FunctionSamples::ProfileIsCSFlat = ProfileIsCSFlat; - FunctionSamples::ProfileIsCSNested = ProfileIsCSNested; + FunctionSamples::ProfileIsCS = ProfileIsCS; + FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined; if (Result == sampleprof_error::success) computeSummary(); @@ -630,7 +630,7 @@ ErrorOr SampleProfileReaderExtBinaryBase::readSampleContextFromTable() { - if (ProfileIsCSFlat) { + if (ProfileIsCS) { auto FContext(readContextFromTable()); if (std::error_code EC = FContext.getError()) return EC; @@ -654,9 +654,9 @@ if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) Summary->setPartialProfile(true); if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) - FunctionSamples::ProfileIsCSFlat = ProfileIsCSFlat = true; - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsCSNested)) - FunctionSamples::ProfileIsCSNested = ProfileIsCSNested = true; + FunctionSamples::ProfileIsCS = ProfileIsCS = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) + FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) FunctionSamples::ProfileIsFS = ProfileIsFS = true; break; @@ -777,7 +777,7 @@ } } - if (ProfileIsCSFlat) { + if (ProfileIsCS) { DenseSet FuncGuidsToUse; if (useMD5()) { for (auto Name : FuncsToUse) @@ -847,7 +847,7 @@ } assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile"); - assert((!CSProfileCount || ProfileIsCSFlat) && + assert((!CSProfileCount || ProfileIsCS) && "Section flag should be consistent with actual profile"); return sampleprof_error::success; } @@ -1105,7 +1105,7 @@ FProfile->getContext().setAllAttributes(*Attributes); } - if (!ProfileIsCSFlat) { + if (!ProfileIsCS) { // Read all the attributes for inlined function calls. auto NumCallsites = readNumber(); if (std::error_code EC = NumCallsites.getError()) @@ -1275,8 +1275,8 @@ Flags.append("partial,"); if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) Flags.append("context,"); - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsCSNested)) - Flags.append("context-nested,"); + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) + Flags.append("preInlined,"); if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) Flags.append("fs-discriminator,"); break; diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -169,7 +169,7 @@ return (std::error_code)sampleprof_error::success; }; - if (FunctionSamples::ProfileIsCSFlat) { + if (FunctionSamples::ProfileIsCS) { // Sort the contexts before writing them out. This is to help fast load all // context profiles for a function as well as their callee contexts which // can help profile-guided importing for ThinLTO. @@ -199,11 +199,11 @@ if (FunctionSamples::ProfileIsProbeBased) encodeULEB128(FunctionProfile.getFunctionHash(), OS); - if (FunctionSamples::ProfileIsCSFlat || FunctionSamples::ProfileIsCSNested) { + if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) { encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS); } - if (!FunctionSamples::ProfileIsCSFlat) { + if (!FunctionSamples::ProfileIsCS) { // Recursively emit attributes for all callee samples. uint64_t NumCallsites = 0; for (const auto &J : FunctionProfile.getCallsiteSamples()) @@ -226,7 +226,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( const SampleProfileMap &Profiles) { if (!FunctionSamples::ProfileIsProbeBased && - !FunctionSamples::ProfileIsCSFlat && !FunctionSamples::ProfileIsCSNested) + !FunctionSamples::ProfileIsCS && !FunctionSamples::ProfileIsPreInlined) return sampleprof_error::success; for (const auto &Entry : Profiles) { if (std::error_code EC = writeFuncMetadata(Entry.second)) @@ -321,12 +321,12 @@ if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased) addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased); if (Type == SecFuncMetadata && - (FunctionSamples::ProfileIsCSFlat || FunctionSamples::ProfileIsCSNested)) + (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined)) addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute); - if (Type == SecProfSummary && FunctionSamples::ProfileIsCSFlat) + if (Type == SecProfSummary && FunctionSamples::ProfileIsCS) addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext); - if (Type == SecProfSummary && FunctionSamples::ProfileIsCSNested) - addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsCSNested); + if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined) + addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined); if (Type == SecProfSummary && FunctionSamples::ProfileIsFS) addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator); @@ -468,7 +468,7 @@ /// it needs to be parsed by the SampleProfileReaderText class. std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; - if (FunctionSamples::ProfileIsCSFlat) + if (FunctionSamples::ProfileIsCS) OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples(); else OS << S.getName() << ":" << S.getTotalSamples(); @@ -868,7 +868,7 @@ std::unique_ptr Writer; // Currently only Text and Extended Binary format are supported for CSSPGO. - if ((FunctionSamples::ProfileIsCSFlat || + if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) && (Format == SPF_Binary || Format == SPF_Compact_Binary)) return sampleprof_error::unsupported_writing_format; diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -488,7 +488,7 @@ std::unique_ptr ContextTracker; /// Flag indicating whether input profile is context-sensitive - bool ProfileIsCSFlat = false; + bool ProfileIsCS = false; /// Flag indicating which LTO/ThinLTO phase the pass is invoked in. /// @@ -605,7 +605,7 @@ // call instruction should have 0 count. // For CS profile, the callsite count of previously inlined callees is // populated with the entry count of the callees. - if (!ProfileIsCSFlat) + if (!ProfileIsCS) if (const auto *CB = dyn_cast(&Inst)) if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) return 0; @@ -644,7 +644,7 @@ // call instruction should have 0 count. // For CS profile, the callsite count of previously inlined callees is // populated with the entry count of the callees. - if (!ProfileIsCSFlat) + if (!ProfileIsCS) if (const auto *CB = dyn_cast(&Inst)) if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) return 0; @@ -698,7 +698,7 @@ if (Function *Callee = Inst.getCalledFunction()) CalleeName = Callee->getName(); - if (ProfileIsCSFlat) + if (ProfileIsCS) return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName); const FunctionSamples *FS = findFunctionSamples(Inst); @@ -730,7 +730,7 @@ FunctionSamples::getGUID(R->getName()); }; - if (ProfileIsCSFlat) { + if (ProfileIsCS) { auto CalleeSamples = ContextTracker->getIndirectCalleeContextSamplesFor(DIL); if (CalleeSamples.empty()) @@ -783,7 +783,7 @@ auto it = DILocation2SampleMap.try_emplace(DIL,nullptr); if (it.second) { - if (ProfileIsCSFlat) + if (ProfileIsCS) it.first->second = ContextTracker->getContextSamplesFor(DIL); else it.first->second = @@ -1057,7 +1057,7 @@ // For AutoFDO profile, retrieve candidate profiles by walking over // the nested inlinee profiles. - if (!ProfileIsCSFlat) { + if (!ProfileIsCS) { Samples->findInlinedFunctions(InlinedGUIDs, SymbolMap, Threshold); return; } @@ -1161,7 +1161,7 @@ assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && "GUIDToFuncNameMap has to be populated"); AllCandidates.push_back(CB); - if (FS->getEntrySamples() > 0 || ProfileIsCSFlat) + if (FS->getEntrySamples() > 0 || ProfileIsCS) LocalNotInlinedCallSites.try_emplace(CB, FS); if (callsiteIsHot(FS, PSI, ProfAccForSymsInList)) Hot = true; @@ -1224,7 +1224,7 @@ // For CS profile, profile for not inlined context will be merged when // base profile is being retrieved. - if (!FunctionSamples::ProfileIsCSFlat) + if (!FunctionSamples::ProfileIsCS) promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F); return Changed; } @@ -1272,7 +1272,7 @@ InlinedCallSites->push_back(I); } - if (ProfileIsCSFlat) + if (ProfileIsCS) ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples); ++NumCSInlined; @@ -1534,7 +1534,7 @@ // For CS profile, profile for not inlined context will be merged when // base profile is being retrieved. - if (!FunctionSamples::ProfileIsCSFlat) + if (!FunctionSamples::ProfileIsCS) promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F); return Changed; } @@ -1648,7 +1648,7 @@ // With CSSPGO all indirect call targets are counted torwards the // original indirect call site in the profile, including both // inlined and non-inlined targets. - if (!FunctionSamples::ProfileIsCSFlat) { + if (!FunctionSamples::ProfileIsCS) { if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) { for (const auto &NameFS : *M) @@ -1823,7 +1823,7 @@ std::unique_ptr SampleProfileLoader::buildProfiledCallGraph(CallGraph &CG) { std::unique_ptr ProfiledCG; - if (ProfileIsCSFlat) + if (ProfileIsCS) ProfiledCG = std::make_unique(*ContextTracker); else ProfiledCG = std::make_unique(Reader->getProfiles()); @@ -1869,7 +1869,7 @@ assert(&CG->getModule() == &M); if (UseProfiledCallGraph || - (ProfileIsCSFlat && !UseProfiledCallGraph.getNumOccurrences())) { + (ProfileIsCS && !UseProfiledCallGraph.getNumOccurrences())) { // Use profiled call edges to augment the top-down order. There are cases // that the top-down order computed based on the static call graph doesn't // reflect real execution order. For example @@ -2002,7 +2002,7 @@ } // Apply tweaks if context-sensitive or probe-based profile is available. - if (Reader->profileIsCSFlat() || Reader->profileIsCSNested() || + if (Reader->profileIsCS() || Reader->profileIsPreInlined() || Reader->profileIsProbeBased()) { if (!UseIterativeBFIInference.getNumOccurrences()) UseIterativeBFIInference = true; @@ -2012,8 +2012,8 @@ EnableExtTspBlockPlacement = true; } - if (Reader->profileIsCSFlat() || Reader->profileIsCSNested()) { - ProfileIsCSFlat = Reader->profileIsCSFlat(); + if (Reader->profileIsCS() || Reader->profileIsPreInlined()) { + ProfileIsCS = Reader->profileIsCS(); // Enable priority-base inliner and size inline by default for CSSPGO. if (!ProfileSizeInline.getNumOccurrences()) ProfileSizeInline = true; @@ -2028,7 +2028,7 @@ if (!AllowRecursiveInline.getNumOccurrences()) AllowRecursiveInline = true; - if (FunctionSamples::ProfileIsCSFlat) { + if (FunctionSamples::ProfileIsCS) { // Tracker for profiles under different context ContextTracker = std::make_unique( Reader->getProfiles(), &GUIDToFuncNameMap); @@ -2110,7 +2110,7 @@ } // Account for cold calls not inlined.... - if (!ProfileIsCSFlat) + if (!ProfileIsCS) for (const std::pair &pair : notInlinedCallInfo) updateProfileCallee(pair.first, pair.second.entryCount); @@ -2186,7 +2186,7 @@ ORE = OwnedORE.get(); } - if (ProfileIsCSFlat) + if (ProfileIsCS) Samples = ContextTracker->getBaseSamplesFor(F); else Samples = Reader->getSamplesFor(F); diff --git a/llvm/test/tools/llvm-profdata/cs-sample-nested-profile.test b/llvm/test/tools/llvm-profdata/cs-sample-nested-profile.test --- a/llvm/test/tools/llvm-profdata/cs-sample-nested-profile.test +++ b/llvm/test/tools/llvm-profdata/cs-sample-nested-profile.test @@ -156,7 +156,7 @@ ; PROBE-NEXT: !CFGChecksum: 563022570642068 -; PREINLINE: ProfileSummarySection {{.*}} Flags: {context-nested} +; PREINLINE: ProfileSummarySection {{.*}} Flags: {preInlined} ; SUMMARY: Total functions: 4 diff --git a/llvm/test/tools/llvm-profgen/cs-preinline.test b/llvm/test/tools/llvm-profgen/cs-preinline.test --- a/llvm/test/tools/llvm-profgen/cs-preinline.test +++ b/llvm/test/tools/llvm-profgen/cs-preinline.test @@ -1,23 +1,36 @@ ; Test default llvm-profgen with preinline off -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=0 --gen-cs-nested-profile=0 --output=%t -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-DEFAULT +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=0 --gen-cs-nested-profile=0 --output=%t1 +; RUN: FileCheck %s --input-file %t1 --check-prefix=CHECK-DEFAULT ; Test llvm-profgen with preinliner on will merge not inlinable profile into base profile. -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-PREINL +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t2 +; RUN: FileCheck %s --input-file %t2 --check-prefix=CHECK-PREINL ; Test preinliner threshold that prevents all possible inlining and merges everything into base profile. -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=0 --output=%t -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-NO-PREINL +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=0 --output=%t3 +; RUN: FileCheck %s --input-file %t3 --check-prefix=CHECK-NO-PREINL ; Test cold profile trimming. Only base profiles should be dropped. -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t --trim-cold-profile=1 --profile-summary-hot-count=400 +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t4 --trim-cold-profile=1 --profile-summary-hot-count=400 -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-TRIM +; RUN: FileCheck %s --input-file %t4 --check-prefix=CHECK-TRIM ; Test llvm-profgen with preinliner on will merge not inlinable profile into base profile. -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t --gen-cs-nested-profile=1 -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-PREINL-NEST +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t5 --gen-cs-nested-profile=1 +; RUN: FileCheck %s --input-file %t5 --check-prefix=CHECK-PREINL-NEST + +; Test preInlined flat is set for extbinary profiles with preinliner on. +; RUN: llvm-profgen --format=extbinary --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --gen-cs-nested-profile=0 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t2.prof +; RUN: llvm-profdata show --sample -show-sec-info-only %t2.prof | FileCheck %s -check-prefix=CHECK-PREINL-FLAG +; RUN: llvm-profgen --format=extbinary --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --csspgo-preinliner=1 --sample-profile-hot-inline-threshold=3000 --sample-profile-cold-inline-threshold=45 --output=%t5.prof --gen-cs-nested-profile=1 +; RUN: llvm-profdata show --sample -show-sec-info-only %t5.prof | FileCheck %s -check-prefix=CHECK-PREINL-FLAG + +; Test preInlined flat is set when converted from a text profile that has "ShouldInline" contexts. +; RUN: llvm-profdata merge --sample %t2 -o %t2.prof --extbinary +; RUN: llvm-profdata show --sample -show-sec-info-only %t2.prof | FileCheck %s -check-prefix=CHECK-PREINL-FLAG +; RUN: llvm-profdata merge --sample %t5 -o %t5.prof --extbinary +; RUN: llvm-profdata show --sample -show-sec-info-only %t5.prof | FileCheck %s -check-prefix=CHECK-PREINL-FLAG + ; CHECK-DEFAULT: [main:1 @ foo]:309:0 ; CHECK-DEFAULT-NEXT: 2.1: 14 @@ -66,3 +79,6 @@ ; CHECK-PREINL-NEST-NEXT: 3.1: bar:84 ; CHECK-PREINL-NEST-NEXT: 1: 14 ; CHECK-PREINL-NEST-NEXT: !Attributes: 7 + + +; CHECK-PREINL-FLAG: ProfileSummarySection {{.*}} Flags: {{{.*}}preInlined} diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -741,7 +741,7 @@ LLVMContext Context; sampleprof::ProfileSymbolList WriterList; Optional ProfileIsProbeBased; - Optional ProfileIsCSFlat; + Optional ProfileIsCS; for (const auto &Input : Inputs) { auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context, FSDiscriminatorPassOption); @@ -768,10 +768,10 @@ exitWithError( "cannot merge probe-based profile with non-probe-based profile"); ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased; - if (ProfileIsCSFlat.hasValue() && - ProfileIsCSFlat != FunctionSamples::ProfileIsCSFlat) + if (ProfileIsCS.hasValue() && + ProfileIsCS != FunctionSamples::ProfileIsCS) exitWithError("cannot merge CS profile with non-CS profile"); - ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat; + ProfileIsCS = FunctionSamples::ProfileIsCS; for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end(); I != E; ++I) { sampleprof_error Result = sampleprof_error::success; @@ -794,7 +794,7 @@ WriterList.merge(*ReaderList); } - if (ProfileIsCSFlat && (SampleMergeColdContext || SampleTrimColdContext)) { + if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) { // Use threshold calculated from profile summary unless specified. SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); auto Summary = Builder.computeSummaryForProfiles(ProfileMap); @@ -809,10 +809,10 @@ SampleMergeColdContext, SampleColdContextFrameDepth, false); } - if (ProfileIsCSFlat && GenCSNestedProfile) { + if (ProfileIsCS && GenCSNestedProfile) { CSProfileConverter CSConverter(ProfileMap); CSConverter.convertProfiles(); - ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat = false; + ProfileIsCS = FunctionSamples::ProfileIsCS = false; } auto WriterOrErr = @@ -1972,7 +1972,7 @@ if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased()) exitWithError( "cannot compare probe-based profile with non-probe-based profile"); - if (BaseReader->profileIsCSFlat() != TestReader->profileIsCSFlat()) + if (BaseReader->profileIsCS() != TestReader->profileIsCS()) exitWithError("cannot compare CS profile with non-CS profile"); // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp --- a/llvm/tools/llvm-profgen/CSPreInliner.cpp +++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp @@ -195,6 +195,9 @@ ContextTracker.markContextSamplesInlined(Candidate.CalleeSamples); Candidate.CalleeSamples->getContext().setAttribute( ContextShouldBeInlined); + // Contexts coming with a `ContextShouldBeInlined` attribute means this + // is a preinliner-computed profile. + FunctionSamples::ProfileIsPreInlined = true; FuncFinalSize += Candidate.SizeCost; getInlineCandidates(CQueue, Candidate.CalleeSamples); } else { diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h --- a/llvm/tools/llvm-profgen/PerfReader.h +++ b/llvm/tools/llvm-profgen/PerfReader.h @@ -538,14 +538,14 @@ const ContextSampleCounterMap &getSampleCounters() const { return SampleCounters; } - bool profileIsCSFlat() { return ProfileIsCSFlat; } + bool profileIsCS() { return ProfileIsCS; } protected: ProfiledBinary *Binary = nullptr; StringRef PerfTraceFile; ContextSampleCounterMap SampleCounters; - bool ProfileIsCSFlat = false; + bool ProfileIsCS = false; uint64_t NumTotalSample = 0; uint64_t NumLeafExternalFrame = 0; diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp --- a/llvm/tools/llvm-profgen/PerfReader.cpp +++ b/llvm/tools/llvm-profgen/PerfReader.cpp @@ -793,7 +793,7 @@ for (auto &CI : OrderedCounters) { uint32_t Indent = 0; - if (ProfileIsCSFlat) { + if (ProfileIsCS) { // Context string key OS << "[" << CI.first << "]\n"; Indent = 2; @@ -880,7 +880,7 @@ StringRef Line = TraceIt.getCurrentLine(); // Read context stack for CS profile. if (Line.startswith("[")) { - ProfileIsCSFlat = true; + ProfileIsCS = true; auto I = ContextStrSet.insert(Line.str()); SampleContext::createCtxVectorFromStr(*I.first, Key->Context); TraceIt.advance(); @@ -1097,8 +1097,8 @@ } void HybridPerfReader::generateUnsymbolizedProfile() { - ProfileIsCSFlat = !IgnoreStackSamples; - if (ProfileIsCSFlat) + ProfileIsCS = !IgnoreStackSamples; + if (ProfileIsCS) unwindSamples(); else PerfScriptReader::generateUnsymbolizedProfile(); diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -38,7 +38,7 @@ virtual ~ProfileGeneratorBase() = default; static std::unique_ptr create(ProfiledBinary *Binary, const ContextSampleCounterMap &SampleCounters, - bool ProfileIsCSFlat); + bool ProfileIsCS); virtual void generateProfile() = 0; void write(); diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -110,9 +110,9 @@ std::unique_ptr ProfileGeneratorBase::create(ProfiledBinary *Binary, const ContextSampleCounterMap &SampleCounters, - bool ProfileIsCSFlat) { + bool ProfileIsCS) { std::unique_ptr Generator; - if (ProfileIsCSFlat) { + if (ProfileIsCS) { if (Binary->useFSDiscriminator()) exitWithError("FS discriminator is not supported in CS profile."); Generator.reset(new CSProfileGenerator(Binary, SampleCounters)); @@ -683,7 +683,7 @@ } void CSProfileGenerator::generateProfile() { - FunctionSamples::ProfileIsCSFlat = true; + FunctionSamples::ProfileIsCS = true; collectProfiledFunctions(); @@ -888,8 +888,7 @@ if (GenCSNestedProfile) { CSProfileConverter CSConverter(ProfileMap); CSConverter.convertProfiles(); - FunctionSamples::ProfileIsCSFlat = false; - FunctionSamples::ProfileIsCSNested = EnableCSPreInliner; + FunctionSamples::ProfileIsCS = false; } } diff --git a/llvm/tools/llvm-profgen/llvm-profgen.cpp b/llvm/tools/llvm-profgen/llvm-profgen.cpp --- a/llvm/tools/llvm-profgen/llvm-profgen.cpp +++ b/llvm/tools/llvm-profgen/llvm-profgen.cpp @@ -157,7 +157,7 @@ std::unique_ptr Generator = ProfileGeneratorBase::create(Binary.get(), Reader->getSampleCounters(), - Reader->profileIsCSFlat()); + Reader->profileIsCS()); Generator->generateProfile(); Generator->write();