diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -723,6 +723,8 @@ // memory, which is *very* significant for large profiles. using FunctionSamplesMap = std::map>; using CallsiteSampleMap = std::map; +using LocToLocMap = + std::unordered_map; /// Representation of the samples collected for a function. /// @@ -837,12 +839,26 @@ } } + // Query the stale profile matching results and remap the location. + // There is no remapping if the profile is not stale or the matching gives the + // same location. + const LineLocation &remapLoc(const LineLocation &Loc) const { + if (!StaleProfileMappings) + return Loc; + auto NewLoc = StaleProfileMappings->find(Loc); + if (NewLoc != StaleProfileMappings->end()) + return NewLoc->second; + else + return Loc; + } + /// Return the number of samples collected at the given location. /// Each location is specified by \p LineOffset and \p Discriminator. /// If the location is not found in profile, return error. ErrorOr findSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const { - const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); + const auto &ret = + BodySamples.find(remapLoc(LineLocation(LineOffset, Discriminator))); if (ret == BodySamples.end()) return std::error_code(); return ret->second.getSamples(); @@ -853,7 +869,8 @@ /// If the location is not found in profile, return error. ErrorOr findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const { - const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); + const auto &ret = + BodySamples.find(remapLoc(LineLocation(LineOffset, Discriminator))); if (ret == BodySamples.end()) return std::error_code(); return ret->second.getCallTargets(); @@ -863,7 +880,7 @@ /// CallSite. If the location is not found in profile, return error. ErrorOr findCallTargetMapAt(const LineLocation &CallSite) const { - const auto &Ret = BodySamples.find(CallSite); + const auto &Ret = BodySamples.find(remapLoc(CallSite)); if (Ret == BodySamples.end()) return std::error_code(); return Ret->second.getCallTargets(); @@ -871,13 +888,13 @@ /// Return the function samples at the given callsite location. FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) { - return CallsiteSamples[Loc]; + return CallsiteSamples[remapLoc(Loc)]; } /// Returns the FunctionSamplesMap at the given \p Loc. const FunctionSamplesMap * findFunctionSamplesMapAt(const LineLocation &Loc) const { - auto iter = CallsiteSamples.find(Loc); + auto iter = CallsiteSamples.find(remapLoc(Loc)); if (iter == CallsiteSamples.end()) return nullptr; return &iter->second; @@ -1041,6 +1058,8 @@ uint64_t getFunctionHash() const { return FunctionHash; } + void setStaleProfileMappings(LocToLocMap *M) { StaleProfileMappings = M; } + /// Return the canonical name for a function, taking into account /// suffix elision policy attributes. static StringRef getCanonicalFnName(const Function &F) { @@ -1224,6 +1243,25 @@ /// in the call to bar() at line offset 1, the other for all the samples /// collected in the call to baz() at line offset 8. CallsiteSampleMap CallsiteSamples; + + /// Location mappings generated by stale profile matching. + /// + /// Each entry is a mapping from the location on current build to the matched + /// location in the "stale" profile. For example: + /// Profiled source code: + /// void foo() { + /// 1 bar(); + /// } + /// + /// Current source code: + /// void foo() { + /// 1 // Code change + /// 2 bar(); + /// } + /// Supposing the stale profile matching algorithm generated the mapping [2 -> + /// 1], the profile query using the location of bar on the IR which is 2 will + /// be remapped to 1 and find the location of bar in the profile. + LocToLocMap *StaleProfileMappings = nullptr; }; raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -291,7 +291,7 @@ std::string CalleeGUID; CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID); - auto iter = CallsiteSamples.find(Loc); + auto iter = CallsiteSamples.find(remapLoc(Loc)); if (iter == CallsiteSamples.end()) return nullptr; auto FS = iter->second.find(CalleeName); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -130,6 +130,11 @@ "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden); +static cl::opt SampleProfileMatching( + "sample-profile-matching", cl::Hidden, cl::init(false), + cl::desc("Run stale profile matching algorithm and use the remapped " + "location for sample profile query.")); + static cl::opt ReportProfileStaleness( "report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics.")); @@ -440,6 +445,10 @@ SampleProfileReader &Reader; const PseudoProbeManager *ProbeManager; SampleProfileMap FlattenedProfiles; + // The output of stale profile matching. For each function, the matching + // algorithm generates a map, each entry is a mapping meaning the location + // of current build is matched to the location in the profile. + StringMap FuncToMatchingsMap; // Profile mismatching statstics. uint64_t TotalProfiledCallsites = 0; @@ -469,8 +478,14 @@ return nullptr; } - void detectProfileMismatch(); - void detectProfileMismatch(const Function &F, const FunctionSamples &FS); + void runOnModule(); + void runOnFunction(const Function &F, const FunctionSamples &FS); + void setStaleProfileMappings(); + void setStaleProfileMappings(FunctionSamples &FS); + void + detectProfileMismatch(const FunctionSamples &FS, + const std::unordered_set + &MatchedCallsiteLocs); }; /// Sample profile pass. @@ -2073,7 +2088,8 @@ } } - if (ReportProfileStaleness || PersistProfileStaleness) { + if (ReportProfileStaleness || PersistProfileStaleness || + SampleProfileMatching) { MatchingManager = std::make_unique(M, *Reader, ProbeManager.get()); } @@ -2081,8 +2097,53 @@ return true; } -void SampleProfileMatcher::detectProfileMismatch(const Function &F, - const FunctionSamples &FS) { +void SampleProfileMatcher::detectProfileMismatch( + const FunctionSamples &FS, + const std::unordered_set + &MatchedCallsiteLocs) { + + auto isInvalidLineOffset = [](uint32_t LineOffset) { + return LineOffset & 0x8000; + }; + + // Check if there are any callsites in the profile that does not match to any + // IR callsites, those callsite samples will be discarded. + for (auto &I : FS.getBodySamples()) { + const LineLocation &Loc = I.first; + if (isInvalidLineOffset(Loc.LineOffset)) + continue; + + uint64_t Count = I.second.getSamples(); + if (!I.second.getCallTargets().empty()) { + TotalCallsiteSamples += Count; + TotalProfiledCallsites++; + if (!MatchedCallsiteLocs.count(Loc)) { + MismatchedCallsiteSamples += Count; + NumMismatchedCallsites++; + } + } + } + + for (auto &I : FS.getCallsiteSamples()) { + const LineLocation &Loc = I.first; + if (isInvalidLineOffset(Loc.LineOffset)) + continue; + + uint64_t Count = 0; + for (auto &FM : I.second) { + Count += FM.second.getHeadSamplesEstimate(); + } + TotalCallsiteSamples += Count; + TotalProfiledCallsites++; + if (!MatchedCallsiteLocs.count(Loc)) { + MismatchedCallsiteSamples += Count; + NumMismatchedCallsites++; + } + } +} + +void SampleProfileMatcher::runOnFunction(const Function &F, + const FunctionSamples &FS) { if (FunctionSamples::ProfileIsProbeBased) { uint64_t Count = FS.getTotalSamples(); TotalFuncHashSamples += Count; @@ -2132,47 +2193,11 @@ } } - auto isInvalidLineOffset = [](uint32_t LineOffset) { - return LineOffset & 0x8000; - }; - - // Check if there are any callsites in the profile that does not match to any - // IR callsites, those callsite samples will be discarded. - for (auto &I : FS.getBodySamples()) { - const LineLocation &Loc = I.first; - if (isInvalidLineOffset(Loc.LineOffset)) - continue; - - uint64_t Count = I.second.getSamples(); - if (!I.second.getCallTargets().empty()) { - TotalCallsiteSamples += Count; - TotalProfiledCallsites++; - if (!MatchedCallsiteLocs.count(Loc)) { - MismatchedCallsiteSamples += Count; - NumMismatchedCallsites++; - } - } - } - - for (auto &I : FS.getCallsiteSamples()) { - const LineLocation &Loc = I.first; - if (isInvalidLineOffset(Loc.LineOffset)) - continue; - - uint64_t Count = 0; - for (auto &FM : I.second) { - Count += FM.second.getHeadSamplesEstimate(); - } - TotalCallsiteSamples += Count; - TotalProfiledCallsites++; - if (!MatchedCallsiteLocs.count(Loc)) { - MismatchedCallsiteSamples += Count; - NumMismatchedCallsites++; - } - } + // Detect profile mismatch for profile staleness metrics report. + detectProfileMismatch(FS, MatchedCallsiteLocs); } -void SampleProfileMatcher::detectProfileMismatch() { +void SampleProfileMatcher::runOnModule() { for (auto &F : M) { if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile")) continue; @@ -2183,9 +2208,12 @@ FS = Reader.getSamplesFor(F); if (!FS) continue; - detectProfileMismatch(F, *FS); + runOnFunction(F, *FS); } + if (SampleProfileMatching) + setStaleProfileMappings(); + if (ReportProfileStaleness) { if (FunctionSamples::ProfileIsProbeBased) { errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")" @@ -2227,6 +2255,27 @@ } } +void SampleProfileMatcher::setStaleProfileMappings(FunctionSamples &FS) { + const auto ProfileMappings = FuncToMatchingsMap.find(FS.getName()); + if (ProfileMappings != FuncToMatchingsMap.end()) { + FS.setStaleProfileMappings(&(ProfileMappings->second)); + } + + for (auto &Inlinees : FS.getCallsiteSamples()) { + for (auto FS : Inlinees.second) { + setStaleProfileMappings(FS.second); + } + } +} + +// Use a central place to distribute the matching results. Outlined and inlined +// profile with the function name will be set to the same pointer. +void SampleProfileMatcher::setStaleProfileMappings() { + for (auto &I : Reader.getProfiles()) { + setStaleProfileMappings(I.second); + } +} + bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummaryInfo *_PSI, LazyCallGraph &CG) { @@ -2272,8 +2321,10 @@ assert(SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"); - if (ReportProfileStaleness || PersistProfileStaleness) - MatchingManager->detectProfileMismatch(); + if (ReportProfileStaleness || PersistProfileStaleness || + SampleProfileMatching) { + MatchingManager->runOnModule(); + } bool retval = false; for (auto *F : buildFunctionOrder(M, CG)) {