diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -723,6 +723,8 @@ // memory, which is *very* significant for large profiles. using FunctionSamplesMap = std::map>; using CallsiteSampleMap = std::map; +using LocToLocMap = + std::unordered_map; /// Representation of the samples collected for a function. /// @@ -837,12 +839,26 @@ } } + // Query the stale profile matching results and remap the location. + const LineLocation &mapIRLocToProfileLoc(const LineLocation &IRLoc) const { + // There is no remapping if the profile is not stale or the matching gives + // the same location. + if (!IRToProfileLocationMap) + return IRLoc; + const auto &ProfileLoc = IRToProfileLocationMap->find(IRLoc); + if (ProfileLoc != IRToProfileLocationMap->end()) + return ProfileLoc->second; + else + return IRLoc; + } + /// Return the number of samples collected at the given location. /// Each location is specified by \p LineOffset and \p Discriminator. /// If the location is not found in profile, return error. ErrorOr findSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const { - const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); + const auto &ret = BodySamples.find( + mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator))); if (ret == BodySamples.end()) return std::error_code(); return ret->second.getSamples(); @@ -853,7 +869,8 @@ /// If the location is not found in profile, return error. ErrorOr findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const { - const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); + const auto &ret = BodySamples.find( + mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator))); if (ret == BodySamples.end()) return std::error_code(); return ret->second.getCallTargets(); @@ -863,7 +880,7 @@ /// CallSite. If the location is not found in profile, return error. ErrorOr findCallTargetMapAt(const LineLocation &CallSite) const { - const auto &Ret = BodySamples.find(CallSite); + const auto &Ret = BodySamples.find(mapIRLocToProfileLoc(CallSite)); if (Ret == BodySamples.end()) return std::error_code(); return Ret->second.getCallTargets(); @@ -871,13 +888,13 @@ /// Return the function samples at the given callsite location. FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) { - return CallsiteSamples[Loc]; + return CallsiteSamples[mapIRLocToProfileLoc(Loc)]; } /// Returns the FunctionSamplesMap at the given \p Loc. const FunctionSamplesMap * findFunctionSamplesMapAt(const LineLocation &Loc) const { - auto iter = CallsiteSamples.find(Loc); + auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc)); if (iter == CallsiteSamples.end()) return nullptr; return &iter->second; @@ -1041,6 +1058,11 @@ uint64_t getFunctionHash() const { return FunctionHash; } + void setIRToProfileLocationMap(const LocToLocMap *LTLM) { + assert(IRToProfileLocationMap == nullptr && "this should be set only once"); + IRToProfileLocationMap = LTLM; + } + /// Return the canonical name for a function, taking into account /// suffix elision policy attributes. static StringRef getCanonicalFnName(const Function &F) { @@ -1224,6 +1246,25 @@ /// in the call to bar() at line offset 1, the other for all the samples /// collected in the call to baz() at line offset 8. CallsiteSampleMap CallsiteSamples; + + /// IR to profile location map generated by stale profile matching. + /// + /// Each entry is a mapping from the location on current build to the matched + /// location in the "stale" profile. For example: + /// Profiled source code: + /// void foo() { + /// 1 bar(); + /// } + /// + /// Current source code: + /// void foo() { + /// 1 // Code change + /// 2 bar(); + /// } + /// Supposing the stale profile matching algorithm generated the mapping [2 -> + /// 1], the profile query using the location of bar on the IR which is 2 will + /// be remapped to 1 and find the location of bar in the profile. + const LocToLocMap *IRToProfileLocationMap = nullptr; }; raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -291,7 +291,7 @@ std::string CalleeGUID; CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID); - auto iter = CallsiteSamples.find(Loc); + auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc)); if (iter == CallsiteSamples.end()) return nullptr; auto FS = iter->second.find(CalleeName); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -130,6 +130,11 @@ "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden); +static cl::opt SalvageStaleProfile( + "salvage-stale-profile", cl::Hidden, cl::init(false), + cl::desc("Salvage stale profile by fuzzy matching and use the remapped " + "location for sample profile query.")); + static cl::opt ReportProfileStaleness( "report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics.")); @@ -460,7 +465,9 @@ FunctionSamples::ProfileIsCS); } } + void runOnModule(); +private: FunctionSamples *getFlattenedSamplesFor(const Function &F) { StringRef CanonFName = FunctionSamples::getCanonicalFnName(F); auto It = FlattenedProfiles.find(CanonFName); @@ -468,9 +475,11 @@ return &It->second; return nullptr; } - - void detectProfileMismatch(); - void detectProfileMismatch(const Function &F, const FunctionSamples &FS); + void runOnFunction(const Function &F, const FunctionSamples &FS); + void countProfileMismatches( + const FunctionSamples &FS, + const std::unordered_set + &MatchedCallsiteLocs); }; /// Sample profile pass. @@ -2073,7 +2082,8 @@ } } - if (ReportProfileStaleness || PersistProfileStaleness) { + if (ReportProfileStaleness || PersistProfileStaleness || + SalvageStaleProfile) { MatchingManager = std::make_unique(M, *Reader, ProbeManager.get()); } @@ -2081,8 +2091,53 @@ return true; } -void SampleProfileMatcher::detectProfileMismatch(const Function &F, - const FunctionSamples &FS) { +void SampleProfileMatcher::countProfileMismatches( + const FunctionSamples &FS, + const std::unordered_set + &MatchedCallsiteLocs) { + + auto isInvalidLineOffset = [](uint32_t LineOffset) { + return LineOffset & 0x8000; + }; + + // Check if there are any callsites in the profile that does not match to any + // IR callsites, those callsite samples will be discarded. + for (auto &I : FS.getBodySamples()) { + const LineLocation &Loc = I.first; + if (isInvalidLineOffset(Loc.LineOffset)) + continue; + + uint64_t Count = I.second.getSamples(); + if (!I.second.getCallTargets().empty()) { + TotalCallsiteSamples += Count; + TotalProfiledCallsites++; + if (!MatchedCallsiteLocs.count(Loc)) { + MismatchedCallsiteSamples += Count; + NumMismatchedCallsites++; + } + } + } + + for (auto &I : FS.getCallsiteSamples()) { + const LineLocation &Loc = I.first; + if (isInvalidLineOffset(Loc.LineOffset)) + continue; + + uint64_t Count = 0; + for (auto &FM : I.second) { + Count += FM.second.getHeadSamplesEstimate(); + } + TotalCallsiteSamples += Count; + TotalProfiledCallsites++; + if (!MatchedCallsiteLocs.count(Loc)) { + MismatchedCallsiteSamples += Count; + NumMismatchedCallsites++; + } + } +} + +void SampleProfileMatcher::runOnFunction(const Function &F, + const FunctionSamples &FS) { if (FunctionSamples::ProfileIsProbeBased) { uint64_t Count = FS.getTotalSamples(); TotalFuncHashSamples += Count; @@ -2132,47 +2187,12 @@ } } - auto isInvalidLineOffset = [](uint32_t LineOffset) { - return LineOffset & 0x8000; - }; - - // Check if there are any callsites in the profile that does not match to any - // IR callsites, those callsite samples will be discarded. - for (auto &I : FS.getBodySamples()) { - const LineLocation &Loc = I.first; - if (isInvalidLineOffset(Loc.LineOffset)) - continue; - - uint64_t Count = I.second.getSamples(); - if (!I.second.getCallTargets().empty()) { - TotalCallsiteSamples += Count; - TotalProfiledCallsites++; - if (!MatchedCallsiteLocs.count(Loc)) { - MismatchedCallsiteSamples += Count; - NumMismatchedCallsites++; - } - } - } - - for (auto &I : FS.getCallsiteSamples()) { - const LineLocation &Loc = I.first; - if (isInvalidLineOffset(Loc.LineOffset)) - continue; - - uint64_t Count = 0; - for (auto &FM : I.second) { - Count += FM.second.getHeadSamplesEstimate(); - } - TotalCallsiteSamples += Count; - TotalProfiledCallsites++; - if (!MatchedCallsiteLocs.count(Loc)) { - MismatchedCallsiteSamples += Count; - NumMismatchedCallsites++; - } - } + // Detect profile mismatch for profile staleness metrics report. + if (ReportProfileStaleness || PersistProfileStaleness) + countProfileMismatches(FS, MatchedCallsiteLocs); } -void SampleProfileMatcher::detectProfileMismatch() { +void SampleProfileMatcher::runOnModule() { for (auto &F : M) { if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile")) continue; @@ -2183,7 +2203,7 @@ FS = Reader.getSamplesFor(F); if (!FS) continue; - detectProfileMismatch(F, *FS); + runOnFunction(F, *FS); } if (ReportProfileStaleness) { @@ -2272,8 +2292,10 @@ assert(SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"); - if (ReportProfileStaleness || PersistProfileStaleness) - MatchingManager->detectProfileMismatch(); + if (ReportProfileStaleness || PersistProfileStaleness || + SalvageStaleProfile) { + MatchingManager->runOnModule(); + } bool retval = false; for (auto *F : buildFunctionOrder(M, CG)) {