diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h --- a/bolt/include/bolt/Profile/YAMLProfileReader.h +++ b/bolt/include/bolt/Profile/YAMLProfileReader.h @@ -46,48 +46,53 @@ bool NormalizeByCalls{false}; /// Binary profile in YAML format. - yaml::bolt::BinaryProfile YamlBP; + std::vector YamlBPs; /// Map a function ID from a YAML profile to a BinaryFunction object. - std::vector YamlProfileToFunction; + std::vector> YamlProfileToFunction; /// To keep track of functions that have a matched profile before the profile /// is attributed. - std::unordered_set ProfiledFunctions; + std::vector> ProfiledFunctions; /// For LTO symbol resolution. /// Map a common LTO prefix to a list of YAML profiles matching the prefix. - StringMap> LTOCommonNameMap; + std::vector>> + LTOCommonNameMap; /// Map a common LTO prefix to a set of binary functions. - StringMap> + std::vector>> LTOCommonNameFunctionMap; /// Strict matching of a name in a profile to its contents. - StringMap ProfileNameToProfile; + std::vector> + ProfileNameToProfile; /// Populate \p Function profile with the one supplied in YAML format. bool parseFunctionProfile(BinaryFunction &Function, - const yaml::bolt::BinaryFunctionProfile &YamlBF); + const yaml::bolt::BinaryFunctionProfile &YamlBF, + const yaml::bolt::BinaryProfile &YamlBP, + unsigned Epoch); /// Initialize maps for profile matching. - void buildNameMaps(std::map &Functions); + void buildNameMaps(std::map &Functions, + unsigned Epoch); /// Update matched YAML -> BinaryFunction pair. void matchProfileToFunction(yaml::bolt::BinaryFunctionProfile &YamlBF, - BinaryFunction &BF) { - if (YamlBF.Id >= YamlProfileToFunction.size()) - YamlProfileToFunction.resize(YamlBF.Id + 1); - YamlProfileToFunction[YamlBF.Id] = &BF; + BinaryFunction &BF, unsigned Epoch) { + if (YamlBF.Id >= YamlProfileToFunction[Epoch].size()) + YamlProfileToFunction[Epoch].resize(YamlBF.Id + 1); + YamlProfileToFunction[Epoch][YamlBF.Id] = &BF; YamlBF.Used = true; - assert(!ProfiledFunctions.count(&BF) && + assert(!ProfiledFunctions[Epoch].count(&BF) && "function already has an assigned profile"); - ProfiledFunctions.emplace(&BF); + ProfiledFunctions[Epoch].emplace(&BF); } /// Check if the profile uses an event with a given \p Name. - bool usesEvent(StringRef Name) const; + bool usesEvent(StringRef Name, const yaml::bolt::BinaryProfile &YamlBP) const; }; } // namespace bolt diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp --- a/bolt/lib/Profile/YAMLProfileReader.cpp +++ b/bolt/lib/Profile/YAMLProfileReader.cpp @@ -42,36 +42,39 @@ } void YAMLProfileReader::buildNameMaps( - std::map &Functions) { + std::map &Functions, unsigned Epoch) { + yaml::bolt::BinaryProfile &YamlBP = YamlBPs[Epoch]; for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) { StringRef Name = YamlBF.Name; const size_t Pos = Name.find("(*"); if (Pos != StringRef::npos) Name = Name.substr(0, Pos); - ProfileNameToProfile[Name] = &YamlBF; + ProfileNameToProfile[Epoch][Name] = &YamlBF; if (const std::optional CommonName = getLTOCommonName(Name)) - LTOCommonNameMap[*CommonName].push_back(&YamlBF); + LTOCommonNameMap[Epoch][*CommonName].push_back(&YamlBF); } for (auto &BFI : Functions) { const BinaryFunction &Function = BFI.second; for (StringRef Name : Function.getNames()) if (const std::optional CommonName = getLTOCommonName(Name)) - LTOCommonNameFunctionMap[*CommonName].insert(&Function); + LTOCommonNameFunctionMap[Epoch][*CommonName].insert(&Function); } } bool YAMLProfileReader::hasLocalsWithFileName() const { - for (const StringMapEntry &KV : - ProfileNameToProfile) { - const StringRef &FuncName = KV.getKey(); - if (FuncName.count('/') == 2 && FuncName[0] != '/') - return true; - } - return false; + auto CheckFuncName = [](const StringRef &FuncName) { + return FuncName.count('/') == 2 && FuncName[0] != '/'; + }; + auto CheckMapKeys = + [&](const StringMap &Map) { + return llvm::any_of(Map.keys(), CheckFuncName); + }; + return llvm::any_of(ProfileNameToProfile, CheckMapKeys); } bool YAMLProfileReader::parseFunctionProfile( - BinaryFunction &BF, const yaml::bolt::BinaryFunctionProfile &YamlBF) { + BinaryFunction &BF, const yaml::bolt::BinaryFunctionProfile &YamlBF, + const yaml::bolt::BinaryProfile &YamlBP, unsigned Epoch) { BinaryContext &BC = BF.getBinaryContext(); bool ProfileMatched = true; @@ -81,7 +84,7 @@ uint64_t FunctionExecutionCount = 0; - BF.setExecutionCount(YamlBF.ExecCount); + BF.setExecutionCount(BF.getExecutionCount() + YamlBF.ExecCount); if (!opts::IgnoreHash && YamlBF.Hash != BF.computeHash(/*UseDFS=*/true)) { if (opts::Verbosity >= 1) @@ -111,28 +114,27 @@ // Basic samples profile (without LBR) does not have branches information // and needs a special processing. if (YamlBP.Header.Flags & BinaryFunction::PF_SAMPLE) { - if (!YamlBB.EventCount) { - BB.setExecutionCount(0); + if (!YamlBB.EventCount) continue; - } uint64_t NumSamples = YamlBB.EventCount * 1000; if (NormalizeByInsnCount && BB.getNumNonPseudos()) NumSamples /= BB.getNumNonPseudos(); else if (NormalizeByCalls) NumSamples /= BB.getNumCalls() + 1; - BB.setExecutionCount(NumSamples); + BB.setExecutionCount(BB.getExecutionCount() + NumSamples); if (BB.isEntryPoint()) FunctionExecutionCount += NumSamples; continue; } - BB.setExecutionCount(YamlBB.ExecCount); + BB.setExecutionCount(BB.getExecutionCount() + YamlBB.ExecCount); for (const yaml::bolt::CallSiteInfo &YamlCSI : YamlBB.CallSites) { - BinaryFunction *Callee = YamlCSI.DestId < YamlProfileToFunction.size() - ? YamlProfileToFunction[YamlCSI.DestId] - : nullptr; + BinaryFunction *Callee = + YamlCSI.DestId < YamlProfileToFunction[Epoch].size() + ? YamlProfileToFunction[Epoch][YamlCSI.DestId] + : nullptr; bool IsFunction = Callee ? true : false; MCSymbol *CalleeSymbol = nullptr; if (IsFunction) @@ -219,7 +221,7 @@ BB.setExecutionCount(0); if (YamlBP.Header.Flags & BinaryFunction::PF_SAMPLE) { - BF.setExecutionCount(FunctionExecutionCount); + BF.setExecutionCount(BF.getExecutionCount() + FunctionExecutionCount); estimateEdgeCounts(BF); } @@ -246,36 +248,42 @@ yaml::Input YamlInput(MB.get()->getBuffer()); // Consume YAML file. - YamlInput >> YamlBP; + YamlInput >> YamlBPs; if (YamlInput.error()) { errs() << "BOLT-ERROR: syntax error parsing profile in " << Filename << " : " << YamlInput.error().message() << '\n'; return errorCodeToError(YamlInput.error()); } - // Sanity check. - if (YamlBP.Header.Version != 1) - return make_error( - Twine("cannot read profile : unsupported version"), - inconvertibleErrorCode()); - - if (YamlBP.Header.EventNames.find(',') != StringRef::npos) - return make_error( - Twine("multiple events in profile are not supported"), - inconvertibleErrorCode()); - - // Match profile to function based on a function name. - buildNameMaps(BC.getBinaryFunctions()); - - // Preliminary assign function execution count. - for (auto &KV : BC.getBinaryFunctions()) { - BinaryFunction &BF = KV.second; - for (StringRef Name : BF.getNames()) { - auto PI = ProfileNameToProfile.find(Name); - if (PI != ProfileNameToProfile.end()) { - yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); - BF.setExecutionCount(YamlBF.ExecCount); - break; + ProfileNameToProfile.resize(YamlBPs.size()); + LTOCommonNameMap.resize(YamlBPs.size()); + LTOCommonNameFunctionMap.resize(YamlBPs.size()); + for (unsigned Epoch = 0, End = YamlBPs.size(); Epoch != End; ++Epoch) { + yaml::bolt::BinaryProfile YamlBP = YamlBPs[Epoch]; + // Sanity check. + if (YamlBP.Header.Version != 2) + return make_error( + Twine("cannot read profile : unsupported version"), + inconvertibleErrorCode()); + + if (YamlBP.Header.EventNames.find(',') != StringRef::npos) + return make_error( + Twine("multiple events in profile are not supported"), + inconvertibleErrorCode()); + + // Match profile to function based on a function name. + buildNameMaps(BC.getBinaryFunctions(), Epoch); + + // Preliminary assign function execution count. + for (auto &KV : BC.getBinaryFunctions()) { + BinaryFunction &BF = KV.second; + for (StringRef Name : BF.getNames()) { + auto PI = ProfileNameToProfile[Epoch].find(Name); + if (PI != ProfileNameToProfile[Epoch].end()) { + yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); + BF.setExecutionCount(BF.getExecutionCount() + YamlBF.ExecCount); + break; + } } } } @@ -285,11 +293,16 @@ bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) { for (StringRef Name : BF.getNames()) { - if (ProfileNameToProfile.find(Name) != ProfileNameToProfile.end()) - return true; - if (const std::optional CommonName = getLTOCommonName(Name)) { - if (LTOCommonNameMap.find(*CommonName) != LTOCommonNameMap.end()) + for (unsigned Epoch = 0, End = ProfileNameToProfile.size(); Epoch != End; + ++Epoch) { + if (ProfileNameToProfile[Epoch].find(Name) != + ProfileNameToProfile[Epoch].end()) return true; + if (const std::optional CommonName = getLTOCommonName(Name)) { + if (LTOCommonNameMap[Epoch].find(*CommonName) != + LTOCommonNameMap[Epoch].end()) + return true; + } } } @@ -297,113 +310,119 @@ } Error YAMLProfileReader::readProfile(BinaryContext &BC) { - YamlProfileToFunction.resize(YamlBP.Functions.size() + 1); - - auto profileMatches = [](const yaml::bolt::BinaryFunctionProfile &Profile, - BinaryFunction &BF) { - if (opts::IgnoreHash && Profile.NumBasicBlocks == BF.size()) - return true; - if (!opts::IgnoreHash && - Profile.Hash == static_cast(BF.getHash())) - return true; - return false; - }; - - // We have to do 2 passes since LTO introduces an ambiguity in function - // names. The first pass assigns profiles that match 100% by name and - // by hash. The second pass allows name ambiguity for LTO private functions. - for (auto &BFI : BC.getBinaryFunctions()) { - BinaryFunction &Function = BFI.second; + uint64_t NumUnused = 0; + YamlProfileToFunction.resize(YamlBPs.size()); + ProfiledFunctions.resize(YamlBPs.size()); + for (unsigned Epoch = 0, End = YamlBPs.size(); Epoch != End; ++Epoch) { + auto YamlBP = YamlBPs[Epoch]; + YamlProfileToFunction[Epoch].resize(YamlBP.Functions.size() + 1); + + auto profileMatches = [](const yaml::bolt::BinaryFunctionProfile &Profile, + BinaryFunction &BF) { + if (opts::IgnoreHash && Profile.NumBasicBlocks == BF.size()) + return true; + if (!opts::IgnoreHash && + Profile.Hash == static_cast(BF.getHash())) + return true; + return false; + }; + + // We have to do 2 passes since LTO introduces an ambiguity in function + // names. The first pass assigns profiles that match 100% by name and + // by hash. The second pass allows name ambiguity for LTO private functions. + for (auto &BFI : BC.getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; + + // Clear function call count that may have been set while pre-processing + // the profile. + Function.setExecutionCount(BinaryFunction::COUNT_NO_PROFILE); + + // Recompute hash once per function. + if (!opts::IgnoreHash) + Function.computeHash(/*UseDFS=*/true); + + for (StringRef FunctionName : Function.getNames()) { + auto PI = ProfileNameToProfile[Epoch].find(FunctionName); + if (PI == ProfileNameToProfile[Epoch].end()) + continue; - // Clear function call count that may have been set while pre-processing - // the profile. - Function.setExecutionCount(BinaryFunction::COUNT_NO_PROFILE); + yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); + if (profileMatches(YamlBF, Function)) + matchProfileToFunction(YamlBF, Function, Epoch); + } + } - // Recompute hash once per function. - if (!opts::IgnoreHash) - Function.computeHash(/*UseDFS=*/true); + for (auto &BFI : BC.getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; - for (StringRef FunctionName : Function.getNames()) { - auto PI = ProfileNameToProfile.find(FunctionName); - if (PI == ProfileNameToProfile.end()) + if (ProfiledFunctions[Epoch].count(&Function)) continue; - yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); - if (profileMatches(YamlBF, Function)) - matchProfileToFunction(YamlBF, Function); - } - } - - for (auto &BFI : BC.getBinaryFunctions()) { - BinaryFunction &Function = BFI.second; - - if (ProfiledFunctions.count(&Function)) - continue; + for (StringRef FunctionName : Function.getNames()) { + const std::optional CommonName = + getLTOCommonName(FunctionName); + if (CommonName) { + auto I = LTOCommonNameMap[Epoch].find(*CommonName); + if (I == LTOCommonNameMap[Epoch].end()) + continue; - for (StringRef FunctionName : Function.getNames()) { - const std::optional CommonName = - getLTOCommonName(FunctionName); - if (CommonName) { - auto I = LTOCommonNameMap.find(*CommonName); - if (I == LTOCommonNameMap.end()) - continue; + bool ProfileMatched = false; + std::vector <OProfiles = + I->getValue(); + for (yaml::bolt::BinaryFunctionProfile *YamlBF : LTOProfiles) { + if (YamlBF->Used) + continue; + if ((ProfileMatched = profileMatches(*YamlBF, Function))) { + matchProfileToFunction(*YamlBF, Function, Epoch); + break; + } + } + if (ProfileMatched) + break; - bool ProfileMatched = false; - std::vector <OProfiles = - I->getValue(); - for (yaml::bolt::BinaryFunctionProfile *YamlBF : LTOProfiles) { - if (YamlBF->Used) - continue; - if ((ProfileMatched = profileMatches(*YamlBF, Function))) { - matchProfileToFunction(*YamlBF, Function); + // If there's only one function with a given name, try to + // match it partially. + if (LTOProfiles.size() == 1 && + LTOCommonNameFunctionMap[Epoch][*CommonName].size() == 1 && + !LTOProfiles.front()->Used) { + matchProfileToFunction(*LTOProfiles.front(), Function, Epoch); break; } - } - if (ProfileMatched) - break; - - // If there's only one function with a given name, try to - // match it partially. - if (LTOProfiles.size() == 1 && - LTOCommonNameFunctionMap[*CommonName].size() == 1 && - !LTOProfiles.front()->Used) { - matchProfileToFunction(*LTOProfiles.front(), Function); - break; - } - } else { - auto PI = ProfileNameToProfile.find(FunctionName); - if (PI == ProfileNameToProfile.end()) - continue; + } else { + auto PI = ProfileNameToProfile[Epoch].find(FunctionName); + if (PI == ProfileNameToProfile[Epoch].end()) + continue; - yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); - if (!YamlBF.Used) { - matchProfileToFunction(YamlBF, Function); - break; + yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue(); + if (!YamlBF.Used) { + matchProfileToFunction(YamlBF, Function, Epoch); + break; + } } } } - } - for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) - if (!YamlBF.Used && opts::Verbosity >= 1) - errs() << "BOLT-WARNING: profile ignored for function " << YamlBF.Name - << '\n'; + for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) + if (!YamlBF.Used && opts::Verbosity >= 1) + errs() << "BOLT-WARNING: profile ignored for function " << YamlBF.Name + << '\n'; - // Set for parseFunctionProfile(). - NormalizeByInsnCount = usesEvent("cycles") || usesEvent("instructions"); - NormalizeByCalls = usesEvent("branches"); + // Set for parseFunctionProfile(). + NormalizeByInsnCount = + usesEvent("cycles", YamlBP) || usesEvent("instructions", YamlBP); + NormalizeByCalls = usesEvent("branches", YamlBP); - uint64_t NumUnused = 0; - for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) { - if (YamlBF.Id >= YamlProfileToFunction.size()) { - // Such profile was ignored. - ++NumUnused; - continue; + for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) { + if (YamlBF.Id >= YamlProfileToFunction[Epoch].size()) { + // Such profile was ignored. + ++NumUnused; + continue; + } + if (BinaryFunction *BF = YamlProfileToFunction[Epoch][YamlBF.Id]) + parseFunctionProfile(*BF, YamlBF, YamlBP, Epoch); + else + ++NumUnused; } - if (BinaryFunction *BF = YamlProfileToFunction[YamlBF.Id]) - parseFunctionProfile(*BF, YamlBF); - else - ++NumUnused; } BC.setNumUnusedProfiledObjects(NumUnused); @@ -411,7 +430,8 @@ return Error::success(); } -bool YAMLProfileReader::usesEvent(StringRef Name) const { +bool YAMLProfileReader::usesEvent( + StringRef Name, const yaml::bolt::BinaryProfile &YamlBP) const { return YamlBP.Header.EventNames.find(std::string(Name)) != StringRef::npos; }