Index: include/llvm/LTO/Config.h =================================================================== --- include/llvm/LTO/Config.h +++ include/llvm/LTO/Config.h @@ -73,6 +73,9 @@ /// Sample PGO profile path. std::string SampleProfile; + /// Name remapping file for profile data. + std::string ProfileRemapping; + /// The directory to store .dwo files. std::string DwoDir; Index: include/llvm/Passes/PassBuilder.h =================================================================== --- include/llvm/Passes/PassBuilder.h +++ include/llvm/Passes/PassBuilder.h @@ -32,10 +32,13 @@ /// A struct capturing PGO tunables. struct PGOOptions { PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "", - std::string SampleProfileFile = "", bool RunProfileGen = false, - bool SamplePGOSupport = false) + std::string SampleProfileFile = "", + std::string ProfileRemappingFile = "", + bool RunProfileGen = false, bool SamplePGOSupport = false) : ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile), - SampleProfileFile(SampleProfileFile), RunProfileGen(RunProfileGen), + SampleProfileFile(SampleProfileFile), + ProfileRemappingFile(ProfileRemappingFile), + RunProfileGen(RunProfileGen), SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) { assert((RunProfileGen || !SampleProfileFile.empty() || @@ -45,6 +48,7 @@ std::string ProfileGenFile; std::string ProfileUseFile; std::string SampleProfileFile; + std::string ProfileRemappingFile; bool RunProfileGen; bool SamplePGOSupport; }; @@ -585,7 +589,8 @@ void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, OptimizationLevel Level, bool RunProfileGen, std::string ProfileGenFile, - std::string ProfileUseFile); + std::string ProfileUseFile, + std::string ProfileRemappingFile); void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); Index: include/llvm/Transforms/IPO/SampleProfile.h =================================================================== --- include/llvm/Transforms/IPO/SampleProfile.h +++ include/llvm/Transforms/IPO/SampleProfile.h @@ -25,13 +25,16 @@ /// The sample profiler data loader pass. class SampleProfileLoaderPass : public PassInfoMixin { public: - SampleProfileLoaderPass(std::string File = "", bool IsThinLTOPreLink = false) - : ProfileFileName(File), IsThinLTOPreLink(IsThinLTOPreLink) {} + SampleProfileLoaderPass(std::string File = "", std::string RemappingFile = "", + bool IsThinLTOPreLink = false) + : ProfileFileName(File), ProfileRemappingFileName(RemappingFile), + IsThinLTOPreLink(IsThinLTOPreLink) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; + std::string ProfileRemappingFileName; bool IsThinLTOPreLink; }; Index: include/llvm/Transforms/Instrumentation/PGOInstrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -36,12 +36,14 @@ /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: - PGOInstrumentationUse(std::string Filename = ""); + PGOInstrumentationUse(std::string Filename = "", + std::string RemappingFilename = ""); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; + std::string ProfileRemappingFileName; }; /// The indirect function call promotion pass. Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -271,8 +271,15 @@ if (!Conf.SampleProfile.empty()) { auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile); - if (FileOrErr) + if (FileOrErr) { Hasher.update(FileOrErr.get()->getBuffer()); + + if (!Conf.ProfileRemapping.empty()) { + FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping); + if (FileOrErr) + Hasher.update(FileOrErr.get()->getBuffer()); + } + } } Key = toHex(Hasher.result()); Index: lib/LTO/LTOBackend.cpp =================================================================== --- lib/LTO/LTOBackend.cpp +++ lib/LTO/LTOBackend.cpp @@ -149,7 +149,8 @@ const ModuleSummaryIndex *ImportSummary) { Optional PGOOpt; if (!Conf.SampleProfile.empty()) - PGOOpt = PGOOptions("", "", Conf.SampleProfile, false, true); + PGOOpt = PGOOptions("", "", Conf.SampleProfile, Conf.ProfileRemapping, + false, true); PassBuilder PB(TM, PGOOpt); AAManager AA; Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -493,7 +493,8 @@ PassBuilder::OptimizationLevel Level, bool RunProfileGen, std::string ProfileGenFile, - std::string ProfileUseFile) { + std::string ProfileUseFile, + std::string ProfileRemappingFile) { // Generally running simplification passes and the inliner with an high // threshold results in smaller executables, but there may be cases where // the size grows, so let's be conservative here and skip this simplification @@ -547,7 +548,7 @@ } if (!ProfileUseFile.empty()) - MPM.addPass(PGOInstrumentationUse(ProfileUseFile)); + MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile)); } static InlineParams @@ -593,6 +594,7 @@ // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile, + PGOOpt->ProfileRemappingFile, Phase == ThinLTOPhase::PreLink)); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. @@ -642,7 +644,8 @@ if (PGOOpt && Phase != ThinLTOPhase::PostLink && (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) { addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen, - PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile); + PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile, + PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } Index: lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- lib/Transforms/IPO/SampleProfile.cpp +++ lib/Transforms/IPO/SampleProfile.cpp @@ -96,6 +96,10 @@ "sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden); +static cl::opt SampleProfileRemappingFile( + "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), + cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden); + static cl::opt SampleProfileMaxPropagateIterations( "sample-profile-max-propagate-iterations", cl::init(100), cl::desc("Maximum number of iterations to go through when propagating " @@ -183,12 +187,12 @@ class SampleProfileLoader { public: SampleProfileLoader( - StringRef Name, bool IsThinLTOPreLink, + StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, std::function GetAssumptionCache, std::function GetTargetTransformInfo) : GetAC(std::move(GetAssumptionCache)), GetTTI(std::move(GetTargetTransformInfo)), Filename(Name), - IsThinLTOPreLink(IsThinLTOPreLink) {} + RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {} bool doInitialization(Module &M); bool runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -282,6 +286,9 @@ /// Name of the profile file to load. std::string Filename; + /// Name of the profile remapping file to load. + std::string RemappingFilename; + /// Flag indicating whether the profile input loaded successfully. bool ProfileIsValid = false; @@ -311,13 +318,14 @@ SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, bool IsThinLTOPreLink = false) - : ModulePass(ID), SampleLoader(Name, IsThinLTOPreLink, - [&](Function &F) -> AssumptionCache & { - return ACT->getAssumptionCache(F); - }, - [&](Function &F) -> TargetTransformInfo & { - return TTIWP->getTTI(F); - }) { + : ModulePass(ID), + SampleLoader(Name, SampleProfileRemappingFile, IsThinLTOPreLink, + [&](Function &F) -> AssumptionCache & { + return ACT->getAssumptionCache(F); + }, + [&](Function &F) -> TargetTransformInfo & { + return TTIWP->getTTI(F); + }) { initializeSampleProfileLoaderLegacyPassPass( *PassRegistry::getPassRegistry()); } @@ -1520,11 +1528,23 @@ } Reader = std::move(ReaderOrErr.get()); ProfileIsValid = (Reader->read() == sampleprof_error::success); + + if (!RemappingFilename.empty()) { + ReaderOrErr = SampleProfileReaderItaniumRemapper::create( + RemappingFilename, Ctx, std::move(Reader)); + if (std::error_code EC = ReaderOrErr.getError()) { + std::string Msg = "Could not open profile remapping file: " + EC.message(); + Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); + return false; + } + Reader = std::move(ReaderOrErr.get()); + ProfileIsValid = (Reader->read() == sampleprof_error::success); + } return true; } ModulePass *llvm::createSampleProfileLoaderPass() { - return new SampleProfileLoaderLegacyPass(SampleProfileFile); + return new SampleProfileLoaderLegacyPass(); } ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) { @@ -1616,6 +1636,8 @@ SampleProfileLoader SampleLoader( ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, + ProfileRemappingFileName.empty() ? SampleProfileRemappingFile + : ProfileRemappingFileName, IsThinLTOPreLink, GetAssumptionCache, GetTTI); SampleLoader.doInitialization(M); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -141,6 +141,11 @@ cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose.")); +static cl::opt PGOTestProfileRemappingFile( + "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, + cl::value_desc("filename"), + cl::desc("Specify the path of profile remapping file. This is mainly for " + "test purpose.")); // Command line option to disable value profiling. The default is false: // i.e. value profiling is enabled by default. This is for debug purpose. @@ -1429,13 +1434,14 @@ } static bool annotateAllFunctions( - Module &M, StringRef ProfileFileName, + Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref LookupBPI, function_ref LookupBFI) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. - auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName); + auto ReaderOrErr = + IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName); if (Error E = ReaderOrErr.takeError()) { handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { Ctx.diagnose( @@ -1529,10 +1535,14 @@ return true; } -PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename) - : ProfileFileName(std::move(Filename)) { +PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename, + std::string RemappingFilename) + : ProfileFileName(std::move(Filename)), + ProfileRemappingFileName(std::move(RemappingFilename)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; + if (!PGOTestProfileRemappingFile.empty()) + ProfileRemappingFileName = PGOTestProfileRemappingFile; } PreservedAnalyses PGOInstrumentationUse::run(Module &M, @@ -1547,7 +1557,8 @@ return &FAM.getResult(F); }; - if (!annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI)) + if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, + LookupBPI, LookupBFI)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1564,7 +1575,7 @@ return &this->getAnalysis(F).getBFI(); }; - return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI); + return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI); } static std::string getSimpleNodeName(const BasicBlock *Node) { Index: test/Transforms/PGOProfile/Inputs/remap.map =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/Inputs/remap.map @@ -0,0 +1,8 @@ +# foo:: and foo::detail:: are equivalent +name 3foo N3foo6detailE + +# foo::qux and foo::quux are equivalent +type N3foo3quxE N3foo4quuxE + +# N::X and M::X are equivalent +name N1N1XE N1M1XE Index: test/Transforms/PGOProfile/Inputs/remap.proftext =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/Inputs/remap.proftext @@ -0,0 +1,8 @@ +# :ir is the flag to indicate this is IR level profile. +:ir +_ZN3foo3barERKN1N1XINS_4quuxEEE +25571299074 +2 +3 +2 + Index: test/Transforms/PGOProfile/remap.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/remap.ll @@ -0,0 +1,28 @@ +; RUN: llvm-profdata merge %S/Inputs/remap.proftext -o %t.profdata +; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-test-profile-remapping-file=%S/Inputs/remap.map -S | FileCheck %s --check-prefix=USE + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @_ZN3foo3barERKN1M1XINS_6detail3quxEEE(i32 %i) { +; USE-LABEL: @_ZN3foo3barERKN1M1XINS_6detail3quxEEE +; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]] +entry: + %cmp = icmp sgt i32 %i, 0 + br i1 %cmp, label %if.then, label %if.end +; USE: br i1 %cmp, label %if.then, label %if.end +; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]] + +if.then: + %add = add nsw i32 %i, 2 + br label %if.end + +if.end: + %retv = phi i32 [ %add, %if.then ], [ %i, %entry ] + ret i32 %retv +} + +; USE-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}} +; USE-DAG: {{![0-9]+}} = !{!"DetailedSummary", {{![0-9]+}}} +; USE-DAG: ![[FUNC_ENTRY_COUNT]] = !{!"function_entry_count", i64 3} +; USE-DAG: ![[BW_ENTRY]] = !{!"branch_weights", i32 2, i32 1} Index: test/Transforms/SampleProfile/Inputs/remap.map =================================================================== --- /dev/null +++ test/Transforms/SampleProfile/Inputs/remap.map @@ -0,0 +1,8 @@ +# foo:: and foo::detail:: are equivalent +name 3foo N3foo6detailE + +# foo::qux and foo::quux are equivalent +type N3foo3quxE N3foo4quuxE + +# N::X and M::X are equivalent +name N1N1XE N1M1XE Index: test/Transforms/SampleProfile/Inputs/remap.prof =================================================================== --- /dev/null +++ test/Transforms/SampleProfile/Inputs/remap.prof @@ -0,0 +1,10 @@ +_ZN3foo3barERKN1N1XINS_4quuxEEE:15680:2500 + 1: 2500 + 4: 1000 + 5: 1000 + 6: 800 + 7: 500 + 9: 10226 + 10: 2243 + 16: 0 + 18: 0 Index: test/Transforms/SampleProfile/remap.ll =================================================================== --- /dev/null +++ test/Transforms/SampleProfile/remap.ll @@ -0,0 +1,60 @@ +; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s + +; Reduced from branch.ll + +declare i1 @foo() + +define void @_ZN3foo3barERKN1M1XINS_6detail3quxEEE() !dbg !2 { +; CHECK: Printing analysis 'Branch Probability Analysis' for function '_ZN3foo3barERKN1M1XINS_6detail3quxEEE': + +entry: + %cmp = call i1 @foo(), !dbg !6 + br i1 %cmp, label %if.then, label %if.end +; CHECK: edge entry -> if.then probability is 0x4ccf6b16 / 0x80000000 = 60.01% +; CHECK: edge entry -> if.end probability is 0x333094ea / 0x80000000 = 39.99% + +if.then: + br label %return + +if.end: + %cmp1 = call i1 @foo(), !dbg !7 + br i1 %cmp1, label %if.then.2, label %if.else +; CHECK: edge if.end -> if.then.2 probability is 0x6652c748 / 0x80000000 = 79.94% +; CHECK: edge if.end -> if.else probability is 0x19ad38b8 / 0x80000000 = 20.06% + +if.then.2: + call i1 @foo(), !dbg !8 + br label %for.cond + +for.cond: + %cmp5 = call i1 @foo() + br i1 %cmp5, label %for.body, label %for.end, !prof !9 +; CHECK: edge for.cond -> for.body probability is 0x73333333 / 0x80000000 = 90.00% +; CHECK: edge for.cond -> for.end probability is 0x0ccccccd / 0x80000000 = 10.00% + +for.body: + br label %for.cond + +for.end: + br label %return + +if.else: + br label %return + +return: + ret void +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!4, !5} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "foo++", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !{}, retainedTypes: !{}) +!1 = !DIFile(filename: "test.cc", directory: "/foo/bar") +!2 = distinct !DISubprogram(name: "_ZN3foo3barERKN1M1XINS_6detail3quxEEE", scope: !1, file: !1, line: 4, type: !3, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !{}) +!3 = !DISubroutineType(types: !{}) +!4 = !{i32 2, !"Dwarf Version", i32 4} +!5 = !{i32 2, !"Debug Info Version", i32 3} +!6 = !DILocation(line: 5, column: 8, scope: !2) +!7 = !DILocation(line: 8, column: 6, scope: !2) +!8 = !DILocation(line: 10, column: 11, scope: !2) +!9 = !{!"branch_weights", i32 90, i32 10} Index: tools/opt/NewPMDriver.cpp =================================================================== --- tools/opt/NewPMDriver.cpp +++ tools/opt/NewPMDriver.cpp @@ -107,6 +107,10 @@ "Use sampled profile to guide PGO."))); static cl::opt ProfileFile( "profile-file", cl::desc("Path to the profile."), cl::Hidden); +static cl::opt + ProfileRemappingFile("profile-remapping-file", + cl::desc("Path to the profile remapping file."), + cl::Hidden); static cl::opt DebugInfoForProfiling( "new-pm-debug-info-for-profiling", cl::init(false), cl::Hidden, cl::desc("Emit special debug info to enable PGO profile generation.")); @@ -199,17 +203,17 @@ Optional P; switch (PGOKindFlag) { case InstrGen: - P = PGOOptions(ProfileFile, "", "", true); + P = PGOOptions(ProfileFile, "", "", "", true); break; case InstrUse: - P = PGOOptions("", ProfileFile, "", false); + P = PGOOptions("", ProfileFile, "", ProfileRemappingFile, false); break; case SampleUse: - P = PGOOptions("", "", ProfileFile, false); + P = PGOOptions("", "", ProfileFile, ProfileRemappingFile, false); break; case NoPGO: if (DebugInfoForProfiling) - P = PGOOptions("", "", "", false, true); + P = PGOOptions("", "", "", "", false, true); else P = None; }