Index: clang/docs/ReleaseNotes.rst =================================================================== --- clang/docs/ReleaseNotes.rst +++ clang/docs/ReleaseNotes.rst @@ -46,6 +46,11 @@ Major New Features ------------------ +- Clang supports use of a profile remapping file, which permits + profile data captured for one version of a program to be applied + when building another version where symbols have changed (for + example, due to renaming a class or namespace). + See the :doc:`UsersManual` for details. Improvements to Clang's diagnostics ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Index: clang/docs/UsersManual.rst =================================================================== --- clang/docs/UsersManual.rst +++ clang/docs/UsersManual.rst @@ -1799,6 +1799,69 @@ Note that these flags should appear after the corresponding profile flags to have an effect. +Profile remapping +^^^^^^^^^^^^^^^^^ + +When the program is compiled after a change that affects many symbol names, +pre-existing profile data may no longer match the program. For example: + + * switching from libstdc++ to libc++ will result in the mangled names of all + functions taking standard library types to change + * renaming a widely-used type in C++ will result in the mangled names of all + functions that have parameters involving that type to change + * moving from a 32-bit compilation to a 64-bit compilation may change the + underlying type of ``size_t`` and similar types, resulting in changes to + manglings + +Clang allows use of a profile remapping file to specify that such differences +in mangled names should be ignored when matching the profile data against the +program. + +.. option:: -fprofile-remapping-file= + + Specifies a file containing profile remapping information, that will be + used to match mangled names in the profile data to mangled names in the + program. + +The profile remapping file is a text file containing lines of the form + +.. code-block:: + + fragmentkind fragment1 fragment2 + +where ``fragmentkind`` is one of ``name``, ``type``, or ``encoding``, +indicating whether the following mangled name fragments are +<`name `>s, +<`type `>s, or +<`encoding `>s, +respectively. + +Blank lines and lines starting with ``#`` are ignored. + +For example, to specify that ``absl::string_view`` and ``std::string_view`` +should be treated as equivalent when matching profile data, the following +remapping file could be used: + +.. code-block:: + + # absl::string_view is considered equivalent to std::string_view + type N4absl11string_viewE St17basic_string_viewIcSt11char_traitsIcEE + + # std:: might be std::__1:: in libc++ or std::__cxx11:: in libstdc++ + name 3std St3__1 + name 3std St7__cxx11 + +Matching profile data using a profile remapping file is supported on a +best-effort basis. For example, information regarding indirect call targets is +currently not remapped. For best results, you are encouraged to generate new +profile data matching the updated program. + +.. note:: + + Profile data remapping is currently only supported for C++ mangled names + following the Itanium C++ ABI mangling scheme. This covers all C++ targets + supported by Clang other than Windows. + Controlling Debug Information ----------------------------- Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -729,6 +729,11 @@ def fprofile_instr_use_EQ : Joined<["-"], "fprofile-instr-use=">, Group, Flags<[CoreOption]>, HelpText<"Use instrumentation data for profile-guided optimization">; +def fprofile_remapping_file_EQ : Joined<["-"], "fprofile-remapping-file=">, + Group, Flags<[CC1Option, CoreOption]>, MetaVarName<"">, + HelpText<"Use the remappings described in to match the profile data against names in the program">; +def fprofile_remapping_file : Separate<["-"], "fprofile-remapping-file">, + Group, Flags<[CoreOption]>, Alias; def fcoverage_mapping : Flag<["-"], "fcoverage-mapping">, Group, Flags<[CC1Option, CoreOption]>, HelpText<"Generate coverage mapping to enable code coverage analysis">; Index: clang/include/clang/Frontend/CodeGenOptions.h =================================================================== --- clang/include/clang/Frontend/CodeGenOptions.h +++ clang/include/clang/Frontend/CodeGenOptions.h @@ -200,6 +200,10 @@ /// Name of the profile file to use as input for -fprofile-instr-use std::string ProfileInstrumentUsePath; + /// Name of the profile remapping file to apply to the profile data supplied + /// by -fprofile-sample-use or -fprofile-instr-use. + std::string ProfileRemappingFile; + /// Name of the function summary index file to use for ThinLTO function /// importing. std::string ThinLTOIndexFile; Index: clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- clang/lib/CodeGen/BackendUtil.cpp +++ clang/lib/CodeGen/BackendUtil.cpp @@ -910,18 +910,21 @@ PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() ? DefaultProfileGenName : CodeGenOpts.InstrProfileOutput, - "", "", true, CodeGenOpts.DebugInfoForProfiling); + "", "", "", true, + CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.hasProfileIRUse()) // -fprofile-use. - PGOOpt = PGOOptions("", CodeGenOpts.ProfileInstrumentUsePath, "", false, + PGOOpt = PGOOptions("", CodeGenOpts.ProfileInstrumentUsePath, "", + CodeGenOpts.ProfileRemappingFile, false, CodeGenOpts.DebugInfoForProfiling); else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use - PGOOpt = PGOOptions("", "", CodeGenOpts.SampleProfileFile, false, + PGOOpt = PGOOptions("", "", CodeGenOpts.SampleProfileFile, + CodeGenOpts.ProfileRemappingFile, false, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling - PGOOpt = PGOOptions("", "", "", false, true); + PGOOpt = PGOOptions("", "", "", "", false, true); PassBuilder PB(TM.get(), PGOOpt); @@ -1109,6 +1112,7 @@ const LangOptions &LOpts, std::unique_ptr OS, std::string SampleProfile, + std::string ProfileRemapping, BackendAction Action) { StringMap> ModuleToDefinedGVSummaries; @@ -1181,6 +1185,7 @@ Conf.CGOptLevel = getCGOptLevel(CGOpts); initTargetOptions(Conf.Options, CGOpts, TOpts, LOpts, HeaderOpts); Conf.SampleProfile = std::move(SampleProfile); + Conf.ProfileRemapping = std::move(ProfileRemapping); Conf.UseNewPM = CGOpts.ExperimentalNewPassManager; Conf.DebugPassManager = CGOpts.DebugPassManager; Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness; @@ -1247,7 +1252,7 @@ if (!CombinedIndex->skipModuleByDistributedBackend()) { runThinLTOBackend(CombinedIndex.get(), M, HeaderOpts, CGOpts, TOpts, LOpts, std::move(OS), CGOpts.SampleProfileFile, - Action); + CGOpts.ProfileRemappingFile, Action); return; } // Distributed indexing detected that nothing from the module is needed Index: clang/lib/CodeGen/CodeGenModule.cpp =================================================================== --- clang/lib/CodeGen/CodeGenModule.cpp +++ clang/lib/CodeGen/CodeGenModule.cpp @@ -152,7 +152,7 @@ if (CodeGenOpts.hasProfileClangUse()) { auto ReaderOrErr = llvm::IndexedInstrProfReader::create( - CodeGenOpts.ProfileInstrumentUsePath); + CodeGenOpts.ProfileInstrumentUsePath, CodeGenOpts.ProfileRemappingFile); if (auto E = ReaderOrErr.takeError()) { unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, "Could not read profile %0: %1"); Index: clang/lib/Driver/ToolChains/Clang.cpp =================================================================== --- clang/lib/Driver/ToolChains/Clang.cpp +++ clang/lib/Driver/ToolChains/Clang.cpp @@ -4166,6 +4166,7 @@ else A->render(Args, CmdArgs); } + Args.AddLastArg(CmdArgs, options::OPT_fprofile_remapping_file_EQ); RenderBuiltinOptions(getToolChain(), RawTriple, Args, CmdArgs); Index: clang/lib/Frontend/CompilerInvocation.cpp =================================================================== --- clang/lib/Frontend/CompilerInvocation.cpp +++ clang/lib/Frontend/CompilerInvocation.cpp @@ -652,6 +652,13 @@ Args.getLastArgValue(OPT_fprofile_instrument_use_path_EQ); if (!Opts.ProfileInstrumentUsePath.empty()) setPGOUseInstrumentor(Opts, Opts.ProfileInstrumentUsePath); + Opts.ProfileRemappingFile = + Args.getLastArgValue(OPT_fprofile_remapping_file_EQ); + if (!Opts.ProfileRemappingFile.empty() && !Opts.ExperimentalNewPassManager) { + Diags.Report(diag::err_drv_argument_only_allowed_with) + << Args.getLastArg(OPT_fprofile_remapping_file_EQ)->getAsString(Args) + << "-fexperimental-new-pass-manager"; + } Opts.CoverageMapping = Args.hasFlag(OPT_fcoverage_mapping, OPT_fno_coverage_mapping, false); Index: clang/test/CodeGenCXX/Inputs/profile-remap.map =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/Inputs/profile-remap.map @@ -0,0 +1,2 @@ +name 3Foo 3Bar +type N3Foo1XE N3Baz1YE Index: clang/test/CodeGenCXX/Inputs/profile-remap.proftext =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/Inputs/profile-remap.proftext @@ -0,0 +1,7 @@ +:ir +_ZN3Foo8functionENS_1XE +29667547796 +2 +10 +90 + Index: clang/test/CodeGenCXX/Inputs/profile-remap.samples =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/Inputs/profile-remap.samples @@ -0,0 +1,3 @@ +_ZN3Bar8functionEN3Baz1YE:100:0 + 2: 10 + 4: 90 Index: clang/test/CodeGenCXX/profile-remap.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/profile-remap.cpp @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -fprofile-sample-use=%S/Inputs/profile-remap.samples -fprofile-remapping-file=%S/Inputs/profile-remap.map -fexperimental-new-pass-manager -O2 %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SAMPLES +// RUN: llvm-profdata merge -output %T.profdata %S/Inputs/profile-remap.proftext +// RUN: %clang_cc1 -triple x86_64-linux-gnu -fprofile-instrument-use-path=%T.profdata -fprofile-remapping-file=%S/Inputs/profile-remap.map -fexperimental-new-pass-manager -O2 %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-INSTR + +namespace Foo { + struct X {}; + bool cond(); + void bar(); + void baz(); + void function(X x) { + if (cond()) + bar(); + else + baz(); + } +} + +// CHECK: define {{.*}} @_ZN3Foo8functionENS_1XE() {{.*}} !prof [[FUNC_ENTRY:![0-9]*]] +// CHECK: br i1 {{.*}} !prof [[BR_WEIGHTS:![0-9]*]] +// +// FIXME: Laplace's rule of succession is applied to sample profiles... +// CHECK-SAMPLES-DAG: [[FUNC_ENTRY]] = !{!"function_entry_count", i64 1} +// CHECK-SAMPLES-DAG: [[BR_WEIGHTS]] = !{!"branch_weights", i32 11, i32 91} +// +// ... but not to instruction profiles. +// CHECK-INSTR-DAG: [[FUNC_ENTRY]] = !{!"function_entry_count", i64 100} +// CHECK-INSTR-DAG: [[BR_WEIGHTS]] = !{!"branch_weights", i32 10, i32 90} Index: clang/test/Driver/clang_f_opts.c =================================================================== --- clang/test/Driver/clang_f_opts.c +++ clang/test/Driver/clang_f_opts.c @@ -116,6 +116,7 @@ // RUN: %clang -### -S -fcoverage-mapping %s 2>&1 | FileCheck -check-prefix=CHECK-COVERAGE-AND-GEN %s // RUN: %clang -### -S -fcoverage-mapping -fno-coverage-mapping %s 2>&1 | FileCheck -check-prefix=CHECK-DISABLE-COVERAGE %s // RUN: %clang -### -S -fprofile-instr-generate -fcoverage-mapping -fno-coverage-mapping %s 2>&1 | FileCheck -check-prefix=CHECK-DISABLE-COVERAGE %s +// RUN: %clang -### -S -fprofile-remapping-file foo/bar.txt %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-REMAP %s // CHECK-PROFILE-GENERATE: "-fprofile-instrument=clang" // CHECK-PROFILE-GENERATE-LLVM: "-fprofile-instrument=llvm" // CHECK-PROFILE-GENERATE-DIR: "-fprofile-instrument-path=/some/dir{{/|\\\\}}{{.*}}" @@ -126,6 +127,7 @@ // CHECK-DISABLE-USE-NOT: "-fprofile-instr-use" // CHECK-COVERAGE-AND-GEN: '-fcoverage-mapping' only allowed with '-fprofile-instr-generate' // CHECK-DISABLE-COVERAGE-NOT: "-fcoverage-mapping" +// CHECK-PROFILE-REMAP: "-fprofile-remapping-file=foo/bar.txt" // RUN: %clang -### -S -fprofile-use %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-USE %s // RUN: %clang -### -S -fprofile-instr-use %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-USE %s Index: llvm/include/llvm/LTO/Config.h =================================================================== --- llvm/include/llvm/LTO/Config.h +++ llvm/include/llvm/LTO/Config.h @@ -73,6 +73,9 @@ /// Sample PGO profile path. std::string SampleProfile; + /// Name remapping file for profile data. + std::string ProfileRemapping; + /// The directory to store .dwo files. std::string DwoDir; Index: llvm/include/llvm/Passes/PassBuilder.h =================================================================== --- llvm/include/llvm/Passes/PassBuilder.h +++ llvm/include/llvm/Passes/PassBuilder.h @@ -32,10 +32,13 @@ /// A struct capturing PGO tunables. struct PGOOptions { PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "", - std::string SampleProfileFile = "", bool RunProfileGen = false, - bool SamplePGOSupport = false) + std::string SampleProfileFile = "", + std::string ProfileRemappingFile = "", + bool RunProfileGen = false, bool SamplePGOSupport = false) : ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile), - SampleProfileFile(SampleProfileFile), RunProfileGen(RunProfileGen), + SampleProfileFile(SampleProfileFile), + ProfileRemappingFile(ProfileRemappingFile), + RunProfileGen(RunProfileGen), SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) { assert((RunProfileGen || !SampleProfileFile.empty() || @@ -45,6 +48,7 @@ std::string ProfileGenFile; std::string ProfileUseFile; std::string SampleProfileFile; + std::string ProfileRemappingFile; bool RunProfileGen; bool SamplePGOSupport; }; @@ -585,7 +589,8 @@ void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, OptimizationLevel Level, bool RunProfileGen, std::string ProfileGenFile, - std::string ProfileUseFile); + std::string ProfileUseFile, + std::string ProfileRemappingFile); void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); Index: llvm/include/llvm/ProfileData/InstrProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/InstrProfReader.h +++ llvm/include/llvm/ProfileData/InstrProfReader.h @@ -348,6 +348,9 @@ using OnDiskHashTableImplV3 = OnDiskIterableChainedHashTable; +template +class InstrProfReaderItaniumRemapper; + template class InstrProfReaderIndex : public InstrProfReaderIndexBase { private: @@ -355,6 +358,8 @@ typename HashTableImpl::data_iterator RecordIterator; uint64_t FormatVersion; + friend class InstrProfReaderItaniumRemapper; + public: InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, @@ -391,6 +396,8 @@ private: /// The profile data file contents. std::unique_ptr DataBuffer; + /// The profile remapping file contents. + std::unique_ptr RemappingBuffer; /// The index into the profile data. std::unique_ptr Index; /// Profile summary data. @@ -404,8 +411,11 @@ const unsigned char *Cur); public: - IndexedInstrProfReader(std::unique_ptr DataBuffer) - : DataBuffer(std::move(DataBuffer)), RecordIndex(0) {} + IndexedInstrProfReader( + std::unique_ptr DataBuffer, + std::unique_ptr RemappingBuffer = nullptr) + : DataBuffer(std::move(DataBuffer)), + RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {} IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; @@ -434,10 +444,11 @@ /// Factory method to create an indexed reader. static Expected> - create(const Twine &Path); + create(const Twine &Path, const Twine &RemappingPath = ""); static Expected> - create(std::unique_ptr Buffer); + create(std::unique_ptr Buffer, + std::unique_ptr RemappingBuffer = nullptr); // Used for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness) { Index: llvm/include/llvm/ProfileData/ProfRemappingReader.h =================================================================== --- /dev/null +++ llvm/include/llvm/ProfileData/ProfRemappingReader.h @@ -0,0 +1,125 @@ +//===- ProfRemappingReader.h - Read profile remapping file ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions needed for reading and applying profile +// remapping files. +// +// NOTE: If you are making changes to this file format, please remember +// to document them in the Clang documentation at +// tools/clang/docs/UsersManual.rst. +// +// File format +// ----------- +// +// The profile remappings are written as an ASCII text file. Blank lines and +// lines starting with a # are ignored. All other lines specify a kind of +// mangled name fragment, along with two fragments of that kind that should +// be treated as equivalent, separated by spaces. +// +// See http://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling for a +// description of the Itanium name mangling scheme. +// +// The accepted fragment kinds are: +// +// * name A , such as 6foobar or St3__1 +// * type A , such as Ss or N4llvm9StringRefE +// * encoding An (a complete mangling without the leading _Z) +// +// For example: +// +// # Ignore int / long differences to allow profile reuse between +// # 32-bit and 64-bit builds with differing size_t / ptrdiff_t / intptr_t. +// type i l +// type j m +// +// # Ignore differences between libc++ and libstdc++, and between libstdc++'s +// # C++98 and C++11 ABIs. +// name 3std St3__1 +// name 3std St7__cxx11 +// +// # Substitutions must be remapped separately from namespace 'std' for now. +// name Sa NSt3__19allocatorE +// name Sb NSt3__112basic_stringE +// type Ss NSt3__112basic_stringIcSt11char_traitsIcESaE +// # ... +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_PROFREMAPPINGREADER_H +#define LLVM_PROFILEDATA_PROFREMAPPINGREADER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Support/ItaniumManglingCanonicalizer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +class ProfileRemappingParseError + : public ErrorInfo { +public: + ProfileRemappingParseError(StringRef File, int64_t Line, Twine Message) + : File(File), Line(Line), Message(Message.str()) {} + + void log(llvm::raw_ostream &OS) const override { + OS << File << ':' << Line << ": " << Message; + } + std::error_code convertToErrorCode() const override { + return llvm::inconvertibleErrorCode(); + } + + StringRef getFileName() const { return File; } + int64_t getLineNum() const { return Line; } + StringRef getMessage() const { return Message; } + + static char ID; + +private: + std::string File; + int64_t Line; + std::string Message; +}; + +/// Profile name remapper. +/// +/// Remaps the symbol names in profile data to match those in the program +/// according to a set of rules specified in a given file. +class ItaniumProfileRemappingReader { +public: + /// Read remappings from the given buffer, which must live as long as + /// the remapper. + Error read(MemoryBuffer &B); + + using Key = ItaniumManglingCanonicalizer::Key; + + /// Map the given function name from the profiling data into a profile + /// data key. The result will be Key() if the name cannot be remapped. + Key remap(StringRef FunctionName) { + return Canonicalizer.canonicalize(FunctionName); + } + + /// Map the given function name from the program into a profile data key. + /// + /// The result will be Key() if the name has no corresponding entry in the + /// profile data, in which case the original name should be looked up in the + /// profile data. + /// + /// There is no guarantee that return values other than Key() have ever been + /// returned by remap(...), but this should be rare. + Key lookup(StringRef FunctionName) { + return Canonicalizer.lookup(FunctionName); + } + +private: + ItaniumManglingCanonicalizer Canonicalizer; +}; + +} // end namespace llvm + +#endif // LLVM_PROFILEDATA_PROFREMAPPINGREADER_H Index: llvm/include/llvm/ProfileData/SampleProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfReader.h +++ llvm/include/llvm/ProfileData/SampleProfReader.h @@ -219,6 +219,7 @@ #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/GCOV.h" #include "llvm/ProfileData/SampleProf.h" +#include "llvm/ProfileData/ProfRemappingReader.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" @@ -287,11 +288,16 @@ // The function name may have been updated by adding suffix. In sample // profile, the function names are all stripped, so we need to strip // the function name suffix before matching with profile. - StringRef Fname = F.getName().split('.').first; + return getSamplesFor(F.getName().split('.').first); + } + + /// Return the samples collected for function \p F. + virtual FunctionSamples *getSamplesFor(StringRef Fname) { std::string FGUID; Fname = getRepInFormat(Fname, getFormat(), FGUID); - if (Profiles.count(Fname)) - return &Profiles[Fname]; + auto It = Profiles.find(Fname); + if (It != Profiles.end()) + return &It->second; return nullptr; } @@ -335,6 +341,12 @@ /// Profile summary information. std::unique_ptr Summary; + /// Take ownership of the summary of this reader. + static std::unique_ptr + takeSummary(SampleProfileReader &Reader) { + return std::move(Reader.Summary); + } + /// Compute summary for this profile. void computeSummary(); @@ -503,6 +515,37 @@ static const uint32_t GCOVTagAFDOFunction = 0xac000000; }; +class SampleProfileReaderItaniumRemapper : public SampleProfileReader { +public: + SampleProfileReaderItaniumRemapper( + std::unique_ptr B, LLVMContext &C, + std::unique_ptr Underlying) + : SampleProfileReader(std::move(B), C, Underlying->getFormat()) { + Profiles = std::move(Underlying->getProfiles()); + Summary = takeSummary(*Underlying); + } + + /// Create a remapped sample profile from the given remapping file and + /// underlying samples. + static ErrorOr> + create(const Twine &Filename, LLVMContext &C, + std::unique_ptr Underlying); + + /// Read and validate the file header. + std::error_code readHeader() override { return sampleprof_error::success; } + + /// Read sample profiles from the associated file. + std::error_code read() override; + + /// Return the samples collected for function \p F. + FunctionSamples *getSamplesFor(StringRef FunctionName) override; + using SampleProfileReader::getSamplesFor; + +private: + ItaniumProfileRemappingReader Remappings; + DenseMap SampleMap; +}; + } // end namespace sampleprof } // end namespace llvm Index: llvm/include/llvm/Transforms/IPO/SampleProfile.h =================================================================== --- llvm/include/llvm/Transforms/IPO/SampleProfile.h +++ llvm/include/llvm/Transforms/IPO/SampleProfile.h @@ -25,13 +25,16 @@ /// The sample profiler data loader pass. class SampleProfileLoaderPass : public PassInfoMixin { public: - SampleProfileLoaderPass(std::string File = "", bool IsThinLTOPreLink = false) - : ProfileFileName(File), IsThinLTOPreLink(IsThinLTOPreLink) {} + SampleProfileLoaderPass(std::string File = "", std::string RemappingFile = "", + bool IsThinLTOPreLink = false) + : ProfileFileName(File), ProfileRemappingFileName(RemappingFile), + IsThinLTOPreLink(IsThinLTOPreLink) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; + std::string ProfileRemappingFileName; bool IsThinLTOPreLink; }; Index: llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h =================================================================== --- llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -36,12 +36,14 @@ /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: - PGOInstrumentationUse(std::string Filename = ""); + PGOInstrumentationUse(std::string Filename = "", + std::string RemappingFilename = ""); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; + std::string ProfileRemappingFileName; }; /// The indirect function call promotion pass. Index: llvm/lib/LTO/LTO.cpp =================================================================== --- llvm/lib/LTO/LTO.cpp +++ llvm/lib/LTO/LTO.cpp @@ -271,8 +271,15 @@ if (!Conf.SampleProfile.empty()) { auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile); - if (FileOrErr) + if (FileOrErr) { Hasher.update(FileOrErr.get()->getBuffer()); + + if (!Conf.ProfileRemapping.empty()) { + FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping); + if (FileOrErr) + Hasher.update(FileOrErr.get()->getBuffer()); + } + } } Key = toHex(Hasher.result()); Index: llvm/lib/LTO/LTOBackend.cpp =================================================================== --- llvm/lib/LTO/LTOBackend.cpp +++ llvm/lib/LTO/LTOBackend.cpp @@ -149,7 +149,8 @@ const ModuleSummaryIndex *ImportSummary) { Optional PGOOpt; if (!Conf.SampleProfile.empty()) - PGOOpt = PGOOptions("", "", Conf.SampleProfile, false, true); + PGOOpt = PGOOptions("", "", Conf.SampleProfile, Conf.ProfileRemapping, + false, true); PassBuilder PB(TM, PGOOpt); AAManager AA; Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -493,7 +493,8 @@ PassBuilder::OptimizationLevel Level, bool RunProfileGen, std::string ProfileGenFile, - std::string ProfileUseFile) { + std::string ProfileUseFile, + std::string ProfileRemappingFile) { // Generally running simplification passes and the inliner with an high // threshold results in smaller executables, but there may be cases where // the size grows, so let's be conservative here and skip this simplification @@ -547,7 +548,7 @@ } if (!ProfileUseFile.empty()) - MPM.addPass(PGOInstrumentationUse(ProfileUseFile)); + MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile)); } static InlineParams @@ -593,6 +594,7 @@ // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile, + PGOOpt->ProfileRemappingFile, Phase == ThinLTOPhase::PreLink)); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. @@ -642,7 +644,8 @@ if (PGOOpt && Phase != ThinLTOPhase::PostLink && (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) { addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen, - PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile); + PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile, + PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } Index: llvm/lib/ProfileData/CMakeLists.txt =================================================================== --- llvm/lib/ProfileData/CMakeLists.txt +++ llvm/lib/ProfileData/CMakeLists.txt @@ -4,6 +4,7 @@ InstrProfReader.cpp InstrProfWriter.cpp ProfileSummaryBuilder.cpp + ProfRemappingReader.cpp SampleProf.cpp SampleProfReader.cpp SampleProfWriter.cpp Index: llvm/lib/ProfileData/InstrProfReader.cpp =================================================================== --- llvm/lib/ProfileData/InstrProfReader.cpp +++ llvm/lib/ProfileData/InstrProfReader.cpp @@ -19,6 +19,7 @@ #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/ProfileData/ProfRemappingReader.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorOr.h" @@ -88,16 +89,29 @@ } Expected> -IndexedInstrProfReader::create(const Twine &Path) { +IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); if (Error E = BufferOrError.takeError()) return std::move(E); - return IndexedInstrProfReader::create(std::move(BufferOrError.get())); + + // Set up the remapping buffer if requested. + std::unique_ptr RemappingBuffer; + std::string RemappingPathStr = RemappingPath.str(); + if (!RemappingPathStr.empty()) { + auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr); + if (Error E = RemappingBufferOrError.takeError()) + return std::move(E); + RemappingBuffer = std::move(RemappingBufferOrError.get()); + } + + return IndexedInstrProfReader::create(std::move(BufferOrError.get()), + std::move(RemappingBuffer)); } Expected> -IndexedInstrProfReader::create(std::unique_ptr Buffer) { +IndexedInstrProfReader::create(std::unique_ptr Buffer, + std::unique_ptr RemappingBuffer) { // Sanity check the buffer. if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits::max()) return make_error(instrprof_error::too_large); @@ -105,7 +119,8 @@ // Create the reader. if (!IndexedInstrProfReader::hasFormat(*Buffer)) return make_error(instrprof_error::bad_magic); - auto Result = llvm::make_unique(std::move(Buffer)); + auto Result = llvm::make_unique( + std::move(Buffer), std::move(RemappingBuffer)); // Initialize the reader and return the result. if (Error E = initializeReader(*Result)) @@ -587,6 +602,124 @@ RecordIterator = HashTable->data_begin(); } +template +class llvm::InstrProfReaderItaniumRemapper : public InstrProfReaderIndexBase { +public: + InstrProfReaderItaniumRemapper( + std::unique_ptr RemapBuffer, + std::unique_ptr> Underlying) + : RemapBuffer(std::move(RemapBuffer)), Underlying(std::move(Underlying)) { + } + + /// Extract the original function name from a PGO function name. + static StringRef extractName(StringRef Name) { + // We can have multiple :-separated pieces; there can be pieces both + // before and after the mangled name. Find the first part that starts + // with '_Z'; we'll assume that's the mangled name we want. + std::pair Parts = {StringRef(), Name}; + while (true) { + Parts = Parts.second.split(':'); + if (Parts.first.startswith("_Z")) + return Parts.first; + if (Parts.second.empty()) + return Name; + } + } + + /// Given a mangled name extracted from a PGO function name, and a new + /// form for that mangled name, reconstitute the name. + static void reconstituteName(StringRef OrigName, StringRef ExtractedName, + StringRef Replacement, + llvm::SmallVectorImpl &Out) { + Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size()); + Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin()); + Out.insert(Out.end(), Replacement.begin(), Replacement.end()); + Out.insert(Out.end(), ExtractedName.end(), OrigName.end()); + } + + Error populateRemappings() { + if (Error E = Remappings.read(*RemapBuffer)) + return E; + for (StringRef Name : Underlying->HashTable->keys()) { + StringRef RealName = extractName(Name); + if (auto Key = Remappings.remap(RealName)) { + // FIXME: We could theoretically map the same equivalence class to + // multiple names in the profile data. If that happens, we should + // return NamedInstrProfRecords from all of them. + MappedNames.insert({Key, RealName}); + } + } + return Error::success(); + } + + Error getRecords(StringRef FuncName, + ArrayRef &Data) override { + StringRef RealName = extractName(FuncName); + if (auto Key = Remappings.lookup(RealName)) { + StringRef Remapped = MappedNames.lookup(Key); + if (!Remapped.empty()) { + if (RealName.begin() == FuncName.begin() && + RealName.end() == FuncName.end()) + FuncName = Remapped; + else { + // Try rebuilding the name from the given remapping. + llvm::SmallString<256> Reconstituted; + reconstituteName(FuncName, RealName, Remapped, Reconstituted); + Error E = Underlying->getRecords(Reconstituted, Data); + if (!E) + return E; + + // If we failed because the name doesn't exist, fall back to asking + // about the original name. + if (Error Unhandled = handleErrors( + std::move(E), [](std::unique_ptr Err) { + return Err->get() == instrprof_error::unknown_function + ? Error::success() + : Error(std::move(Err)); + })) + return Unhandled; + } + } + } + return Underlying->getRecords(FuncName, Data); + } + + Error getRecords(ArrayRef &Data) override { + return Underlying->getRecords(Data); + } + void advanceToNextKey() override { return Underlying->advanceToNextKey(); } + bool atEnd() const override { return Underlying->atEnd(); } + void setValueProfDataEndianness(support::endianness Endianness) override { + return Underlying->setValueProfDataEndianness(Endianness); + } + uint64_t getVersion() const override { + return Underlying->getVersion(); + } + bool isIRLevelProfile() const override { + return Underlying->isIRLevelProfile(); + } + Error populateSymtab(InstrProfSymtab &Symtab) override { + return Underlying->populateSymtab(Symtab); + } + +private: + /// The memory buffer containing the remapping configuration. Remappings + /// holds pointers into this buffer. + std::unique_ptr RemapBuffer; + + /// The mangling remapper. + ItaniumProfileRemappingReader Remappings; + + /// Mapping from mangled name keys to the name used for the key in the + /// profile data. + /// FIXME: Can we store a location within the on-disk hash table instead of + /// redoing lookup? + llvm::DenseMap MappedNames; + + /// The real profile data reader. + std::unique_ptr> Underlying; +}; + bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { using namespace support; @@ -683,10 +816,22 @@ uint64_t HashOffset = endian::byte_swap(Header->HashOffset); // The rest of the file is an on disk hash table. - InstrProfReaderIndexBase *IndexPtr = nullptr; - IndexPtr = new InstrProfReaderIndex( - Start + HashOffset, Cur, Start, HashType, FormatVersion); - Index.reset(IndexPtr); + auto IndexPtr = + llvm::make_unique>( + Start + HashOffset, Cur, Start, HashType, FormatVersion); + + // Load the remapping table now if requested. + if (RemappingBuffer) { + auto Remapper = llvm::make_unique< + InstrProfReaderItaniumRemapper>( + std::move(RemappingBuffer), std::move(IndexPtr)); + if (Error E = Remapper->populateRemappings()) + return E; + Index = std::move(Remapper); + } else { + Index = std::move(IndexPtr); + } + return success(); } Index: llvm/lib/ProfileData/ProfRemappingReader.cpp =================================================================== --- /dev/null +++ llvm/lib/ProfileData/ProfRemappingReader.cpp @@ -0,0 +1,83 @@ +//===- ProfRemappingReader.cpp - Read profile remapping file --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions needed for reading and applying profile +// remapping files. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/ProfRemappingReader.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/Support/LineIterator.h" + +using namespace llvm; + +char ProfileRemappingParseError::ID; + +/// Load a set of name remappings from a text file. +/// +/// See the documentation at the top of the file for an explanation of +/// the expected format. +Error ItaniumProfileRemappingReader::read(MemoryBuffer &B) { + line_iterator LineIt(B, /*SkipBlanks=*/true, '#'); + + auto ReportError = [&](Twine Msg) { + return llvm::make_error( + B.getBufferIdentifier(), LineIt.line_number(), Msg); + }; + + for (; !LineIt.is_at_eof(); ++LineIt) { + StringRef Line = *LineIt; + Line = Line.ltrim(' '); + // line_iterator only detects comments starting in column 1. + if (Line.startswith("#") || Line.empty()) + continue; + + SmallVector Parts; + Line.split(Parts, ' ', /*MaxSplits*/-1, /*KeepEmpty*/false); + + if (Parts.size() != 3) + return ReportError("Expected 'kind mangled_name mangled_name', " + "found '" + Line + "'"); + + using FK = ItaniumManglingCanonicalizer::FragmentKind; + Optional FragmentKind = StringSwitch>(Parts[0]) + .Case("name", FK::Name) + .Case("type", FK::Type) + .Case("encoding", FK::Encoding) + .Default(None); + if (!FragmentKind) + return ReportError("Invalid kind, expected 'name', 'type', or 'encoding'," + " found '" + Parts[0] + "'"); + + using EE = ItaniumManglingCanonicalizer::EquivalenceError; + switch (Canonicalizer.addEquivalence(*FragmentKind, Parts[1], Parts[2])) { + case EE::Success: + break; + + case EE::ManglingAlreadyUsed: + return ReportError("Manglings '" + Parts[1] + "' and '" + Parts[2] + "' " + "have both been used in prior remappings. Move this " + "remapping earlier in the file."); + + case EE::InvalidFirstMangling: + return ReportError("Could not demangle '" + Parts[1] + "' " + "as a <" + Parts[0] + ">; invalid mangling?"); + + case EE::InvalidSecondMangling: + return ReportError("Could not demangle '" + Parts[2] + "' " + "as a <" + Parts[0] + ">; invalid mangling?"); + } + } + + return Error::success(); +} Index: llvm/lib/ProfileData/SampleProfReader.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfReader.cpp +++ llvm/lib/ProfileData/SampleProfReader.cpp @@ -827,6 +827,40 @@ return Magic == "adcg*704"; } +std::error_code SampleProfileReaderItaniumRemapper::read() { + // If the underlying data is in compact format, we can't remap it because + // we don't know what the original function names were. + if (getFormat() == SPF_Compact_Binary) { + Ctx.diagnose(DiagnosticInfoSampleProfile( + Buffer->getBufferIdentifier(), + "Profile data remapping cannot be applied to profile data " + "in compact format (original mangled names are not available).", + DS_Warning)); + return sampleprof_error::success; + } + + if (Error E = Remappings.read(*Buffer)) { + handleAllErrors( + std::move(E), [&](const ProfileRemappingParseError &ParseError) { + reportError(ParseError.getLineNum(), ParseError.getMessage()); + }); + return sampleprof_error::malformed; + } + + for (auto &Sample : getProfiles()) + if (auto Key = Remappings.remap(Sample.first())) + SampleMap.insert({Key, &Sample.second}); + + return sampleprof_error::success; +} + +FunctionSamples * +SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) { + if (auto Key = Remappings.lookup(Fname)) + return SampleMap.lookup(Key); + return SampleProfileReader::getSamplesFor(Fname); +} + /// Prepare a memory buffer for the contents of \p Filename. /// /// \returns an error code indicating the status of the buffer. @@ -859,6 +893,27 @@ return create(BufferOrError.get(), C); } +/// Create a sample profile remapper from the given input, to remap the +/// function names in the given profile data. +/// +/// \param Filename The file to open. +/// +/// \param C The LLVM context to use to emit diagnostics. +/// +/// \param Underlying The underlying profile data reader to remap. +/// +/// \returns an error code indicating the status of the created reader. +ErrorOr> +SampleProfileReaderItaniumRemapper::create( + const Twine &Filename, LLVMContext &C, + std::unique_ptr Underlying) { + auto BufferOrError = setupMemoryBuffer(Filename); + if (std::error_code EC = BufferOrError.getError()) + return EC; + return llvm::make_unique( + std::move(BufferOrError.get()), C, std::move(Underlying)); +} + /// Create a sample profile reader based on the format of the input data. /// /// \param B The memory buffer to create the reader from (assumes ownership). Index: llvm/lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- llvm/lib/Transforms/IPO/SampleProfile.cpp +++ llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -96,6 +96,10 @@ "sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden); +static cl::opt SampleProfileRemappingFile( + "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), + cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden); + static cl::opt SampleProfileMaxPropagateIterations( "sample-profile-max-propagate-iterations", cl::init(100), cl::desc("Maximum number of iterations to go through when propagating " @@ -183,12 +187,12 @@ class SampleProfileLoader { public: SampleProfileLoader( - StringRef Name, bool IsThinLTOPreLink, + StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, std::function GetAssumptionCache, std::function GetTargetTransformInfo) : GetAC(std::move(GetAssumptionCache)), GetTTI(std::move(GetTargetTransformInfo)), Filename(Name), - IsThinLTOPreLink(IsThinLTOPreLink) {} + RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {} bool doInitialization(Module &M); bool runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -282,6 +286,9 @@ /// Name of the profile file to load. std::string Filename; + /// Name of the profile remapping file to load. + std::string RemappingFilename; + /// Flag indicating whether the profile input loaded successfully. bool ProfileIsValid = false; @@ -311,13 +318,14 @@ SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, bool IsThinLTOPreLink = false) - : ModulePass(ID), SampleLoader(Name, IsThinLTOPreLink, - [&](Function &F) -> AssumptionCache & { - return ACT->getAssumptionCache(F); - }, - [&](Function &F) -> TargetTransformInfo & { - return TTIWP->getTTI(F); - }) { + : ModulePass(ID), + SampleLoader(Name, SampleProfileRemappingFile, IsThinLTOPreLink, + [&](Function &F) -> AssumptionCache & { + return ACT->getAssumptionCache(F); + }, + [&](Function &F) -> TargetTransformInfo & { + return TTIWP->getTTI(F); + }) { initializeSampleProfileLoaderLegacyPassPass( *PassRegistry::getPassRegistry()); } @@ -1520,11 +1528,23 @@ } Reader = std::move(ReaderOrErr.get()); ProfileIsValid = (Reader->read() == sampleprof_error::success); + + if (!RemappingFilename.empty()) { + ReaderOrErr = SampleProfileReaderItaniumRemapper::create( + RemappingFilename, Ctx, std::move(Reader)); + if (std::error_code EC = ReaderOrErr.getError()) { + std::string Msg = "Could not open profile remapping file: " + EC.message(); + Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); + return false; + } + Reader = std::move(ReaderOrErr.get()); + ProfileIsValid = (Reader->read() == sampleprof_error::success); + } return true; } ModulePass *llvm::createSampleProfileLoaderPass() { - return new SampleProfileLoaderLegacyPass(SampleProfileFile); + return new SampleProfileLoaderLegacyPass(); } ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) { @@ -1616,6 +1636,8 @@ SampleProfileLoader SampleLoader( ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, + ProfileRemappingFileName.empty() ? SampleProfileRemappingFile + : ProfileRemappingFileName, IsThinLTOPreLink, GetAssumptionCache, GetTTI); SampleLoader.doInitialization(M); Index: llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -141,6 +141,11 @@ cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose.")); +static cl::opt PGOTestProfileRemappingFile( + "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, + cl::value_desc("filename"), + cl::desc("Specify the path of profile remapping file. This is mainly for " + "test purpose.")); // Command line option to disable value profiling. The default is false: // i.e. value profiling is enabled by default. This is for debug purpose. @@ -1429,13 +1434,14 @@ } static bool annotateAllFunctions( - Module &M, StringRef ProfileFileName, + Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref LookupBPI, function_ref LookupBFI) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. - auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName); + auto ReaderOrErr = + IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName); if (Error E = ReaderOrErr.takeError()) { handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { Ctx.diagnose( @@ -1529,10 +1535,14 @@ return true; } -PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename) - : ProfileFileName(std::move(Filename)) { +PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename, + std::string RemappingFilename) + : ProfileFileName(std::move(Filename)), + ProfileRemappingFileName(std::move(RemappingFilename)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; + if (!PGOTestProfileRemappingFile.empty()) + ProfileRemappingFileName = PGOTestProfileRemappingFile; } PreservedAnalyses PGOInstrumentationUse::run(Module &M, @@ -1547,7 +1557,8 @@ return &FAM.getResult(F); }; - if (!annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI)) + if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, + LookupBPI, LookupBFI)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1564,7 +1575,7 @@ return &this->getAnalysis(F).getBFI(); }; - return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI); + return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI); } static std::string getSimpleNodeName(const BasicBlock *Node) { Index: llvm/test/Transforms/PGOProfile/Inputs/remap.map =================================================================== --- /dev/null +++ llvm/test/Transforms/PGOProfile/Inputs/remap.map @@ -0,0 +1,8 @@ +# foo:: and foo::detail:: are equivalent +name 3foo N3foo6detailE + +# foo::qux and foo::quux are equivalent +type N3foo3quxE N3foo4quuxE + +# N::X and M::X are equivalent +name N1N1XE N1M1XE Index: llvm/test/Transforms/PGOProfile/Inputs/remap.proftext =================================================================== --- /dev/null +++ llvm/test/Transforms/PGOProfile/Inputs/remap.proftext @@ -0,0 +1,8 @@ +# :ir is the flag to indicate this is IR level profile. +:ir +_ZN3foo3barERKN1N1XINS_4quuxEEE +25571299074 +2 +3 +2 + Index: llvm/test/Transforms/PGOProfile/remap.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/PGOProfile/remap.ll @@ -0,0 +1,28 @@ +; RUN: llvm-profdata merge %S/Inputs/remap.proftext -o %t.profdata +; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-test-profile-remapping-file=%S/Inputs/remap.map -S | FileCheck %s --check-prefix=USE + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @_ZN3foo3barERKN1M1XINS_6detail3quxEEE(i32 %i) { +; USE-LABEL: @_ZN3foo3barERKN1M1XINS_6detail3quxEEE +; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]] +entry: + %cmp = icmp sgt i32 %i, 0 + br i1 %cmp, label %if.then, label %if.end +; USE: br i1 %cmp, label %if.then, label %if.end +; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]] + +if.then: + %add = add nsw i32 %i, 2 + br label %if.end + +if.end: + %retv = phi i32 [ %add, %if.then ], [ %i, %entry ] + ret i32 %retv +} + +; USE-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}} +; USE-DAG: {{![0-9]+}} = !{!"DetailedSummary", {{![0-9]+}}} +; USE-DAG: ![[FUNC_ENTRY_COUNT]] = !{!"function_entry_count", i64 3} +; USE-DAG: ![[BW_ENTRY]] = !{!"branch_weights", i32 2, i32 1} Index: llvm/test/Transforms/SampleProfile/Inputs/remap.map =================================================================== --- /dev/null +++ llvm/test/Transforms/SampleProfile/Inputs/remap.map @@ -0,0 +1,8 @@ +# foo:: and foo::detail:: are equivalent +name 3foo N3foo6detailE + +# foo::qux and foo::quux are equivalent +type N3foo3quxE N3foo4quuxE + +# N::X and M::X are equivalent +name N1N1XE N1M1XE Index: llvm/test/Transforms/SampleProfile/Inputs/remap.prof =================================================================== --- /dev/null +++ llvm/test/Transforms/SampleProfile/Inputs/remap.prof @@ -0,0 +1,10 @@ +_ZN3foo3barERKN1N1XINS_4quuxEEE:15680:2500 + 1: 2500 + 4: 1000 + 5: 1000 + 6: 800 + 7: 500 + 9: 10226 + 10: 2243 + 16: 0 + 18: 0 Index: llvm/test/Transforms/SampleProfile/remap.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/SampleProfile/remap.ll @@ -0,0 +1,60 @@ +; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s + +; Reduced from branch.ll + +declare i1 @foo() + +define void @_ZN3foo3barERKN1M1XINS_6detail3quxEEE() !dbg !2 { +; CHECK: Printing analysis 'Branch Probability Analysis' for function '_ZN3foo3barERKN1M1XINS_6detail3quxEEE': + +entry: + %cmp = call i1 @foo(), !dbg !6 + br i1 %cmp, label %if.then, label %if.end +; CHECK: edge entry -> if.then probability is 0x4ccf6b16 / 0x80000000 = 60.01% +; CHECK: edge entry -> if.end probability is 0x333094ea / 0x80000000 = 39.99% + +if.then: + br label %return + +if.end: + %cmp1 = call i1 @foo(), !dbg !7 + br i1 %cmp1, label %if.then.2, label %if.else +; CHECK: edge if.end -> if.then.2 probability is 0x6652c748 / 0x80000000 = 79.94% +; CHECK: edge if.end -> if.else probability is 0x19ad38b8 / 0x80000000 = 20.06% + +if.then.2: + call i1 @foo(), !dbg !8 + br label %for.cond + +for.cond: + %cmp5 = call i1 @foo() + br i1 %cmp5, label %for.body, label %for.end, !prof !9 +; CHECK: edge for.cond -> for.body probability is 0x73333333 / 0x80000000 = 90.00% +; CHECK: edge for.cond -> for.end probability is 0x0ccccccd / 0x80000000 = 10.00% + +for.body: + br label %for.cond + +for.end: + br label %return + +if.else: + br label %return + +return: + ret void +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!4, !5} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "foo++", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !{}, retainedTypes: !{}) +!1 = !DIFile(filename: "test.cc", directory: "/foo/bar") +!2 = distinct !DISubprogram(name: "_ZN3foo3barERKN1M1XINS_6detail3quxEEE", scope: !1, file: !1, line: 4, type: !3, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !{}) +!3 = !DISubroutineType(types: !{}) +!4 = !{i32 2, !"Dwarf Version", i32 4} +!5 = !{i32 2, !"Debug Info Version", i32 3} +!6 = !DILocation(line: 5, column: 8, scope: !2) +!7 = !DILocation(line: 8, column: 6, scope: !2) +!8 = !DILocation(line: 10, column: 11, scope: !2) +!9 = !{!"branch_weights", i32 90, i32 10} Index: llvm/tools/opt/NewPMDriver.cpp =================================================================== --- llvm/tools/opt/NewPMDriver.cpp +++ llvm/tools/opt/NewPMDriver.cpp @@ -107,6 +107,10 @@ "Use sampled profile to guide PGO."))); static cl::opt ProfileFile( "profile-file", cl::desc("Path to the profile."), cl::Hidden); +static cl::opt + ProfileRemappingFile("profile-remapping-file", + cl::desc("Path to the profile remapping file."), + cl::Hidden); static cl::opt DebugInfoForProfiling( "new-pm-debug-info-for-profiling", cl::init(false), cl::Hidden, cl::desc("Emit special debug info to enable PGO profile generation.")); @@ -199,17 +203,17 @@ Optional P; switch (PGOKindFlag) { case InstrGen: - P = PGOOptions(ProfileFile, "", "", true); + P = PGOOptions(ProfileFile, "", "", "", true); break; case InstrUse: - P = PGOOptions("", ProfileFile, "", false); + P = PGOOptions("", ProfileFile, "", ProfileRemappingFile, false); break; case SampleUse: - P = PGOOptions("", "", ProfileFile, false); + P = PGOOptions("", "", ProfileFile, ProfileRemappingFile, false); break; case NoPGO: if (DebugInfoForProfiling) - P = PGOOptions("", "", "", false, true); + P = PGOOptions("", "", "", "", false, true); else P = None; } Index: llvm/unittests/ProfileData/InstrProfTest.cpp =================================================================== --- llvm/unittests/ProfileData/InstrProfTest.cpp +++ llvm/unittests/ProfileData/InstrProfTest.cpp @@ -42,8 +42,10 @@ void SetUp() { Writer.setOutputSparse(false); } - void readProfile(std::unique_ptr Profile) { - auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile)); + void readProfile(std::unique_ptr Profile, + std::unique_ptr Remapping = nullptr) { + auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile), + std::move(Remapping)); EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded()); Reader = std::move(ReaderOrErr.get()); } @@ -990,6 +992,44 @@ } } +TEST_P(MaybeSparseInstrProfTest, remapping_test) { + Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err); + Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"( + type i l + name 3bar 4quux + )")); + + std::vector Counts; + for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) { + EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts), + Succeeded()); + ASSERT_EQ(4u, Counts.size()); + EXPECT_EQ(1u, Counts[0]); + EXPECT_EQ(2u, Counts[1]); + EXPECT_EQ(3u, Counts[2]); + EXPECT_EQ(4u, Counts[3]); + } + + for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) { + EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts), + Succeeded()); + ASSERT_EQ(3u, Counts.size()); + EXPECT_EQ(5u, Counts[0]); + EXPECT_EQ(6u, Counts[1]); + EXPECT_EQ(7u, Counts[2]); + } + + for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ", + "_Z3barf", "otherfile:_Z4quuxf"}) { + EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts), + Failed()); + EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts), + Failed()); + } +} + TEST_F(SparseInstrProfTest, preserve_no_records) { Writer.addRecord({"foo", 0x1234, {0}}, Err); Writer.addRecord({"bar", 0x4321, {0, 0}}, Err); Index: llvm/unittests/ProfileData/SampleProfTest.cpp =================================================================== --- llvm/unittests/ProfileData/SampleProfTest.cpp +++ llvm/unittests/ProfileData/SampleProfTest.cpp @@ -57,7 +57,7 @@ Reader = std::move(ReaderOrErr.get()); } - void testRoundTrip(SampleProfileFormat Format) { + void testRoundTrip(SampleProfileFormat Format, bool Remap) { createWriter(Format); StringRef FooName("_Z3fooi"); @@ -99,22 +99,35 @@ EC = Reader->read(); ASSERT_TRUE(NoError(EC)); - StringMap &ReadProfiles = Reader->getProfiles(); - ASSERT_EQ(2u, ReadProfiles.size()); - - std::string FooGUID; - StringRef FooRep = getRepInFormat(FooName, Format, FooGUID); - FunctionSamples &ReadFooSamples = ReadProfiles[FooRep]; - ASSERT_EQ(7711u, ReadFooSamples.getTotalSamples()); - ASSERT_EQ(610u, ReadFooSamples.getHeadSamples()); - - std::string BarGUID; - StringRef BarRep = getRepInFormat(BarName, Format, BarGUID); - FunctionSamples &ReadBarSamples = ReadProfiles[BarRep]; - ASSERT_EQ(20301u, ReadBarSamples.getTotalSamples()); - ASSERT_EQ(1437u, ReadBarSamples.getHeadSamples()); + if (Remap) { + auto MemBuffer = llvm::MemoryBuffer::getMemBuffer(R"( + # Types 'int' and 'long' are equivalent + type i l + # Function names 'foo' and 'faux' are equivalent + name 3foo 4faux + )"); + Reader.reset(new SampleProfileReaderItaniumRemapper( + std::move(MemBuffer), Context, std::move(Reader))); + FooName = "_Z4fauxi"; + BarName = "_Z3barl"; + + EC = Reader->read(); + ASSERT_TRUE(NoError(EC)); + } + + ASSERT_EQ(2u, Reader->getProfiles().size()); + + FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName); + ASSERT_TRUE(ReadFooSamples != nullptr); + ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples()); + ASSERT_EQ(610u, ReadFooSamples->getHeadSamples()); + + FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName); + ASSERT_TRUE(ReadBarSamples != nullptr); + ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples()); + ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples()); ErrorOr CTMap = - ReadBarSamples.findCallTargetMapAt(1, 0); + ReadBarSamples->findCallTargetMapAt(1, 0); ASSERT_FALSE(CTMap.getError()); std::string MconstructGUID; @@ -176,15 +189,19 @@ }; TEST_F(SampleProfTest, roundtrip_text_profile) { - testRoundTrip(SampleProfileFormat::SPF_Text); + testRoundTrip(SampleProfileFormat::SPF_Text, false); } TEST_F(SampleProfTest, roundtrip_raw_binary_profile) { - testRoundTrip(SampleProfileFormat::SPF_Binary); + testRoundTrip(SampleProfileFormat::SPF_Binary, false); } TEST_F(SampleProfTest, roundtrip_compact_binary_profile) { - testRoundTrip(SampleProfileFormat::SPF_Compact_Binary); + testRoundTrip(SampleProfileFormat::SPF_Compact_Binary, false); +} + +TEST_F(SampleProfTest, remap_text_profile) { + testRoundTrip(SampleProfileFormat::SPF_Text, true); } TEST_F(SampleProfTest, sample_overflow_saturation) {