Index: llvm/include/llvm/ProfileData/SampleProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfReader.h +++ llvm/include/llvm/ProfileData/SampleProfReader.h @@ -235,6 +235,62 @@ namespace sampleprof { +class SampleProfileReader; + +/// SampleProfileReaderItaniumRemapper remaps the profile data from a +/// sample profile data reader, by applying a provided set of equivalences +/// between components of the symbol names in the profile. +class SampleProfileReaderItaniumRemapper { +public: + SampleProfileReaderItaniumRemapper(std::unique_ptr B, + std::unique_ptr SRR, + SampleProfileReader &R) + : Buffer(std::move(B)), Remappings(std::move(SRR)), Reader(R) { + assert(Remappings && "Remappings cannot be nullptr"); + } + + /// Create a remapper from the given remapping file. The remapper will + /// be used for profile read in by Reader. + static ErrorOr> + create(const std::string Filename, SampleProfileReader &Reader, + LLVMContext &C); + + /// Create a remapper from the given Buffer. The remapper will + /// be used for profile read in by Reader. + static ErrorOr> + create(std::unique_ptr &B, SampleProfileReader &Reader, + LLVMContext &C); + + /// Apply remappings to the profile read by Reader. + void applyRemapping(LLVMContext &Ctx); + + bool hasApplied() { return RemappingApplied; } + + /// Insert function name into remapper. + void insert(StringRef FunctionName) { Remappings->insert(FunctionName); } + + /// Query whether there is equivalent in the remapper which has been + /// inserted. + bool exist(StringRef FunctionName) { + return Remappings->lookup(FunctionName); + } + + /// Return the samples collected for function \p F if remapper knows + /// it is present in SampleMap. + FunctionSamples *getSamplesFor(StringRef FunctionName); + +private: + // The buffer holding the content read from remapping file. + std::unique_ptr Buffer; + std::unique_ptr Remappings; + DenseMap SampleMap; + // The Reader the remapper is servicing. + SampleProfileReader &Reader; + // Indicate whether remapping has been applied to the profile read + // by Reader -- by calling applyRemapping. + bool RemappingApplied = false; +}; + /// Sample-based profile reader. /// /// Each profile contains sample counts for all the functions @@ -273,8 +329,17 @@ /// Read and validate the file header. virtual std::error_code readHeader() = 0; - /// Read sample profiles from the associated file. - virtual std::error_code read() = 0; + /// The interface to read sample profiles from the associated file. + std::error_code read() { + if (std::error_code EC = readImpl()) + return EC; + if (Remapper) + Remapper->applyRemapping(Ctx); + return sampleprof_error::success; + } + + /// The implementaion to read sample profiles from the associated file. + virtual std::error_code readImpl() = 0; /// Print the profile for \p FName on stream \p OS. void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); @@ -295,6 +360,10 @@ /// Return the samples collected for function \p F. virtual FunctionSamples *getSamplesFor(StringRef Fname) { + if (Remapper) { + if (auto FS = Remapper->getSamplesFor(Fname)) + return FS; + } std::string FGUID; Fname = getRepInFormat(Fname, getFormat(), FGUID); auto It = Profiles.find(Fname); @@ -313,18 +382,24 @@ } /// Create a sample profile reader appropriate to the file format. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr> - create(const Twine &Filename, LLVMContext &C); + create(const std::string Filename, LLVMContext &C, + const std::string RemapFilename = ""); /// Create a sample profile reader from the supplied memory buffer. + /// Create a remapper underlying if RemapFilename is not empty. static ErrorOr> - create(std::unique_ptr &B, LLVMContext &C); + create(std::unique_ptr &B, LLVMContext &C, + const std::string RemapFilename = ""); /// Return the profile summary. - ProfileSummary &getSummary() { return *(Summary.get()); } + ProfileSummary &getSummary() const { return *(Summary.get()); } + + MemoryBuffer *getBuffer() const { return Buffer.get(); } /// \brief Return the profile format. - SampleProfileFormat getFormat() { return Format; } + SampleProfileFormat getFormat() const { return Format; } virtual std::unique_ptr getProfileSymbolList() { return nullptr; @@ -361,6 +436,8 @@ /// Compute summary for this profile. void computeSummary(); + std::unique_ptr Remapper; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; @@ -374,7 +451,7 @@ std::error_code readHeader() override { return sampleprof_error::success; } /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -390,7 +467,7 @@ virtual std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// It includes all the names that have samples either in outline instance /// or inline instance. @@ -512,7 +589,7 @@ : SampleProfileReaderBinary(std::move(B), C, Format) {} /// Read sample profiles in extensible format from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Get the total size of all \p Type sections. uint64_t getSectionSize(SecType Type); @@ -581,7 +658,7 @@ static bool hasFormat(const MemoryBuffer &Buffer); /// Read samples only for functions to use. - std::error_code read() override; + std::error_code readImpl() override; /// Collect functions to be used when compiling Module \p M. void collectFuncsFrom(const Module &M) override; @@ -612,7 +689,7 @@ std::error_code readHeader() override; /// Read sample profiles from the associated file. - std::error_code read() override; + std::error_code readImpl() override; /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -640,44 +717,6 @@ static const uint32_t GCOVTagAFDOFunction = 0xac000000; }; -/// A profile data reader proxy that remaps the profile data from another -/// sample profile data reader, by applying a provided set of equivalences -/// between components of the symbol names in the profile. -class SampleProfileReaderItaniumRemapper : public SampleProfileReader { -public: - SampleProfileReaderItaniumRemapper( - std::unique_ptr B, LLVMContext &C, - std::unique_ptr Underlying) - : SampleProfileReader(std::move(B), C, Underlying->getFormat()) { - Profiles = std::move(Underlying->getProfiles()); - Summary = takeSummary(*Underlying); - // Keep the underlying reader alive; the profile data may contain - // StringRefs referencing names in its name table. - UnderlyingReader = std::move(Underlying); - } - - /// Create a remapped sample profile from the given remapping file and - /// underlying samples. - static ErrorOr> - create(const Twine &Filename, LLVMContext &C, - std::unique_ptr Underlying); - - /// Read and validate the file header. - std::error_code readHeader() override { return sampleprof_error::success; } - - /// Read remapping file and apply it to the sample profile. - std::error_code read() override; - - /// Return the samples collected for function \p F. - FunctionSamples *getSamplesFor(StringRef FunctionName) override; - using SampleProfileReader::getSamplesFor; - -private: - SymbolRemappingReader Remappings; - DenseMap SampleMap; - std::unique_ptr UnderlyingReader; -}; - } // end namespace sampleprof } // end namespace llvm Index: llvm/lib/ProfileData/SampleProfReader.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfReader.cpp +++ llvm/lib/ProfileData/SampleProfReader.cpp @@ -191,7 +191,7 @@ /// the expected format. /// /// \returns true if the file was loaded successfully, false otherwise. -std::error_code SampleProfileReaderText::read() { +std::error_code SampleProfileReaderText::readImpl() { line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); sampleprof_error Result = sampleprof_error::success; @@ -461,7 +461,7 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderBinary::read() { +std::error_code SampleProfileReaderBinary::readImpl() { while (!at_eof()) { if (std::error_code EC = readFuncProfile(Data)) return EC; @@ -540,15 +540,23 @@ return sampleprof_error::success; } - for (auto Name : FuncsToUse) { - auto iter = FuncOffsetTable.find(Name); - if (iter == FuncOffsetTable.end()) + if (Remapper) { + for (auto Name : FuncsToUse) { + Remapper->insert(Name); + } + } + + for (auto NameOffset : FuncOffsetTable) { + auto FuncName = NameOffset.first; + if (!FuncsToUse.count(FuncName) && + (!Remapper || !Remapper->exist(FuncName))) continue; - const uint8_t *FuncProfileAddr = Start + iter->second; + const uint8_t *FuncProfileAddr = Start + NameOffset.second; assert(FuncProfileAddr < End && "out of LBRProfile section"); if (std::error_code EC = readFuncProfile(FuncProfileAddr)) return EC; } + Data = End; return sampleprof_error::success; } @@ -593,7 +601,7 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinaryBase::read() { +std::error_code SampleProfileReaderExtBinaryBase::readImpl() { const uint8_t *BufStart = reinterpret_cast(Buffer->getBufferStart()); @@ -635,7 +643,7 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderCompactBinary::read() { +std::error_code SampleProfileReaderCompactBinary::readImpl() { std::vector OffsetsToUse; if (UseAllFuncs) { for (auto FuncEntry : FuncOffsetTable) { @@ -1184,7 +1192,7 @@ /// /// This format is generated by the Linux Perf conversion tool at /// https://github.com/google/autofdo. -std::error_code SampleProfileReaderGCC::read() { +std::error_code SampleProfileReaderGCC::readImpl() { // Read the string table. if (std::error_code EC = readNameTable()) return EC; @@ -1201,38 +1209,31 @@ return Magic == "adcg*704"; } -std::error_code SampleProfileReaderItaniumRemapper::read() { - // If the underlying data is in compact format, we can't remap it because +void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { + // If the reader is in compact format, we can't remap it because // we don't know what the original function names were. - if (getFormat() == SPF_Compact_Binary) { + if (Reader.getFormat() == SPF_Compact_Binary) { Ctx.diagnose(DiagnosticInfoSampleProfile( - Buffer->getBufferIdentifier(), + Reader.getBuffer()->getBufferIdentifier(), "Profile data remapping cannot be applied to profile data " "in compact format (original mangled names are not available).", DS_Warning)); - return sampleprof_error::success; - } - - if (Error E = Remappings.read(*Buffer)) { - handleAllErrors( - std::move(E), [&](const SymbolRemappingParseError &ParseError) { - reportError(ParseError.getLineNum(), ParseError.getMessage()); - }); - return sampleprof_error::malformed; + return; } - for (auto &Sample : getProfiles()) - if (auto Key = Remappings.insert(Sample.first())) + assert(Remappings && "should be initialized while creating remapper"); + for (auto &Sample : Reader.getProfiles()) + if (auto Key = Remappings->insert(Sample.first())) SampleMap.insert({Key, &Sample.second}); - return sampleprof_error::success; + RemappingApplied = true; } FunctionSamples * SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) { - if (auto Key = Remappings.lookup(Fname)) + if (auto Key = Remappings->lookup(Fname)) return SampleMap.lookup(Key); - return SampleProfileReader::getSamplesFor(Fname); + return nullptr; } /// Prepare a memory buffer for the contents of \p Filename. @@ -1258,13 +1259,16 @@ /// /// \param C The LLVM context to use to emit diagnostics. /// +/// \param RemapFilename The file used for profile remapping. +/// /// \returns an error code indicating the status of the created reader. ErrorOr> -SampleProfileReader::create(const Twine &Filename, LLVMContext &C) { +SampleProfileReader::create(const std::string Filename, LLVMContext &C, + const std::string RemapFilename) { auto BufferOrError = setupMemoryBuffer(Filename); if (std::error_code EC = BufferOrError.getError()) return EC; - return create(BufferOrError.get(), C); + return create(BufferOrError.get(), C, RemapFilename); } /// Create a sample profile remapper from the given input, to remap the @@ -1272,20 +1276,48 @@ /// /// \param Filename The file to open. /// -/// \param C The LLVM context to use to emit diagnostics. +/// \param Reader The profile reader the remapper is going to be applied to. /// -/// \param Underlying The underlying profile data reader to remap. +/// \param C The LLVM context to use to emit diagnostics. /// /// \returns an error code indicating the status of the created reader. -ErrorOr> -SampleProfileReaderItaniumRemapper::create( - const Twine &Filename, LLVMContext &C, - std::unique_ptr Underlying) { +ErrorOr> +SampleProfileReaderItaniumRemapper::create(const std::string Filename, + SampleProfileReader &Reader, + LLVMContext &C) { auto BufferOrError = setupMemoryBuffer(Filename); if (std::error_code EC = BufferOrError.getError()) return EC; + return create(BufferOrError.get(), Reader, C); +} + +/// Create a sample profile remapper from the given input, to remap the +/// function names in the given profile data. +/// +/// \param B The memory buffer to create the reader from (assumes ownership). +/// +/// \param C The LLVM context to use to emit diagnostics. +/// +/// \param Reader The profile reader the remapper is going to be applied to. +/// +/// \returns an error code indicating the status of the created reader. +ErrorOr> +SampleProfileReaderItaniumRemapper::create(std::unique_ptr &B, + SampleProfileReader &Reader, + LLVMContext &C) { + auto Remappings = std::make_unique(); + if (Error E = Remappings->read(*B.get())) { + handleAllErrors( + std::move(E), [&](const SymbolRemappingParseError &ParseError) { + C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(), + ParseError.getLineNum(), + ParseError.getMessage())); + }); + return sampleprof_error::malformed; + } + return std::make_unique( - std::move(BufferOrError.get()), C, std::move(Underlying)); + std::move(B), std::move(Remappings), Reader); } /// Create a sample profile reader based on the format of the input data. @@ -1294,9 +1326,12 @@ /// /// \param C The LLVM context to use to emit diagnostics. /// +/// \param RemapFilename The file used for profile remapping. +/// /// \returns an error code indicating the status of the created reader. ErrorOr> -SampleProfileReader::create(std::unique_ptr &B, LLVMContext &C) { +SampleProfileReader::create(std::unique_ptr &B, LLVMContext &C, + const std::string RemapFilename) { std::unique_ptr Reader; if (SampleProfileReaderRawBinary::hasFormat(*B)) Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C)); @@ -1311,6 +1346,17 @@ else return sampleprof_error::unrecognized_format; + if (!RemapFilename.empty()) { + auto ReaderOrErr = + SampleProfileReaderItaniumRemapper::create(RemapFilename, *Reader, C); + if (std::error_code EC = ReaderOrErr.getError()) { + std::string Msg = "Could not create remapper: " + EC.message(); + C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg)); + return EC; + } + Reader->Remapper = std::move(ReaderOrErr.get()); + } + FunctionSamples::Format = Reader->getFormat(); if (std::error_code EC = Reader->readHeader()) { return EC; Index: llvm/lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- llvm/lib/Transforms/IPO/SampleProfile.cpp +++ llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1675,7 +1675,10 @@ bool SampleProfileLoader::doInitialization(Module &M) { auto &Ctx = M.getContext(); - auto ReaderOrErr = SampleProfileReader::create(Filename, Ctx); + + std::unique_ptr RemapReader; + auto ReaderOrErr = + SampleProfileReader::create(Filename, Ctx, RemappingFilename); if (std::error_code EC = ReaderOrErr.getError()) { std::string Msg = "Could not open profile: " + EC.message(); Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); @@ -1695,20 +1698,6 @@ NamesInProfile.insert(NameTable->begin(), NameTable->end()); } - if (!RemappingFilename.empty()) { - // Apply profile remappings to the loaded profile data if requested. - // For now, we only support remapping symbols encoded using the Itanium - // C++ ABI's name mangling scheme. - ReaderOrErr = SampleProfileReaderItaniumRemapper::create( - RemappingFilename, Ctx, std::move(Reader)); - if (std::error_code EC = ReaderOrErr.getError()) { - std::string Msg = "Could not open profile remapping file: " + EC.message(); - Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); - return false; - } - Reader = std::move(ReaderOrErr.get()); - ProfileIsValid = (Reader->read() == sampleprof_error::success); - } return true; } Index: llvm/test/Transforms/SampleProfile/remap.ll =================================================================== --- llvm/test/Transforms/SampleProfile/remap.ll +++ llvm/test/Transforms/SampleProfile/remap.ll @@ -1,5 +1,9 @@ ; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s - +; +; Check whether profile remapping work with loading profile on demand used by extbinary format profile. +; RUN: llvm-profdata merge -sample -extbinary %S/Inputs/remap.prof -o %t.extbinary.afdo +; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s +; ; Reduced from branch.ll declare i1 @foo() Index: llvm/unittests/ProfileData/SampleProfTest.cpp =================================================================== --- llvm/unittests/ProfileData/SampleProfTest.cpp +++ llvm/unittests/ProfileData/SampleProfTest.cpp @@ -50,13 +50,31 @@ Writer = std::move(WriterOrErr.get()); } - void readProfile(const Module &M, StringRef Profile) { - auto ReaderOrErr = SampleProfileReader::create(Profile, Context); + void readProfile(const Module &M, StringRef Profile, + StringRef RemapFile = "") { + auto ReaderOrErr = SampleProfileReader::create(Profile, Context, RemapFile); ASSERT_TRUE(NoError(ReaderOrErr.getError())); Reader = std::move(ReaderOrErr.get()); Reader->collectFuncsFrom(M); } + void createRemapFile(SmallVectorImpl &RemapPath, StringRef &RemapFile) { + std::error_code EC = + llvm::sys::fs::createTemporaryFile("remapfile", "", RemapPath); + ASSERT_TRUE(NoError(EC)); + RemapFile = StringRef(RemapPath.data(), RemapPath.size()); + + std::unique_ptr OS( + new raw_fd_ostream(RemapFile, EC, sys::fs::OF_None)); + *OS << R"( + # Types 'int' and 'long' are equivalent + type i l + # Function names 'foo' and 'faux' are equivalent + name 3foo 4faux + )"; + OS->close(); + } + void testRoundTrip(SampleProfileFormat Format, bool Remap) { SmallVector ProfilePath; ASSERT_TRUE(NoError(llvm::sys::fs::createTemporaryFile("profile", "", ProfilePath))); @@ -93,16 +111,34 @@ BazSamples.addHeadSamples(1257); BazSamples.addBodySamples(1, 0, 12557); - Module M("my_module", Context); - FunctionType *fn_type = - FunctionType::get(Type::getVoidTy(Context), {}, false); - M.getOrInsertFunction(FooName, fn_type); - M.getOrInsertFunction(BarName, fn_type); + StringRef BooName("_Z3booi"); + FunctionSamples BooSamples; + BooSamples.setName(BooName); + BooSamples.addTotalSamples(1232); + BooSamples.addHeadSamples(1); + BooSamples.addBodySamples(1, 0, 1232); StringMap Profiles; Profiles[FooName] = std::move(FooSamples); Profiles[BarName] = std::move(BarSamples); Profiles[BazName] = std::move(BazSamples); + Profiles[BooName] = std::move(BooSamples); + + Module M("my_module", Context); + FunctionType *fn_type = + FunctionType::get(Type::getVoidTy(Context), {}, false); + + SmallVector RemapPath; + StringRef RemapFile; + if (Remap) { + createRemapFile(RemapPath, RemapFile); + FooName = "_Z4fauxi"; + BarName = "_Z3barl"; + } + + M.getOrInsertFunction(FooName, fn_type); + M.getOrInsertFunction(BarName, fn_type); + M.getOrInsertFunction(BooName, fn_type); ProfileSymbolList List; if (Format == SampleProfileFormat::SPF_Ext_Binary) { @@ -117,8 +153,7 @@ Writer->getOutputStream().flush(); - readProfile(M, Profile); - + readProfile(M, Profile, RemapFile); EC = Reader->read(); ASSERT_TRUE(NoError(EC)); @@ -129,22 +164,6 @@ ReaderList->contains("moo"); } - if (Remap) { - auto MemBuffer = llvm::MemoryBuffer::getMemBuffer(R"( - # Types 'int' and 'long' are equivalent - type i l - # Function names 'foo' and 'faux' are equivalent - name 3foo 4faux - )"); - Reader.reset(new SampleProfileReaderItaniumRemapper( - std::move(MemBuffer), Context, std::move(Reader))); - FooName = "_Z4fauxi"; - BarName = "_Z3barl"; - - EC = Reader->read(); - ASSERT_TRUE(NoError(EC)); - } - FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName); ASSERT_TRUE(ReadFooSamples != nullptr); if (Format != SampleProfileFormat::SPF_Compact_Binary) { @@ -171,13 +190,17 @@ if (Format == SampleProfileFormat::SPF_Ext_Binary || Format == SampleProfileFormat::SPF_Compact_Binary) { ASSERT_TRUE(ReadBazSamples == nullptr); - ASSERT_EQ(2u, Reader->getProfiles().size()); + ASSERT_EQ(3u, Reader->getProfiles().size()); } else { ASSERT_TRUE(ReadBazSamples != nullptr); ASSERT_EQ(12557u, ReadBazSamples->getTotalSamples()); - ASSERT_EQ(3u, Reader->getProfiles().size()); + ASSERT_EQ(4u, Reader->getProfiles().size()); } + FunctionSamples *ReadBooSamples = Reader->getSamplesFor(BooName); + ASSERT_TRUE(ReadBooSamples != nullptr); + ASSERT_EQ(1232u, ReadBooSamples->getTotalSamples()); + std::string MconstructGUID; StringRef MconstructRep = getRepInFormat(MconstructName, Format, MconstructGUID); @@ -189,9 +212,9 @@ auto VerifySummary = [](ProfileSummary &Summary) mutable { ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind()); - ASSERT_EQ(136160u, Summary.getTotalCount()); - ASSERT_EQ(7u, Summary.getNumCounts()); - ASSERT_EQ(3u, Summary.getNumFunctions()); + ASSERT_EQ(137392u, Summary.getTotalCount()); + ASSERT_EQ(8u, Summary.getNumCounts()); + ASSERT_EQ(4u, Summary.getNumFunctions()); ASSERT_EQ(1437u, Summary.getMaxFunctionCount()); ASSERT_EQ(60351u, Summary.getMaxCount());