diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h --- a/llvm/include/llvm/IR/PseudoProbe.h +++ b/llvm/include/llvm/IR/PseudoProbe.h @@ -13,11 +13,14 @@ #ifndef LLVM_IR_PSEUDOPROBE_H #define LLVM_IR_PSEUDOPROBE_H +#include "llvm/ADT/Optional.h" #include #include namespace llvm { +class Instruction; + constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc"; enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall }; @@ -49,6 +52,15 @@ return (Value >> 29) & 0x7; } }; + +struct PseudoProbe { + uint32_t Id; + uint32_t Type; + uint32_t Attr; +}; + +Optional extractProbe(const Instruction &Inst); + } // end namespace llvm #endif // LLVM_IR_PSEUDOPROBE_H diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -54,7 +54,8 @@ ostream_seek_unsupported, compress_failed, uncompress_failed, - zlib_unavailable + zlib_unavailable, + hash_mismatch }; inline std::error_code make_error_code(sampleprof_error E) { @@ -120,6 +121,7 @@ SecNameTable = 2, SecProfileSymbolList = 3, SecFuncOffsetTable = 4, + SecFuncMetadata = 5, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -137,6 +139,8 @@ return "ProfileSymbolListSection"; case SecFuncOffsetTable: return "FuncOffsetTableSection"; + case SecFuncMetadata: + return "FunctionMetadata"; case SecLBRProfile: return "LBRProfileSection"; } @@ -178,6 +182,11 @@ SecFlagPartial = (1 << 0) }; +enum class SecFuncMetadataFlags : uint32_t { + SecFlagInvalid = 0, + SecFlagIsProbeBased = (1 << 0), +}; + // Verify section specific flag is used for the correct section. template static inline void verifySecFlag(SecType Type, SecFlagType Flag) { @@ -194,6 +203,9 @@ case SecProfSummary: IsFlagLegal = std::is_same(); break; + case SecFuncMetadata: + IsFlagLegal = std::is_same(); + break; default: break; } @@ -502,6 +514,8 @@ : sampleprof_error::success; } + void setTotalSamples(uint64_t Num) { TotalSamples = Num; } + sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) { bool Overflowed; TotalHeadSamples = @@ -537,6 +551,12 @@ if (ProfileIsCS) return 0; return std::error_code(); + // A missing entry for a probe likely means the probe was not executed. + // Treat it as a zero count instead of an unknown count to help edge + // weight inference. + if (FunctionSamples::ProfileIsProbeBased) + return 0; + return std::error_code(); } else { return ret->second.getSamples(); } @@ -553,6 +573,16 @@ return ret->second.getCallTargets(); } + /// Returns the call target map collected at a given location specified by \p + /// CallSite. If the location is not found in profile, return error. + ErrorOr + findCallTargetMapAt(const LineLocation &CallSite) const { + const auto &ret = BodySamples.find(CallSite); + if (ret == BodySamples.end()) + return std::error_code(); + return ret->second.getCallTargets(); + } + /// Return the function samples at the given callsite location. FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) { return CallsiteSamples[Loc]; @@ -641,6 +671,21 @@ Name = Other.getName(); if (!GUIDToFuncNameMap) GUIDToFuncNameMap = Other.GUIDToFuncNameMap; + + if (FunctionHash == 0) { + // Set the function hash code for the target profile. + FunctionHash = Other.getFunctionHash(); + } else if (FunctionHash != Other.getFunctionHash()) { + // The two profiles coming with different valid hash codes indicates + // either: + // 1. They are same-named static functions from different compilation + // units, or + // 2. They are really the same function but from different compilations. + // Let's bail out in either case for now, which means one profile is + // dropped. + return sampleprof_error::hash_mismatch; + } + MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight)); MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight)); for (const auto &I : Other.getBodySamples()) { @@ -700,6 +745,10 @@ /// Return the original function name. StringRef getFuncName() const { return getFuncName(Name); } + void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; } + + uint64_t getFunctionHash() const { return FunctionHash; } + /// Return the canonical name for a function, taking into account /// suffix elision policy attributes. static StringRef getCanonicalFnName(const Function &F) { @@ -754,6 +803,12 @@ /// We assume that a single function will not exceed 65535 LOC. static unsigned getOffset(const DILocation *DIL); + /// Returns a unique call site identifier for a given debug location of a call + /// instruction. This is wrapper of two sceanarios, the hybrid profile and + /// regular profile, to hide implementation details from the sample loader and + /// the context tracker. + static LineLocation getCallSiteIdentifier(const DILocation *DIL); + /// Get the FunctionSamples of the inline instance where DIL originates /// from. /// @@ -769,6 +824,8 @@ const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper = nullptr) const; + static bool ProfileIsProbeBased; + static bool ProfileIsCS; SampleContext &getContext() const { return Context; } @@ -799,6 +856,9 @@ /// Mangled name of the function. StringRef Name; + /// CFG hash value for the function. + uint64_t FunctionHash = 0; + /// Calling context for function profile mutable SampleContext Context; diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -27,8 +27,9 @@ // offsetA[.discriminator]: fnA:num_of_total_samples // offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ] // ... +// !CFGChecksum: num // -// This is a nested tree in which the identation represents the nesting level +// This is a nested tree in which the indentation represents the nesting level // of the inline stack. There are no blank lines in the file. And the spacing // within a single line is fixed. Additional spaces will result in an error // while reading the file. @@ -47,10 +48,11 @@ // in the prologue of the function (second number). This head sample // count provides an indicator of how frequently the function is invoked. // -// There are two types of lines in the function body. +// There are three types of lines in the function body. // // * Sampled line represents the profile information of a source location. // * Callsite line represents the profile information of a callsite. +// * Metadata line represents extra metadata of the function. // // Each sampled line may contain several items. Some are optional (marked // below): @@ -114,6 +116,18 @@ // total number of samples collected for the inlined instance at this // callsite // +// Metadata line can occur in lines with one indent only, containing extra +// information for the top-level function. Furthermore, metadata can only +// occur after all the body samples and callsite samples. +// Each metadata line may contain a particular type of metadata, marked by +// the starting characters annotated with !. We process each metadata line +// independently, hence each metadata line has to form an independent piece +// of information that does not require cross-line reference. +// We support the following types of metadata: +// +// a. CFG Checksum (a.k.a. function hash): +// !CFGChecksum: 12345 +// // // Binary format // ------------- @@ -419,7 +433,10 @@ /// \brief Return the profile format. SampleProfileFormat getFormat() const { return Format; } - /// Whether input profile is fully context-sensitie + /// Whether input profile is based on pseudo probes. + bool profileIsProbeBased() const { return ProfileIsProbeBased; } + + /// Whether input profile is fully context-sensitive bool profileIsCS() const { return ProfileIsCS; } virtual std::unique_ptr getProfileSymbolList() { @@ -464,6 +481,9 @@ std::unique_ptr Remapper; + /// \brief Whether samples are collected based on pseudo probes. + bool ProfileIsProbeBased = false; + bool ProfileIsCS = false; /// \brief The format of sample. @@ -606,6 +626,7 @@ std::error_code readSecHdrTableEntry(); std::error_code readSecHdrTable(); + std::error_code readFuncMetadata(); std::error_code readFuncOffsetTable(); std::error_code readFuncProfiles(); std::error_code readMD5NameTable(); diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -200,6 +200,8 @@ // Helper function to write name table. virtual std::error_code writeNameTable() override; + std::error_code writeFuncMetadata(const StringMap &Profiles); + // Functions to write various kinds of sections. std::error_code writeNameTableSection(const StringMap &ProfileMap); @@ -270,11 +272,10 @@ // SecFuncOffsetTable section is written after SecLBRProfile in the // profile because FuncOffsetTable needs to be populated while section // SecLBRProfile is written. - SectionHdrLayout = {{SecProfSummary, 0, 0, 0}, - {SecNameTable, 0, 0, 0}, - {SecFuncOffsetTable, 0, 0, 0}, - {SecLBRProfile, 0, 0, 0}, - {SecProfileSymbolList, 0, 0, 0}}; + SectionHdrLayout = { + {SecProfSummary, 0, 0, 0}, {SecNameTable, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0}, {SecLBRProfile, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0}, {SecFuncMetadata, 0, 0, 0}}; }; virtual std::error_code writeSections(const StringMap &ProfileMap) override; diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h --- a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PseudoProbe.h" +#include "llvm/ProfileData/SampleProf.h" #include "llvm/Target/TargetMachine.h" #include @@ -25,11 +26,36 @@ class Module; +using namespace sampleprof; using BlockIdMap = std::unordered_map; using InstructionIdMap = std::unordered_map; enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid }; +class PseudoProbeDescriptor { + uint64_t FunctionGUID; + uint64_t FunctionHash; + +public: + PseudoProbeDescriptor(uint64_t GUID, uint64_t Hash) + : FunctionGUID(GUID), FunctionHash(Hash) {} + uint64_t getFunctionGUID() const { return FunctionGUID; } + uint64_t getFunctionHash() const { return FunctionHash; } +}; + +// This class serves sample counts correlation for SampleProfileLoader by +// analyzing pseudo probes and their function descriptors injected by +// SampleProfileProber. +class PseudoProbeManager { + DenseMap GUIDToProbeDescMap; + + const PseudoProbeDescriptor *getDesc(const Function &F) const; + +public: + PseudoProbeManager(const Module &M); + bool moduleIsProbed(const Module &M) const; + bool profileIsValid(const Function &F, const FunctionSamples &Samples) const; +}; /// Sample profile pseudo prober. /// diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt --- a/llvm/lib/IR/CMakeLists.txt +++ b/llvm/lib/IR/CMakeLists.txt @@ -47,6 +47,7 @@ PrintPasses.cpp SafepointIRVerifier.cpp ProfileSummary.cpp + PseudoProbe.cpp Statepoint.cpp StructuralHash.cpp Type.cpp diff --git a/llvm/lib/IR/PseudoProbe.cpp b/llvm/lib/IR/PseudoProbe.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/IR/PseudoProbe.cpp @@ -0,0 +1,50 @@ +//===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the helpers to manipulate pseudo probe IR intrinsic +// calls. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/PseudoProbe.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" + +using namespace llvm; + +namespace llvm { + +Optional extractProbe(const Instruction &Inst) { + if (auto *II = dyn_cast(&Inst)) { + PseudoProbe Probe; + Probe.Id = II->getIndex()->getZExtValue(); + Probe.Type = (uint32_t)PseudoProbeType::Block; + Probe.Attr = II->getAttributes()->getZExtValue(); + return Probe; + } + + if (isa(&Inst) && !isa(&Inst)) { + if (const DebugLoc &DLoc = Inst.getDebugLoc()) { + const DILocation *DIL = DLoc; + auto Discriminator = DIL->getDiscriminator(); + if (DILocation::isPseudoProbeDiscriminator(Discriminator)) { + PseudoProbe Probe; + Probe.Id = + PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator); + Probe.Type = + PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator); + Probe.Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes( + Discriminator); + return Probe; + } + } + } + return None; +} +} // namespace llvm diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -14,6 +14,7 @@ #include "llvm/ProfileData/SampleProf.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/PseudoProbe.h" #include "llvm/ProfileData/SampleProfReader.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" @@ -31,6 +32,7 @@ namespace llvm { namespace sampleprof { SampleProfileFormat FunctionSamples::Format; +bool FunctionSamples::ProfileIsProbeBased = false; bool FunctionSamples::ProfileIsCS = false; bool FunctionSamples::UseMD5; } // namespace sampleprof @@ -77,6 +79,8 @@ return "Uncompress failure"; case sampleprof_error::zlib_unavailable: return "Zlib is unavailable"; + case sampleprof_error::hash_mismatch: + return "Function hash mismatch"; } llvm_unreachable("A value of sampleprof_error has no message."); } @@ -176,6 +180,20 @@ 0xffff; } +LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) { + if (FunctionSamples::ProfileIsProbeBased) + // In a pseudo-probe based profile, a callsite is simply represented by the + // ID of the probe associated with the call instruction. The probe ID is + // encoded in the Discriminator field of the call instruction's debug + // metadata. + return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex( + DIL->getDiscriminator()), + 0); + else + return LineLocation(FunctionSamples::getOffset(DIL), + DIL->getBaseDiscriminator()); +} + const FunctionSamples *FunctionSamples::findFunctionSamples( const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const { assert(DIL); diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -83,26 +83,52 @@ /// Returns true if line offset \p L is legal (only has 16 bits). static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } +/// Parse \p Input that contains metadata. +/// Possible metadata: +/// - CFG Checksum information: +/// !CFGChecksum: 12345 +/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash. +static bool ParseMetadata(const StringRef &Input, uint64_t &FunctionHash) { + if (!Input.startswith("!CFGChecksum:")) + return false; + + StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim(); + return !CFGInfo.getAsInteger(10, FunctionHash); +} + +enum class LineType { + CallSiteProfile, + BodyProfile, + Metadata, +}; + /// Parse \p Input as line sample. /// /// \param Input input line. -/// \param IsCallsite true if the line represents an inlined callsite. +/// \param LineTy Type of this line. /// \param Depth the depth of the inline stack. /// \param NumSamples total samples of the line/inlined callsite. /// \param LineOffset line offset to the start of the function. /// \param Discriminator discriminator of the line. /// \param TargetCountMap map from indirect call target to count. +/// \param FunctionHash the function's CFG hash, used by pseudo probe. /// /// returns true if parsing is successful. -static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, +static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, - DenseMap &TargetCountMap) { + DenseMap &TargetCountMap, + uint64_t &FunctionHash) { for (Depth = 0; Input[Depth] == ' '; Depth++) ; if (Depth == 0) return false; + if (Depth == 1 && Input[Depth] == '!') { + LineTy = LineType::Metadata; + return ParseMetadata(Input.substr(Depth), FunctionHash); + } + size_t n1 = Input.find(':'); StringRef Loc = Input.substr(Depth, n1 - Depth); size_t n2 = Loc.find('.'); @@ -119,7 +145,7 @@ StringRef Rest = Input.substr(n1 + 2); if (Rest[0] >= '0' && Rest[0] <= '9') { - IsCallsite = false; + LineTy = LineType::BodyProfile; size_t n3 = Rest.find(' '); if (n3 == StringRef::npos) { if (Rest.getAsInteger(10, NumSamples)) @@ -176,7 +202,7 @@ n3 = n4; } } else { - IsCallsite = true; + LineTy = LineType::CallSiteProfile; size_t n3 = Rest.find_last_of(':'); CalleeName = Rest.substr(0, n3); if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) @@ -198,6 +224,11 @@ InlineCallStack InlineStack; int CSProfileCount = 0; int RegularProfileCount = 0; + uint32_t ProbeProfileCount = 0; + + // SeenMetadata tracks whether we have processed metadata for the current + // top-level function profile. + bool SeenMetadata = false; for (; !LineIt.is_at_eof(); ++LineIt) { if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') @@ -222,6 +253,7 @@ "Expected 'mangled_name:NUM:NUM', found " + *LineIt); return sampleprof_error::malformed; } + SeenMetadata = false; SampleContext FContext(FName); if (FContext.hasContext()) ++CSProfileCount; @@ -239,25 +271,35 @@ uint64_t NumSamples; StringRef FName; DenseMap TargetCountMap; - bool IsCallsite; uint32_t Depth, LineOffset, Discriminator; - if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset, - Discriminator, FName, TargetCountMap)) { + LineType LineTy; + uint64_t FunctionHash; + if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, + Discriminator, FName, TargetCountMap, FunctionHash)) { reportError(LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); return sampleprof_error::malformed; } - if (IsCallsite) { - while (InlineStack.size() > Depth) { - InlineStack.pop_back(); - } + if (SeenMetadata && LineTy != LineType::Metadata) { + // Metadata must be put at the end of a function profile. + reportError(LineIt.line_number(), + "Found non-metadata after metadata: " + *LineIt); + return sampleprof_error::malformed; + } + while (InlineStack.size() > Depth) { + InlineStack.pop_back(); + } + switch (LineTy) { + case LineType::CallSiteProfile: { FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( LineLocation(LineOffset, Discriminator))[std::string(FName)]; FSamples.setName(FName); MergeResult(Result, FSamples.addTotalSamples(NumSamples)); InlineStack.push_back(&FSamples); - } else { + break; + } + case LineType::BodyProfile: { while (InlineStack.size() > Depth) { InlineStack.pop_back(); } @@ -269,6 +311,15 @@ } MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, NumSamples)); + break; + } + case LineType::Metadata: { + FunctionSamples &FProfile = *InlineStack.back(); + FProfile.setFunctionHash(FunctionHash); + ++ProbeProfileCount; + SeenMetadata = true; + break; + } } } } @@ -276,6 +327,10 @@ assert((RegularProfileCount == 0 || CSProfileCount == 0) && "Cannot have both context-sensitive and regular profile"); ProfileIsCS = (CSProfileCount > 0); + assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) && + "Cannot have both probe-based profiles and regular profiles"); + ProfileIsProbeBased = (ProbeProfileCount > 0); + FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; if (Result == sampleprof_error::success) computeSummary(); @@ -540,6 +595,13 @@ if (std::error_code EC = readFuncOffsetTable()) return EC; break; + case SecFuncMetadata: + ProfileIsProbeBased = + hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); + FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + if (std::error_code EC = readFuncMetadata()) + return EC; + break; case SecProfileSymbolList: if (std::error_code EC = readProfileSymbolList()) return EC; @@ -804,6 +866,24 @@ return SampleProfileReaderBinary::readNameTable(); } +std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() { + if (ProfileIsProbeBased) { + for (unsigned i = 0; i < Profiles.size(); ++i) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + + auto Checksum = readNumber(); + if (std::error_code EC = Checksum.getError()) + return EC; + + Profiles[*FName].setFunctionHash(*Checksum); + } + } + + return sampleprof_error::success; +} + std::error_code SampleProfileReaderCompactBinary::readNameTable() { auto Size = readNumber(); if (std::error_code EC = Size.getError()) diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -166,6 +166,18 @@ return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( + const StringMap &Profiles) { + if (FunctionSamples::ProfileIsProbeBased) { + auto &OS = *OutputStream; + for (const auto &Entry : Profiles) { + writeNameIdx(Entry.first()); + encodeULEB128(Entry.second.getFunctionHash(), OS); + } + } + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { if (!UseMD5) return SampleProfileWriterBinary::writeNameTable(); @@ -209,6 +221,8 @@ // The setting of SecFlagCompress should happen before markSectionStart. if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress()) setToCompressSection(SecProfileSymbolList); + if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased) + addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased); uint64_t SectionStart = markSectionStart(Type); switch (Type) { @@ -230,6 +244,10 @@ if (auto EC = writeFuncOffsetTable()) return EC; break; + case SecFuncMetadata: + if (std::error_code EC = writeFuncMetadata(ProfileMap)) + return EC; + break; case SecProfileSymbolList: if (auto EC = writeProfileSymbolListSection()) return EC; @@ -256,6 +274,8 @@ return EC; if (auto EC = writeOneSection(SecFuncOffsetTable, ProfileMap)) return EC; + if (auto EC = writeOneSection(SecFuncMetadata, ProfileMap)) + return EC; return sampleprof_error::success; } @@ -320,6 +340,13 @@ } Indent -= 1; + if (Indent == 0) { + if (FunctionSamples::ProfileIsProbeBased) { + OS.indent(Indent + 1); + OS << "!CFGChecksum: " << S.getFunctionHash() << "\n"; + } + } + return sampleprof_error::success; } diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -77,6 +77,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/SampleContextTracker.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/CallPromotionUtils.h" #include "llvm/Transforms/Utils/Cloning.h" @@ -103,6 +104,9 @@ "Number of functions inlined with context sensitive profile"); STATISTIC(NumCSNotInlined, "Number of functions not inlined with context sensitive profile"); +STATISTIC(NumMismatchedProfile, + "Number of functions with CFG mismatched profile"); +STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile"); // Command line option to specify the file to read samples from. This is // mainly used for debugging. @@ -340,6 +344,7 @@ unsigned getFunctionLoc(Function &F); bool emitAnnotations(Function &F); ErrorOr getInstWeight(const Instruction &I); + ErrorOr getProbeWeight(const Instruction &I); ErrorOr getBlockWeight(const BasicBlock *BB); const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const; std::vector @@ -490,6 +495,9 @@ // External inline advisor used to replay inline decision from remarks. std::unique_ptr ExternalInlineAdvisor; + + // A pseudo probe helper to correlate the imported sample counts. + std::unique_ptr ProbeManager; }; class SampleProfileLoaderLegacyPass : public ModulePass { @@ -722,6 +730,9 @@ /// /// \returns the weight of \p Inst. ErrorOr SampleProfileLoader::getInstWeight(const Instruction &Inst) { + if (FunctionSamples::ProfileIsProbeBased) + return getProbeWeight(Inst); + const DebugLoc &DLoc = Inst.getDebugLoc(); if (!DLoc) return std::error_code(); @@ -775,6 +786,46 @@ return R; } +ErrorOr SampleProfileLoader::getProbeWeight(const Instruction &Inst) { + assert(FunctionSamples::ProfileIsProbeBased); + Optional Probe = extractProbe(Inst); + if (!Probe) + return std::error_code(); + + const FunctionSamples *FS = findFunctionSamples(Inst); + if (!FS) + return std::error_code(); + + // If a direct call/invoke instruction is inlined in profile + // (findCalleeFunctionSamples returns non-empty result), but not inlined here, + // it means that the inlined callsite has no sample, thus the call + // instruction should have 0 count. + if (auto *CB = dyn_cast(&Inst)) + if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) + return 0; + + const ErrorOr &R = FS->findSamplesAt(Probe->Id, 0); + if (R) { + uint64_t Samples = R.get(); + bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples); + if (FirstMark) { + ORE->emit([&]() { + OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst); + Remark << "Applied " << ore::NV("NumSamples", Samples); + Remark << " samples from profile (ProbeId="; + Remark << ore::NV("ProbeId", Probe->Id); + Remark << ")"; + return Remark; + }); + } + + LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Inst + << " - weight: " << R.get() << ")\n"); + return Samples; + } + return R; +} + /// Compute the weight of a basic block. /// /// The weight of basic block \p BB is the maximum weight of all the @@ -848,8 +899,7 @@ if (FS == nullptr) return nullptr; - return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL), - DIL->getBaseDiscriminator()), + return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL), CalleeName, Reader->getRemapper()); } @@ -870,16 +920,13 @@ if (FS == nullptr) return R; - uint32_t LineOffset = FunctionSamples::getOffset(DIL); - uint32_t Discriminator = DIL->getBaseDiscriminator(); - - auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); + auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + auto T = FS->findCallTargetMapAt(CallSite); Sum = 0; if (T) for (const auto &T_C : T.get()) Sum += T_C.second; - if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(LineLocation( - FunctionSamples::getOffset(DIL), DIL->getBaseDiscriminator()))) { + if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) { if (M->empty()) return R; for (const auto &NameFS : *M) { @@ -907,6 +954,12 @@ /// \returns the FunctionSamples pointer to the inlined instance. const FunctionSamples * SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const { + if (FunctionSamples::ProfileIsProbeBased) { + Optional Probe = extractProbe(Inst); + if (!Probe) + return nullptr; + } + const DILocation *DIL = Inst.getDebugLoc(); if (!DIL) return Samples; @@ -1592,13 +1645,11 @@ if (!DLoc) continue; const DILocation *DIL = DLoc; - uint32_t LineOffset = FunctionSamples::getOffset(DIL); - uint32_t Discriminator = DIL->getBaseDiscriminator(); - const FunctionSamples *FS = findFunctionSamples(I); if (!FS) continue; - auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); + auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + auto T = FS->findCallTargetMapAt(CallSite); if (!T || T.get().empty()) continue; SmallVector SortedCallTargets = @@ -1762,11 +1813,22 @@ bool SampleProfileLoader::emitAnnotations(Function &F) { bool Changed = false; - if (getFunctionLoc(F) == 0) - return false; + if (FunctionSamples::ProfileIsProbeBased) { + if (!ProbeManager->profileIsValid(F, *Samples)) { + LLVM_DEBUG( + dbgs() << "Profile is invalid due to CFG mismatch for Function " + << F.getName()); + ++NumMismatchedProfile; + return false; + } + ++NumMatchedProfile; + } else { + if (getFunctionLoc(F) == 0) + return false; - LLVM_DEBUG(dbgs() << "Line number for the first instruction in " - << F.getName() << ": " << getFunctionLoc(F) << "\n"); + LLVM_DEBUG(dbgs() << "Line number for the first instruction in " + << F.getName() << ": " << getFunctionLoc(F) << "\n"); + } DenseSet InlinedGUIDs; Changed |= inlineHotFunctions(F, InlinedGUIDs); @@ -1912,6 +1974,16 @@ std::make_unique(Reader->getProfiles()); } + // Load pseudo probe descriptors for hybrid function samples. + if (Reader->profileIsProbeBased()) { + ProbeManager = std::make_unique(M); + if (!ProbeManager->moduleIsProbed(M)) { + const char *Msg = "Hybrid profile requires SampleProfileProbePass"; + Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); + return false; + } + } + return true; } diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp --- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -35,6 +35,47 @@ STATISTIC(ArtificialDbgLine, "Number of probes that have an artificial debug line"); +PseudoProbeManager::PseudoProbeManager(const Module &M) { + if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) { + for (const auto *Operand : FuncInfo->operands()) { + const auto *MD = cast(Operand); + auto GUID = + mdconst::dyn_extract(MD->getOperand(0))->getZExtValue(); + auto Hash = + mdconst::dyn_extract(MD->getOperand(1))->getZExtValue(); + GUIDToProbeDescMap.insert({GUID, PseudoProbeDescriptor(GUID, Hash)}); + } + } +} + +const PseudoProbeDescriptor * +PseudoProbeManager::getDesc(const Function &F) const { + auto I = GUIDToProbeDescMap.find( + Function::getGUID(FunctionSamples::getCanonicalFnName(F))); + return I == GUIDToProbeDescMap.end() ? nullptr : &I->second; +} + +bool PseudoProbeManager::moduleIsProbed(const Module &M) const { + return M.getNamedMetadata(PseudoProbeDescMetadataName); +} + +bool PseudoProbeManager::profileIsValid(const Function &F, + const FunctionSamples &Samples) const { + auto Desc = getDesc(F); + if (!Desc) { + LLVM_DEBUG(dbgs() << "Probe descriptor missing for Function " << F.getName() + << "\n"); + return false; + } else { + if (Desc->getFunctionHash() != Samples.getFunctionHash()) { + LLVM_DEBUG(dbgs() << "Hash mismatch for Function " << F.getName() + << "\n"); + return false; + } + } + return true; +} + SampleProfileProber::SampleProfileProber(Function &Func, const std::string &CurModuleUniqueId) : F(&Func), CurModuleUniqueId(CurModuleUniqueId) { diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile.prof new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-profile.prof @@ -0,0 +1,8 @@ +foo:3200:13 + 1: 13 + 2: 7 + 3: 6 + 4: 13 + 5: 7 _Z3barv:2 _Z3foov:5 + 6: 6 _Z3barv:4 _Z3foov:2 + !CFGChecksum: 563022570642068 diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile.ll @@ -0,0 +1,115 @@ +; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile.prof -pass-remarks=sample-profile -pass-remarks-output=%t.opt.yaml -S | FileCheck %s +; RUN: FileCheck %s -check-prefix=YAML < %t.opt.yaml + +define dso_local i32 @foo(i32 %x, void (i32)* %f) #0 !dbg !4 { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %0 = load i32, i32* %x.addr, align 4 + %cmp = icmp eq i32 %0, 0 + br i1 %cmp, label %if.then, label %if.else + ; CHECK: br i1 %cmp, label %if.then, label %if.else, !prof ![[PD1:[0-9]+]] + +if.then: + ; CHECK: call {{.*}}, !prof ![[PROF1:[0-9]+]] + call void %f(i32 1) + store i32 1, i32* %retval, align 4 + br label %return + +if.else: + ; CHECK: call {{.*}}, !prof ![[PROF2:[0-9]+]] + call void %f(i32 2) + store i32 2, i32* %retval, align 4 + br label %return + +return: + %1 = load i32, i32* %retval, align 4 + ret i32 %1 +} + +attributes #0 = {"use-sample-profile"} + +; CHECK: ![[PD1]] = !{!"branch_weights", i32 8, i32 7} +; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 7, i64 9191153033785521275, i64 5, i64 -1069303473483922844, i64 2} +; CHECK: ![[PROF2]] = !{!"VP", i32 0, i64 6, i64 -1069303473483922844, i64 4, i64 9191153033785521275, i64 2} + +!llvm.module.flags = !{!9, !10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1) +!1 = !DIFile(filename: "test.c", directory: "") +!2 = !{} +!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, unit: !0, retainedNodes: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{!7} +!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) +!9 = !{i32 2, !"Dwarf Version", i32 4} +!10 = !{i32 2, !"Debug Info Version", i32 3} + +; Checking to see if YAML file is generated and contains remarks +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '13' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '1' +;YAML-NEXT: - String: ')' +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '7' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '5' +;YAML-NEXT: - String: ')' +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '7' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '2' +;YAML-NEXT: - String: ')' +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '6' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '6' +;YAML-NEXT: - String: ')' +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '6' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '3' +;YAML-NEXT: - String: ')' +;YAML: --- !Analysis +;YAML-NEXT: Pass: sample-profile +;YAML-NEXT: Name: AppliedSamples +;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 } +;YAML-NEXT: Function: foo +;YAML-NEXT: Args: +;YAML-NEXT: - String: 'Applied ' +;YAML-NEXT: - NumSamples: '13' +;YAML-NEXT: - String: ' samples from profile (ProbeId=' +;YAML-NEXT: - ProbeId: '4' +;YAML-NEXT: - String: ')' diff --git a/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext b/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext @@ -0,0 +1,8 @@ +foo:3200:13 + 1: 13 + 2: 7 + 3: 6 + 4: 13 + 5: 7 _Z3foov:5 _Z3barv:2 + 6: 6 _Z3barv:4 _Z3foov:2 + !CFGChecksum: 563022570642068 diff --git a/llvm/test/tools/llvm-profdata/merge-probe-profile.test b/llvm/test/tools/llvm-profdata/merge-probe-profile.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/merge-probe-profile.test @@ -0,0 +1,23 @@ +# Tests for merge of hybrid profile files. + +RUN: llvm-profdata merge --sample --text %p/Inputs/pseudo-probe-profile.proftext -o - | FileCheck %s --check-prefix=MERGE1 +RUN: llvm-profdata merge --sample --extbinary %p/Inputs/pseudo-probe-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE1 +MERGE1: foo:3200:13 +MERGE1: 1: 13 +MERGE1: 2: 7 +MERGE1: 3: 6 +MERGE1: 4: 13 +MERGE1: 5: 7 _Z3foov:5 _Z3barv:2 +MERGE1: 6: 6 _Z3barv:4 _Z3foov:2 +MERGE1: !CFGChecksum: 563022570642068 + +RUN: llvm-profdata merge --sample --text %p/Inputs/pseudo-probe-profile.proftext %p/Inputs/pseudo-probe-profile.proftext -o - | FileCheck %s --check-prefix=MERGE2 +RUN: llvm-profdata merge --sample --extbinary %p/Inputs/pseudo-probe-profile.proftext %p/Inputs/pseudo-probe-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE2 +MERGE2: foo:6400:26 +MERGE2: 1: 26 +MERGE2: 2: 14 +MERGE2: 3: 12 +MERGE2: 4: 26 +MERGE2: 5: 14 _Z3foov:10 _Z3barv:4 +MERGE2: 6: 12 _Z3barv:8 _Z3foov:4 +MERGE2: !CFGChecksum: 563022570642068 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -660,6 +660,7 @@ SmallVector, 5> Readers; LLVMContext Context; sampleprof::ProfileSymbolList WriterList; + Optional ProfileIsProbeBased; for (const auto &Input : Inputs) { auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context); if (std::error_code EC = ReaderOrErr.getError()) { @@ -680,6 +681,9 @@ } StringMap &Profiles = Reader->getProfiles(); + assert(!ProfileIsProbeBased || + ProfileIsProbeBased == FunctionSamples::ProfileIsProbeBased); + ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased; for (StringMap::iterator I = Profiles.begin(), E = Profiles.end(); I != E; ++I) { @@ -1822,6 +1826,9 @@ exitWithErrorCode(EC, BaseFilename); if (std::error_code EC = TestReader->read()) exitWithErrorCode(EC, TestFilename); + if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased()) + exitWithError( + "cannot compare probe-based profile with non-probe-based profile"); // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in // profile summary.