diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -495,28 +495,32 @@ State = UnknownContext; Name = ContextStr; } else { - // Remove encapsulating '[' and ']' if any - ContextStr = ContextStr.substr(1, ContextStr.size() - 2); CSNameTable.emplace_back(); SampleContextFrameVector &Context = CSNameTable.back(); - /// Create a context vector from a given context string and save it in - /// `Context`. - StringRef ContextRemain = ContextStr; - StringRef ChildContext; - StringRef CalleeName; - while (!ContextRemain.empty()) { - auto ContextSplit = ContextRemain.split(" @ "); - ChildContext = ContextSplit.first; - ContextRemain = ContextSplit.second; - LineLocation CallSiteLoc(0, 0); - decodeContextString(ChildContext, CalleeName, CallSiteLoc); - Context.emplace_back(CalleeName, CallSiteLoc); - } - + createCtxVectorFromStr(ContextStr, Context); setContext(Context, CState); } } + /// Create a context vector from a given context string and save it in + /// `Context`. + static void createCtxVectorFromStr(StringRef ContextStr, + SampleContextFrameVector &Context) { + // Remove encapsulating '[' and ']' if any + ContextStr = ContextStr.substr(1, ContextStr.size() - 2); + StringRef ContextRemain = ContextStr; + StringRef ChildContext; + StringRef CalleeName; + while (!ContextRemain.empty()) { + auto ContextSplit = ContextRemain.split(" @ "); + ChildContext = ContextSplit.first; + ContextRemain = ContextSplit.second; + LineLocation CallSiteLoc(0, 0); + decodeContextString(ChildContext, CalleeName, CallSiteLoc); + Context.emplace_back(CalleeName, CallSiteLoc); + } + } + // Promote context by removing top frames with the length of // `ContextFramesToRemove`. Note that with array representation of context, // the promotion is effectively a slice operation with first diff --git a/llvm/test/tools/llvm-profgen/inline-noprobe.test b/llvm/test/tools/llvm-profgen/inline-noprobe.test --- a/llvm/test/tools/llvm-profgen/inline-noprobe.test +++ b/llvm/test/tools/llvm-profgen/inline-noprobe.test @@ -1,7 +1,7 @@ ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-noprobe.perfscript --binary=%S/Inputs/inline-noprobe.perfbin --output=%t --skip-symbolization ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-RAW-PROFILE -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-noprobe.perfscript --binary=%S/Inputs/inline-noprobe.perfbin --output=%t -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK +; RUN: llvm-profgen --format=text --raw-profile=%t --binary=%S/Inputs/inline-noprobe.perfbin --output=%t1 +; RUN: FileCheck %s --input-file %t1 --check-prefix=CHECK CHECK: main:669:0 CHECK: 0: 0 diff --git a/llvm/test/tools/llvm-profgen/inline-noprobe2.test b/llvm/test/tools/llvm-profgen/inline-noprobe2.test --- a/llvm/test/tools/llvm-profgen/inline-noprobe2.test +++ b/llvm/test/tools/llvm-profgen/inline-noprobe2.test @@ -1,7 +1,7 @@ ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-noprobe2.perfscript --binary=%S/Inputs/inline-noprobe2.perfbin --output=%t --skip-symbolization --use-offset=0 ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-RAW-PROFILE -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-noprobe2.perfscript --binary=%S/Inputs/inline-noprobe2.perfbin --output=%t -; RUN: FileCheck %s --input-file %t --check-prefix=CHECK +; RUN: llvm-profgen --format=text --raw-profile=%t --binary=%S/Inputs/inline-noprobe2.perfbin --output=%t1 --use-offset=0 +; RUN: FileCheck %s --input-file %t1 --check-prefix=CHECK ; RUN: llvm-profgen --format=extbinary --perfscript=%S/Inputs/inline-noprobe2.perfscript --binary=%S/Inputs/inline-noprobe2.perfbin --output=%t ; RUN: llvm-profdata show -show-prof-sym-list -sample %t | FileCheck %s --check-prefix=CHECK-SYM-LIST diff --git a/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test b/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test --- a/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test +++ b/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test @@ -2,8 +2,8 @@ ; REQUIRES: x86_64-linux ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noinline-cs-noprobe.perfscript --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t --skip-symbolization --profile-summary-cold-count=0 ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-UNWINDER -; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noinline-cs-noprobe.perfscript --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t --profile-summary-cold-count=0 -; RUN: FileCheck %s --input-file %t +; RUN: llvm-profgen --format=text --raw-profile=%t --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t1 --profile-summary-cold-count=0 +; RUN: FileCheck %s --input-file %t1 ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noinline-cs-noprobe.perfscript --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t --profile-summary-cold-count=0 --ignore-stack-samples ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-STRIP-CTX ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/noinline-cs-noprobe.aggperfscript --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t --skip-symbolization --profile-summary-cold-count=0 diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h --- a/llvm/tools/llvm-profgen/PerfReader.h +++ b/llvm/tools/llvm-profgen/PerfReader.h @@ -59,11 +59,20 @@ }; // The type of perfscript -enum PerfScriptType { - PERF_UNKNOWN = 0, - PERF_INVALID = 1, - PERF_LBR = 2, // Only LBR sample - PERF_LBR_STACK = 3, // Hybrid sample including call stack and LBR stack. +enum PerfInputType { + PERFSCRIPT_UNKNOWN = 0, + PERFSCRIPT_INVALID = 1, + PERFSCRIPT_LBR = 2, // Only LBR sample + PERFSCRIPT_LBR_STACK = 3, // Hybrid sample including call stack and LBR stack. + PEFFDATA = 4, // Raw linux perf.data. + RAWPROFILE = 5, // Unsymbolized profile generated by llvm-profgen. +}; + +struct PerfInputFile { + std::string InputFile; + PerfInputType InputType; + PerfInputFile(StringRef F, PerfInputType T) + : InputFile(F.str()), InputType(T){}; }; // The parsed LBR sample entry. @@ -503,19 +512,17 @@ // Read perf trace to parse the events and samples. class PerfReaderBase { public: - PerfReaderBase(ProfiledBinary *B, StringRef PerfTrace, - PerfScriptType Type = PERF_UNKNOWN) - : Binary(B), PerfTraceFile(PerfTrace), PerfType(Type) { + PerfReaderBase(ProfiledBinary *B, StringRef PerfTrace) + : Binary(B), PerfTraceFile(PerfTrace) { // Initialize the base address to preferred address. Binary->setBaseAddress(Binary->getPreferredBaseAddress()); }; virtual ~PerfReaderBase() = default; - static std::unique_ptr - create(ProfiledBinary *Binary, StringRef PerfInputFile, bool IsPerfData); + static std::unique_ptr create(ProfiledBinary *Binary, + PerfInputFile &PerfInput); - PerfScriptType getPerfScriptType() const { return PerfType; } // Entry of the reader to parse multiple perf traces - void parsePerfTraces(); + virtual void parsePerfTraces(); const ContextSampleCounterMap &getSampleCounters() const { return SampleCounters; } @@ -538,7 +545,7 @@ // Check whether a given line is MMAP event static bool isMMap2Event(StringRef Line); // Extract perf script type by peaking at the input - static PerfScriptType checkPerfScriptType(StringRef FileName); + static PerfInputType checkPerfScriptType(StringRef FileName); // Parse a single line of a PERF_RECORD_MMAP2 event looking for a // mapping between the binary name and its memory layout. static bool extractMMap2EventForBinary(ProfiledBinary *Binary, StringRef Line, @@ -567,10 +574,10 @@ void parseSample(TraceStream &TraceIt); // An aggregated count is given to indicate how many times the sample is // repeated. - virtual void parseSample(TraceStream &TraceIt, uint64_t Count) = 0; + virtual void parseSample(TraceStream &TraceIt, uint64_t Count){}; // Post process the profile after trace aggregation, we will do simple range // overlap computation for AutoFDO, or unwind for CSSPGO(hybrid sample). - virtual void generateRawProfile() = 0; + virtual void generateRawProfile(){}; void writeRawProfile(StringRef Filename); void writeRawProfile(raw_fd_ostream &OS); @@ -579,7 +586,6 @@ ContextSampleCounterMap SampleCounters; // Samples with the repeating time generated by the perf reader AggregatedCounter AggregatedSamples; - PerfScriptType PerfType = PERF_UNKNOWN; // Keep track of all invalid return addresses std::set InvalidReturnAddresses; @@ -594,9 +600,8 @@ */ class LBRPerfReader : public PerfReaderBase { public: - LBRPerfReader(ProfiledBinary *Binary, StringRef PerfTrace, - PerfScriptType Type = PERF_LBR) - : PerfReaderBase(Binary, PerfTrace, Type){}; + LBRPerfReader(ProfiledBinary *Binary, StringRef PerfTrace) + : PerfReaderBase(Binary, PerfTrace){}; // Parse the LBR only sample. virtual void parseSample(TraceStream &TraceIt, uint64_t Count) override; virtual void generateRawProfile() override; @@ -617,7 +622,7 @@ class HybridPerfReader : public LBRPerfReader { public: HybridPerfReader(ProfiledBinary *Binary, StringRef PerfTrace) - : LBRPerfReader(Binary, PerfTrace, PERF_LBR_STACK){}; + : LBRPerfReader(Binary, PerfTrace){}; // Parse the hybrid sample including the call and LBR line void parseSample(TraceStream &TraceIt, uint64_t Count) override; void generateRawProfile() override; @@ -627,6 +632,19 @@ void unwindSamples(); }; +class RawProfileReader : public PerfReaderBase { +public: + RawProfileReader(ProfiledBinary *Binary, StringRef PerfTrace) + : PerfReaderBase(Binary, PerfTrace){}; + void parsePerfTraces() override; + +private: + void readSampleCounters(TraceStream &TraceIt, SampleCounter &SCounters); + void readRawProfile(StringRef Filename); + + std::unordered_set ContextStrSet; +}; + } // end namespace sampleprof } // end namespace llvm diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp --- a/llvm/tools/llvm-profgen/PerfReader.cpp +++ b/llvm/tools/llvm-profgen/PerfReader.cpp @@ -277,24 +277,29 @@ return true; } -std::unique_ptr PerfReaderBase::create(ProfiledBinary *Binary, - StringRef PerfInputFile, - bool IsPerfData) { +std::unique_ptr +PerfReaderBase::create(ProfiledBinary *Binary, PerfInputFile &PerfInput) { + std::unique_ptr PerfReader; + + if (PerfInput.InputType == RAWPROFILE) { + PerfReader.reset(new RawProfileReader(Binary, PerfInput.InputFile)); + return PerfReader; + } + // For perf data input, we need to convert them into perf script first. - if (IsPerfData) { - std::string ConvertedPerfScript = - convertPerfDataToTrace(Binary, PerfInputFile); - // Let commoand opt own the string for converted perf trace file name - PerfTraceFilename = ConvertedPerfScript; - PerfInputFile = PerfTraceFilename; + if (PerfInput.InputType == PEFFDATA) { + PerfInput.InputFile = convertPerfDataToTrace(Binary, PerfInput.InputFile); + PerfInput.InputType = PERFSCRIPT_UNKNOWN; } - PerfScriptType PerfType = checkPerfScriptType(PerfInputFile); - std::unique_ptr PerfReader; - if (PerfType == PERF_LBR_STACK) { - PerfReader.reset(new HybridPerfReader(Binary, PerfInputFile)); - } else if (PerfType == PERF_LBR) { - PerfReader.reset(new LBRPerfReader(Binary, PerfInputFile)); + assert(PerfInput.InputType == PERFSCRIPT_UNKNOWN && + "Should be a perfscript!"); + + PerfInput.InputType = checkPerfScriptType(PerfInput.InputFile); + if (PerfInput.InputType == PERFSCRIPT_LBR_STACK) { + PerfReader.reset(new HybridPerfReader(Binary, PerfInput.InputFile)); + } else if (PerfInput.InputType == PERFSCRIPT_LBR) { + PerfReader.reset(new LBRPerfReader(Binary, PerfInput.InputFile)); } else { exitWithError("Unsupported perfscript!"); } @@ -695,7 +700,7 @@ OrderedCounters[getContextKeyStr(CI.first.getPtr(), Binary)] = &CI.second; } - auto SCounterPrinter = [&](RangeSample Counter, StringRef Separator, + auto SCounterPrinter = [&](RangeSample &Counter, StringRef Separator, uint32_t Indent) { OS.indent(Indent); OS << Counter.size() << "\n"; @@ -712,7 +717,7 @@ for (auto &CI : OrderedCounters) { uint32_t Indent = 0; - if (!CI.first.empty()) { + if (ProfileIsCS) { // Context string key OS << "[" << CI.first << "]\n"; Indent = 2; @@ -724,6 +729,85 @@ } } +inline void exitWithErrorForTraceLine(TraceStream &TraceIt) { + std::string Msg = TraceIt.isAtEoF() + ? "Invalid raw profile!" + : "Invalid raw profile at line " + + Twine(TraceIt.getLineNumber()).str() + ": " + + TraceIt.getCurrentLine().str(); + exitWithError(Msg); +} + +void RawProfileReader::readSampleCounters(TraceStream &TraceIt, + SampleCounter &SCounters) { + auto ReadNumber = [&](uint64_t &Num) { + if (TraceIt.isAtEoF()) + exitWithErrorForTraceLine(TraceIt); + if (TraceIt.getCurrentLine().ltrim().getAsInteger(10, Num)) + exitWithErrorForTraceLine(TraceIt); + TraceIt.advance(); + }; + + auto ReadCounter = [&](RangeSample &Counter, StringRef Separator) { + uint64_t Num = 0; + ReadNumber(Num); + while (Num--) { + if (TraceIt.isAtEoF()) + exitWithErrorForTraceLine(TraceIt); + StringRef Line = TraceIt.getCurrentLine().ltrim(); + + uint64_t Count = 0; + auto LineSplit = Line.split(":"); + if (LineSplit.second.empty() || LineSplit.second.getAsInteger(10, Count)) + exitWithErrorForTraceLine(TraceIt); + + uint64_t Source = 0; + uint64_t Target = 0; + auto Range = LineSplit.first.split(Separator); + if (Range.second.empty() || Range.first.getAsInteger(16, Source) || + Range.second.getAsInteger(16, Target)) + exitWithErrorForTraceLine(TraceIt); + + if(!UseOffset) { + Source = Binary->virtualAddrToOffset(Source); + Target = Binary->virtualAddrToOffset(Target); + } + + Counter[{Source, Target}] += Count; + TraceIt.advance(); + } + }; + + ReadCounter(SCounters.RangeCounter, "-"); + ReadCounter(SCounters.BranchCounter, "->"); +} + +void RawProfileReader::readRawProfile(StringRef FileName) { + TraceStream TraceIt(FileName); + while (!TraceIt.isAtEoF()) { + std::shared_ptr Key = + std::make_shared(); + StringRef Line = TraceIt.getCurrentLine(); + if (Line.startswith("[")) { + ProfileIsCS = true; + auto I = ContextStrSet.insert(Line.str()); + SampleContext::createCtxVectorFromStr(*I.first, Key->Context); + TraceIt.advance(); + } + Key->genHashCode(); + auto Ret = + SampleCounters.emplace(Hashable(Key), SampleCounter()); + readSampleCounters(TraceIt, Ret.first->second); + } +} + +void RawProfileReader::parsePerfTraces() { + readRawProfile(PerfTraceFile); + + if (SkipSymbolization) + writeRawProfile(OutputFilename); +} + void LBRPerfReader::computeCounterFromLBR(const PerfSample *Sample, uint64_t Repeat) { SampleCounter &Counter = SampleCounters.begin()->second; @@ -890,7 +974,7 @@ // Determine the perfscript contains hybrid samples(call stack + LBRs) by // checking whether there is a non-empty call stack immediately followed by // a LBR sample -PerfScriptType PerfReaderBase::checkPerfScriptType(StringRef FileName) { +PerfInputType PerfReaderBase::checkPerfScriptType(StringRef FileName) { TraceStream TraceIt(FileName); uint64_t FrameAddr = 0; while (!TraceIt.isAtEoF()) { @@ -908,16 +992,16 @@ if (!TraceIt.isAtEoF()) { if (isLBRSample(TraceIt.getCurrentLine())) { if (Count > 0) - return PERF_LBR_STACK; + return PERFSCRIPT_LBR_STACK; else - return PERF_LBR; + return PERFSCRIPT_LBR; } TraceIt.advance(); } } exitWithError("Invalid perf script input!"); - return PERF_INVALID; + return PERFSCRIPT_INVALID; } void HybridPerfReader::generateRawProfile() { diff --git a/llvm/tools/llvm-profgen/llvm-profgen.cpp b/llvm/tools/llvm-profgen/llvm-profgen.cpp --- a/llvm/tools/llvm-profgen/llvm-profgen.cpp +++ b/llvm/tools/llvm-profgen/llvm-profgen.cpp @@ -21,7 +21,7 @@ static cl::OptionCategory ProfGenCategory("ProfGen Options"); -cl::opt PerfTraceFilename( +cl::opt PerfScriptFilename( "perfscript", cl::value_desc("perfscript"), cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated, cl::desc("Path of perf-script trace created by Linux perf tool with " @@ -35,6 +35,13 @@ "profiled with -b)"), cl::cat(ProfGenCategory)); +cl::opt + RawProfileFilename("raw-profile", cl::value_desc("raw-profile"), + cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated, + cl::desc("Path of the raw profile created by " + "`llvm-profgen` with `--skip-symbolization`"), + cl::cat(ProfGenCategory)); + static cl::opt BinaryPath( "binary", cl::value_desc("binary"), cl::Required, cl::desc("Path of profiled binary, only one binary is supported."), @@ -49,36 +56,33 @@ // Validate the command line input. static void validateCommandLine() { - // Validate input profile is provided only once - if (PerfDataFilename.getNumOccurrences() && - PerfTraceFilename.getNumOccurrences()) { - std::string Msg = "`-perfdata` and `-perfscript` cannot be used together."; - exitWithError(Msg); - } - - // Validate input profile is provided - if (!PerfDataFilename.getNumOccurrences() && - !PerfTraceFilename.getNumOccurrences()) { - std::string Msg = - "Use `-perfdata` or `-perfscript` to provide input perf profile."; - exitWithError(Msg); - } - - // Allow the invalid perfscript if we only use to show binary disassembly. + // Allow the missing perfscript if we only use to show binary disassembly. if (!ShowDisassemblyOnly) { - if (PerfTraceFilename.getNumOccurrences() && - !llvm::sys::fs::exists(PerfTraceFilename)) { + // Validate input profile is provided only once + uint16_t HasPerfData = PerfDataFilename.getNumOccurrences(); + uint16_t HasPerfScript = PerfScriptFilename.getNumOccurrences(); + uint16_t HasRawProfile = RawProfileFilename.getNumOccurrences(); + uint16_t S = HasPerfData + HasPerfScript + HasRawProfile; + if (S != 1) { std::string Msg = - "Input perf script(" + PerfTraceFilename + ") doesn't exist."; + S > 1 + ? "`--perfscript`, `--perfdata` and `--raw-profile` cannot be " + "used together." + : "Perf input file is missing, please use one of `--perfscript`, " + "`--perfdata` and `--raw-profile` for the input."; exitWithError(Msg); } - if (PerfDataFilename.getNumOccurrences() && - !llvm::sys::fs::exists(PerfDataFilename)) { - std::string Msg = - "Input perf data(" + PerfDataFilename + ") doesn't exist."; - exitWithError(Msg); - } + auto CheckFileExists = [](bool H, StringRef File) { + if (H && !llvm::sys::fs::exists(File)) { + std::string Msg = "Input perf file(" + File.str() + ") doesn't exist."; + exitWithError(Msg); + } + }; + + CheckFileExists(HasPerfData, PerfDataFilename); + CheckFileExists(HasPerfScript, PerfScriptFilename); + CheckFileExists(HasRawProfile, RawProfileFilename); } if (!llvm::sys::fs::exists(BinaryPath)) { @@ -95,6 +99,23 @@ } } +static PerfInputFile createPerfInputFile() { + // Parse perf events and samples + StringRef InputFile; + PerfInputType Type = PERFSCRIPT_UNKNOWN; + if (PerfDataFilename.getNumOccurrences()) { + InputFile = PerfDataFilename; + Type = PEFFDATA; + } else if (PerfScriptFilename.getNumOccurrences()) { + InputFile = PerfScriptFilename; + Type = PERFSCRIPT_UNKNOWN; + } else if (RawProfileFilename.getNumOccurrences()) { + InputFile = RawProfileFilename; + Type = RAWPROFILE; + } + return PerfInputFile(InputFile, Type); +} + int main(int argc, const char *argv[]) { InitLLVM X(argc, argv); @@ -113,15 +134,9 @@ if (ShowDisassemblyOnly) return EXIT_SUCCESS; - // Parse perf events and samples - StringRef PerfInputFile; - bool IsPerfData = PerfDataFilename.getNumOccurrences(); - if (IsPerfData) - PerfInputFile = PerfDataFilename; - else - PerfInputFile = PerfTraceFilename; + PerfInputFile PerfFile = createPerfInputFile(); std::unique_ptr Reader = - PerfReaderBase::create(Binary.get(), PerfInputFile, IsPerfData); + PerfReaderBase::create(Binary.get(), PerfFile); Reader->parsePerfTraces(); if (SkipSymbolization)