diff --git a/compiler-rt/test/profile/instrprof-timestamp.c b/compiler-rt/test/profile/instrprof-timestamp.c --- a/compiler-rt/test/profile/instrprof-timestamp.c +++ b/compiler-rt/test/profile/instrprof-timestamp.c @@ -2,14 +2,20 @@ // RUN: %clang_pgogen -o %t -mllvm -pgo-temporal-instrumentation %s // RUN: env LLVM_PROFILE_FILE=%t.0.profraw %run %t n // RUN: env LLVM_PROFILE_FILE=%t.1.profraw %run %t y -// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw %t.1.profraw +// RUN: env LLVM_PROFILE_FILE=%t.2.profraw %run %t y +// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw +// RUN: llvm-profdata merge -o %t.profdata --temporal-profile-trace-weight=5 %t.profdata %t.1.profraw +// RUN: llvm-profdata merge -o %t.profdata --temporal-profile-trace-weight=0 %t.profdata %t.2.profraw // RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --implicit-check-not=unused // RUN: rm -f %t.profdata // RUN: %clang_pgogen -o %t -mllvm -pgo-temporal-instrumentation -mllvm -pgo-block-coverage %s // RUN: env LLVM_PROFILE_FILE=%t.0.profraw %run %t n // RUN: env LLVM_PROFILE_FILE=%t.1.profraw %run %t y -// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw %t.1.profraw +// RUN: env LLVM_PROFILE_FILE=%t.2.profraw %run %t y +// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw +// RUN: llvm-profdata merge -o %t.profdata --temporal-profile-trace-weight=5 %t.profdata %t.1.profraw +// RUN: llvm-profdata merge -o %t.profdata --temporal-profile-trace-weight=0 %t.profdata %t.2.profraw // RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --implicit-check-not=unused extern void exit(int); @@ -36,12 +42,12 @@ } // CHECK: Temporal Profile Traces (samples=2 seen=2): -// CHECK: Temporal Profile Trace 0 (count=4): +// CHECK: Temporal Profile Trace 0 (weight=1 count=4): // CHECK: main // CHECK: a // CHECK: b // CHECK: c -// CHECK: Temporal Profile Trace 1 (count=3): +// CHECK: Temporal Profile Trace 1 (weight=5 count=3): // CHECK: a // CHECK: c // CHECK: b diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -197,6 +197,24 @@ When ``-debug-info-correlate`` was used for instrumentation, use this option to correlate the raw profile. +.. option:: --temporal-profile-trace-reservoir-size + + The maximum number of temporal profile traces to be stored in the output + profile. If more traces are added, we will used reservoir sampling to select + which traces to keep. Note that changing this value between different merge + invocations on the same indexed profile could result in sample bias. The + default value is 100. + +.. option:: --max-temporal-profile-trace-length + + The maximum number of functions in a single temporal profile trace. Longer + traces will be truncated. The default value is 1000. + +.. option:: --temporal-profile-trace-weight + + The weight to be assigned to raw temporal profile traces added by this command. + The value 0 means raw traces will not be added. The default value is 1. + EXAMPLES ^^^^^^^^ diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -335,7 +335,10 @@ /// An ordered list of functions identified by their NameRef found in /// INSTR_PROF_DATA -using TemporalProfTraceTy = std::vector; +struct TemporalProfTraceTy { + uint64_t Weight = 1; + std::vector FunctionNameRefs; +}; inline std::error_code make_error_code(instrprof_error E) { return std::error_code(static_cast(E), instrprof_category()); diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -208,8 +208,13 @@ create(std::unique_ptr Buffer, const InstrProfCorrelator *Correlator = nullptr); + /// \param Weight for raw profiles use this as the temporal profile trace + /// weight /// \returns a list of temporal profile traces. - virtual const SmallVector &getTemporalProfTraces() { + virtual const SmallVector & + getTemporalProfTraces(std::optional Weight = {}) { + // For non-raw profiles we ignore the input weight and instead use the + // weights already in the traces. return TemporalProfTraces; } /// \returns the total number of temporal profile traces seen. @@ -395,7 +400,8 @@ return *Symtab.get(); } - const SmallVector &getTemporalProfTraces() override; + const SmallVector & + getTemporalProfTraces(std::optional Weight = {}) override; private: Error createSymtab(InstrProfSymtab &Symtab); diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -281,7 +281,8 @@ /// Temporal profile trace data is stored in the header immediately after /// ":temporal_prof_traces". The first integer is the number of traces, the /// second integer is the stream size, then the following lines are the actual -/// traces which consist of a comma separated list of function names. +/// traces which consist of a weight and a comma separated list of function +/// names. Error TextInstrProfReader::readTemporalProfTraceData() { if ((++Line).is_at_end()) return error(instrprof_error::eof); @@ -301,10 +302,17 @@ return error(instrprof_error::eof); TemporalProfTraceTy Trace; + if (Line->getAsInteger(0, Trace.Weight)) + return error(instrprof_error::malformed); + + if ((++Line).is_at_end()) + return error(instrprof_error::eof); + SmallVector FuncNames; Line->split(FuncNames, ",", /*MaxSplit=*/-1, /*KeepEmpty=*/false); for (auto &FuncName : FuncNames) - Trace.push_back(IndexedInstrProf::ComputeHash(FuncName.trim())); + Trace.FunctionNameRefs.push_back( + IndexedInstrProf::ComputeHash(FuncName.trim())); TemporalProfTraces.push_back(std::move(Trace)); } return success(); @@ -439,7 +447,13 @@ template const SmallVector & -RawInstrProfReader::getTemporalProfTraces() { +RawInstrProfReader::getTemporalProfTraces( + std::optional Weight) { + if (Weight.has_value() && Weight.value() == 0) { + // Drop weight 0 traces + TemporalProfTraces.clear(); + return TemporalProfTraces; + } if (TemporalProfTimestamps.empty()) { assert(TemporalProfTraces.empty()); return TemporalProfTraces; @@ -447,8 +461,10 @@ // Sort functions by their timestamps to build the trace. std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end()); TemporalProfTraceTy Trace; + if (Weight) + Trace.Weight = *Weight; for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps) - Trace.push_back(NameRef); + Trace.FunctionNameRefs.push_back(NameRef); TemporalProfTraces = {std::move(Trace)}; return TemporalProfTraces; } @@ -1147,19 +1163,21 @@ TemporalProfTraceStreamSize = support::endian::readNext(Ptr); for (unsigned i = 0; i < NumTraces; i++) { - // Expect at least one 64 bit field: NumFunctions - if (Ptr + sizeof(uint64_t) > PtrEnd) + // Expect at least two 64 bit fields: Weight and NumFunctions + if (Ptr + 2 * sizeof(uint64_t) > PtrEnd) return error(instrprof_error::truncated); + TemporalProfTraceTy Trace; + Trace.Weight = + support::endian::readNext(Ptr); const uint64_t NumFunctions = support::endian::readNext(Ptr); // Expect at least NumFunctions 64 bit fields if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd) return error(instrprof_error::truncated); - TemporalProfTraceTy Trace; for (unsigned j = 0; j < NumFunctions; j++) { const uint64_t NameRef = support::endian::readNext(Ptr); - Trace.push_back(NameRef); + Trace.FunctionNameRefs.push_back(NameRef); } TemporalProfTraces.push_back(std::move(Trace)); } diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -291,9 +291,9 @@ } void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) { - if (Trace.size() > MaxTemporalProfTraceLength) - Trace.resize(MaxTemporalProfTraceLength); - if (Trace.empty()) + if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength) + Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength); + if (Trace.FunctionNameRefs.empty()) return; if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) { @@ -591,8 +591,9 @@ OS.write(TemporalProfTraces.size()); OS.write(TemporalProfTraceStreamSize); for (auto &Trace : TemporalProfTraces) { - OS.write(Trace.size()); - for (auto &NameRef : Trace) + OS.write(Trace.Weight); + OS.write(Trace.FunctionNameRefs.size()); + for (auto &NameRef : Trace.FunctionNameRefs) OS.write(NameRef); } } @@ -779,7 +780,8 @@ OS << "# Temporal Profile Trace Stream Size:\n" << TemporalProfTraceStreamSize << "\n"; for (auto &Trace : TemporalProfTraces) { - for (auto &NameRef : Trace) + OS << "# Weight:\n" << Trace.Weight << "\n"; + for (auto &NameRef : Trace.FunctionNameRefs) OS << Symtab.getFuncName(NameRef) << ","; OS << "\n"; } diff --git a/llvm/test/tools/llvm-profdata/merge-traces.proftext b/llvm/test/tools/llvm-profdata/merge-traces.proftext --- a/llvm/test/tools/llvm-profdata/merge-traces.proftext +++ b/llvm/test/tools/llvm-profdata/merge-traces.proftext @@ -11,11 +11,11 @@ # SEEN2: Temporal Profile Traces (samples=2 seen=2): # SEEN3: Temporal Profile Traces (samples=2 seen=3): # SEEN4: Temporal Profile Traces (samples=2 seen=4): -# SAMPLE1: Temporal Profile Trace 0 (count=3): +# SAMPLE1: Temporal Profile Trace 0 (weight=1 count=3): # SAMPLE1: a # SAMPLE1: b # SAMPLE1: c -# SAMPLE2: Temporal Profile Trace 1 (count=3): +# SAMPLE2: Temporal Profile Trace 1 (weight=1 count=3): # SAMPLE2: a # SAMPLE2: b # SAMPLE2: c @@ -27,6 +27,8 @@ 1 # Trace Stream Size: 1 +# Weight +1 a, b, c diff --git a/llvm/test/tools/llvm-profdata/read-traces.proftext b/llvm/test/tools/llvm-profdata/read-traces.proftext --- a/llvm/test/tools/llvm-profdata/read-traces.proftext +++ b/llvm/test/tools/llvm-profdata/read-traces.proftext @@ -6,15 +6,15 @@ # RUN: llvm-profdata show --temporal-profile-traces %t.1.proftext | FileCheck %s # CHECK: Temporal Profile Traces (samples=3 seen=3): -# CHECK: Temporal Profile Trace 0 (count=3): +# CHECK: Temporal Profile Trace 0 (weight=1 count=3): # CHECK: foo # CHECK: bar # CHECK: goo -# CHECK: Temporal Profile Trace 1 (count=3): +# CHECK: Temporal Profile Trace 1 (weight=3 count=3): # CHECK: foo # CHECK: goo # CHECK: bar -# CHECK: Temporal Profile Trace 2 (count=1): +# CHECK: Temporal Profile Trace 2 (weight=1 count=1): # CHECK: goo # Header @@ -24,8 +24,14 @@ 3 # Trace Stream Size: 3 +# Weight +1 foo, bar, goo +# Weight +3 foo,goo,bar, +# Weight +1 goo foo diff --git a/llvm/test/tools/llvm-profdata/trace-limit.proftext b/llvm/test/tools/llvm-profdata/trace-limit.proftext --- a/llvm/test/tools/llvm-profdata/trace-limit.proftext +++ b/llvm/test/tools/llvm-profdata/trace-limit.proftext @@ -9,8 +9,8 @@ # NONE: Temporal Profile Traces (samples=0 seen=0): # CHECK: Temporal Profile Traces (samples=1 seen=1): -# SOME: Trace 0 (count=2): -# ALL: Trace 0 (count=3): +# SOME: Trace 0 (weight=1 count=2): +# ALL: Trace 0 (weight=1 count=3): # Header :ir @@ -19,6 +19,8 @@ 1 # Trace Stream Size: 1 +# Weight +1 a, b, c diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -252,7 +252,8 @@ /// Load an input into a writer context. static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, const InstrProfCorrelator *Correlator, - const StringRef ProfiledBinary, WriterContext *WC) { + uint64_t TraceWeight, const StringRef ProfiledBinary, + WriterContext *WC) { std::unique_lock CtxGuard{WC->Lock}; // Copy the filename, because llvm::ThreadPool copied the input "const @@ -344,7 +345,7 @@ } if (Reader->hasTemporalProfile()) { - auto &Traces = Reader->getTemporalProfTraces(); + auto &Traces = Reader->getTemporalProfTraces(TraceWeight); if (!Traces.empty()) WC->Writer.addTemporalProfileTraces( Traces, Reader->getTemporalProfTraceStreamSize()); @@ -403,8 +404,8 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename, SymbolRemapper *Remapper, StringRef OutputFilename, ProfileFormat OutputFormat, uint64_t TraceReservoirSize, - uint64_t MaxTraceLength, bool OutputSparse, - unsigned NumThreads, FailureMode FailMode, + uint64_t MaxTraceLength, uint64_t TraceWeight, + bool OutputSparse, unsigned NumThreads, FailureMode FailMode, const StringRef ProfiledBinary) { if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary && OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text) @@ -436,7 +437,7 @@ if (NumThreads == 1) { for (const auto &Input : Inputs) - loadInput(Input, Remapper, Correlator.get(), ProfiledBinary, + loadInput(Input, Remapper, Correlator.get(), TraceWeight, ProfiledBinary, Contexts[0].get()); } else { ThreadPool Pool(hardware_concurrency(NumThreads)); @@ -444,8 +445,8 @@ // Load the inputs in parallel (N/NumThreads serial steps). unsigned Ctx = 0; for (const auto &Input : Inputs) { - Pool.async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary, - Contexts[Ctx].get()); + Pool.async(loadInput, Input, Remapper, Correlator.get(), TraceWeight, + ProfiledBinary, Contexts[Ctx].get()); Ctx = (Ctx + 1) % NumThreads; } Pool.wait(); @@ -864,7 +865,8 @@ SmallSet WriterErrorCodes; auto WC = std::make_unique(OutputSparse, ErrorLock, WriterErrorCodes); - loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get()); + loadInput(Inputs[0], nullptr, nullptr, /*TraceWeight=*/1, + /*ProfiledBinary=*/"", WC.get()); if (WC->Errors.size() > 0) exitWithError(std::move(WC->Errors[0].first), InstrFilename); @@ -1283,6 +1285,10 @@ "max-temporal-profile-trace-length", cl::init(10000), cl::desc("The maximum length of a single temporal profile trace " "(default: 10000)")); + cl::opt TemporalProfTraceWeight( + "temporal-profile-trace-weight", cl::init(1), + cl::desc("The weight to be assigned to the raw temporal profile traces " + "added by this command")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); @@ -1326,8 +1332,8 @@ mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, - MaxTemporalProfTraceLength, OutputSparse, NumThreads, - FailureMode, ProfiledBinary); + MaxTemporalProfTraceLength, TemporalProfTraceWeight, + OutputSparse, NumThreads, FailureMode, ProfiledBinary); else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, ProfileSymbolListFile, CompressAllSections, @@ -1359,7 +1365,8 @@ OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; exit(0); } - loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context); + loadInput(WeightedInput, nullptr, nullptr, /*TraceWeight=*/1, + /*ProfiledBinary=*/"", &Context); overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, IsCS); Overlap.dump(OS); @@ -2635,9 +2642,9 @@ OS << "Temporal Profile Traces (samples=" << Traces.size() << " seen=" << Reader->getTemporalProfTraceStreamSize() << "):\n"; for (unsigned i = 0; i < Traces.size(); i++) { - OS << " Temporal Profile Trace " << i << " (count=" << Traces[i].size() - << "):\n"; - for (auto &NameRef : Traces[i]) + OS << " Temporal Profile Trace " << i << " (weight=" << Traces[i].Weight + << " count=" << Traces[i].FunctionNameRefs.size() << "):\n"; + for (auto &NameRef : Traces[i].FunctionNameRefs) OS << " " << Reader->getSymtab().getFuncName(NameRef) << "\n"; } } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -39,6 +39,14 @@ return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n"; } +namespace llvm { +bool operator==(const TemporalProfTraceTy &lhs, + const TemporalProfTraceTy &rhs) { + return lhs.Weight == rhs.Weight && + lhs.FunctionNameRefs == rhs.FunctionNameRefs; +} +} // end namespace llvm + namespace { struct InstrProfTest : ::testing::Test { @@ -234,11 +242,12 @@ ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto LargeTrace = {IndexedInstrProf::ComputeHash("foo"), - IndexedInstrProf::ComputeHash("bar"), - IndexedInstrProf::ComputeHash("goo")}; - auto SmallTrace = {IndexedInstrProf::ComputeHash("foo"), - IndexedInstrProf::ComputeHash("bar")}; + TemporalProfTraceTy LargeTrace, SmallTrace; + LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"), + IndexedInstrProf::ComputeHash("bar"), + IndexedInstrProf::ComputeHash("goo")}; + SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"), + IndexedInstrProf::ComputeHash("bar")}; Writer.addTemporalProfileTraces({LargeTrace, SmallTrace}, 2); @@ -261,8 +270,9 @@ ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto FooTrace = {IndexedInstrProf::ComputeHash("foo")}; - auto BarTrace = {IndexedInstrProf::ComputeHash("bar")}; + TemporalProfTraceTy FooTrace, BarTrace; + FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")}; + BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")}; Writer.addTemporalProfileTraces({FooTrace}, 1); Writer2.addTemporalProfileTraces({BarTrace}, 1); @@ -284,9 +294,10 @@ ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto FooTrace = {IndexedInstrProf::ComputeHash("foo")}; - auto BarTrace = {IndexedInstrProf::ComputeHash("bar")}; - auto GooTrace = {IndexedInstrProf::ComputeHash("Goo")}; + TemporalProfTraceTy FooTrace, BarTrace, GooTrace; + FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")}; + BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")}; + GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("Goo")}; // Add some sampled traces Writer.addTemporalProfileTraces({FooTrace, BarTrace, GooTrace}, 5);