diff --git a/compiler-rt/test/profile/instrprof-timestamp.c b/compiler-rt/test/profile/instrprof-timestamp.c --- a/compiler-rt/test/profile/instrprof-timestamp.c +++ b/compiler-rt/test/profile/instrprof-timestamp.c @@ -2,14 +2,14 @@ // RUN: %clang_pgogen -o %t -mllvm -pgo-temporal-instrumentation %s // RUN: env LLVM_PROFILE_FILE=%t.0.profraw %run %t n // RUN: env LLVM_PROFILE_FILE=%t.1.profraw %run %t y -// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw %t.1.profraw +// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw --weighted-input=5,%t.1.profraw // RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --implicit-check-not=unused // RUN: rm -f %t.profdata // RUN: %clang_pgogen -o %t -mllvm -pgo-temporal-instrumentation -mllvm -pgo-block-coverage %s // RUN: env LLVM_PROFILE_FILE=%t.0.profraw %run %t n // RUN: env LLVM_PROFILE_FILE=%t.1.profraw %run %t y -// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw %t.1.profraw +// RUN: llvm-profdata merge -o %t.profdata %t.0.profraw --weighted-input=5,%t.1.profraw // RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --implicit-check-not=unused extern void exit(int); @@ -36,12 +36,12 @@ } // CHECK: Temporal Profile Traces (samples=2 seen=2): -// CHECK: Temporal Profile Trace 0 (count=4): +// CHECK: Temporal Profile Trace 0 (weight=1 count=4): // CHECK: main // CHECK: a // CHECK: b // CHECK: c -// CHECK: Temporal Profile Trace 1 (count=3): +// CHECK: Temporal Profile Trace 1 (weight=5 count=3): // CHECK: a // CHECK: c // CHECK: b diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst --- a/llvm/docs/CommandGuide/llvm-profdata.rst +++ b/llvm/docs/CommandGuide/llvm-profdata.rst @@ -197,6 +197,18 @@ When ``-debug-info-correlate`` was used for instrumentation, use this option to correlate the raw profile. +.. option:: --temporal-profile-trace-reservoir-size + + The maximum number of temporal profile traces to be stored in the output + profile. If more traces are added, we will use reservoir sampling to select + which traces to keep. Note that changing this value between different merge + invocations on the same indexed profile could result in sample bias. The + default value is 100. + +.. option:: --temporal-profile-max-trace-length + + The maximum number of functions in a single temporal profile trace. Longer + traces will be truncated. The default value is 1000. EXAMPLES ^^^^^^^^ diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -335,7 +335,10 @@ /// An ordered list of functions identified by their NameRef found in /// INSTR_PROF_DATA -using TemporalProfTraceTy = std::vector; +struct TemporalProfTraceTy { + uint64_t Weight = 1; + std::vector FunctionNameRefs; +}; inline std::error_code make_error_code(instrprof_error E) { return std::error_code(static_cast(E), instrprof_category()); diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -208,8 +208,13 @@ create(std::unique_ptr Buffer, const InstrProfCorrelator *Correlator = nullptr); + /// \param Weight for raw profiles use this as the temporal profile trace + /// weight /// \returns a list of temporal profile traces. - virtual const SmallVector &getTemporalProfTraces() { + virtual SmallVector & + getTemporalProfTraces(std::optional Weight = {}) { + // For non-raw profiles we ignore the input weight and instead use the + // weights already in the traces. return TemporalProfTraces; } /// \returns the total number of temporal profile traces seen. @@ -395,7 +400,8 @@ return *Symtab.get(); } - const SmallVector &getTemporalProfTraces() override; + SmallVector & + getTemporalProfTraces(std::optional Weight = {}) override; private: Error createSymtab(InstrProfSymtab &Symtab); diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -88,7 +88,7 @@ /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the /// total number of temporal profiling traces the source has seen. - void addTemporalProfileTraces(SmallVector SrcTraces, + void addTemporalProfileTraces(SmallVectorImpl &SrcTraces, uint64_t SrcStreamSize); /// Add a memprof record for a function identified by its \p Id. diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -281,7 +281,8 @@ /// Temporal profile trace data is stored in the header immediately after /// ":temporal_prof_traces". The first integer is the number of traces, the /// second integer is the stream size, then the following lines are the actual -/// traces which consist of a comma separated list of function names. +/// traces which consist of a weight and a comma separated list of function +/// names. Error TextInstrProfReader::readTemporalProfTraceData() { if ((++Line).is_at_end()) return error(instrprof_error::eof); @@ -301,10 +302,17 @@ return error(instrprof_error::eof); TemporalProfTraceTy Trace; + if (Line->getAsInteger(0, Trace.Weight)) + return error(instrprof_error::malformed); + + if ((++Line).is_at_end()) + return error(instrprof_error::eof); + SmallVector FuncNames; Line->split(FuncNames, ",", /*MaxSplit=*/-1, /*KeepEmpty=*/false); for (auto &FuncName : FuncNames) - Trace.push_back(IndexedInstrProf::ComputeHash(FuncName.trim())); + Trace.FunctionNameRefs.push_back( + IndexedInstrProf::ComputeHash(FuncName.trim())); TemporalProfTraces.push_back(std::move(Trace)); } return success(); @@ -438,8 +446,9 @@ } template -const SmallVector & -RawInstrProfReader::getTemporalProfTraces() { +SmallVector & +RawInstrProfReader::getTemporalProfTraces( + std::optional Weight) { if (TemporalProfTimestamps.empty()) { assert(TemporalProfTraces.empty()); return TemporalProfTraces; @@ -447,8 +456,10 @@ // Sort functions by their timestamps to build the trace. std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end()); TemporalProfTraceTy Trace; + if (Weight) + Trace.Weight = *Weight; for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps) - Trace.push_back(NameRef); + Trace.FunctionNameRefs.push_back(NameRef); TemporalProfTraces = {std::move(Trace)}; return TemporalProfTraces; } @@ -1147,19 +1158,21 @@ TemporalProfTraceStreamSize = support::endian::readNext(Ptr); for (unsigned i = 0; i < NumTraces; i++) { - // Expect at least one 64 bit field: NumFunctions - if (Ptr + sizeof(uint64_t) > PtrEnd) + // Expect at least two 64 bit fields: Weight and NumFunctions + if (Ptr + 2 * sizeof(uint64_t) > PtrEnd) return error(instrprof_error::truncated); + TemporalProfTraceTy Trace; + Trace.Weight = + support::endian::readNext(Ptr); const uint64_t NumFunctions = support::endian::readNext(Ptr); // Expect at least NumFunctions 64 bit fields if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd) return error(instrprof_error::truncated); - TemporalProfTraceTy Trace; for (unsigned j = 0; j < NumFunctions; j++) { const uint64_t NameRef = support::endian::readNext(Ptr); - Trace.push_back(NameRef); + Trace.FunctionNameRefs.push_back(NameRef); } TemporalProfTraces.push_back(std::move(Trace)); } diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -291,9 +291,9 @@ } void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) { - if (Trace.size() > MaxTemporalProfTraceLength) - Trace.resize(MaxTemporalProfTraceLength); - if (Trace.empty()) + if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength) + Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength); + if (Trace.FunctionNameRefs.empty()) return; if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) { @@ -311,7 +311,7 @@ } void InstrProfWriter::addTemporalProfileTraces( - SmallVector SrcTraces, uint64_t SrcStreamSize) { + SmallVectorImpl &SrcTraces, uint64_t SrcStreamSize) { // Assume that the source has the same reservoir size as the destination to // avoid needing to record it in the indexed profile format. bool IsDestSampled = @@ -356,7 +356,7 @@ for (auto &I : IPW.BinaryIds) addBinaryIds(I); - addTemporalProfileTraces(std::move(IPW.TemporalProfTraces), + addTemporalProfileTraces(IPW.TemporalProfTraces, IPW.TemporalProfTraceStreamSize); MemProfFrameData.reserve(IPW.MemProfFrameData.size()); @@ -591,8 +591,9 @@ OS.write(TemporalProfTraces.size()); OS.write(TemporalProfTraceStreamSize); for (auto &Trace : TemporalProfTraces) { - OS.write(Trace.size()); - for (auto &NameRef : Trace) + OS.write(Trace.Weight); + OS.write(Trace.FunctionNameRefs.size()); + for (auto &NameRef : Trace.FunctionNameRefs) OS.write(NameRef); } } @@ -779,7 +780,8 @@ OS << "# Temporal Profile Trace Stream Size:\n" << TemporalProfTraceStreamSize << "\n"; for (auto &Trace : TemporalProfTraces) { - for (auto &NameRef : Trace) + OS << "# Weight:\n" << Trace.Weight << "\n"; + for (auto &NameRef : Trace.FunctionNameRefs) OS << Symtab.getFuncName(NameRef) << ","; OS << "\n"; } diff --git a/llvm/test/tools/llvm-profdata/merge-traces.proftext b/llvm/test/tools/llvm-profdata/merge-traces.proftext --- a/llvm/test/tools/llvm-profdata/merge-traces.proftext +++ b/llvm/test/tools/llvm-profdata/merge-traces.proftext @@ -11,11 +11,11 @@ # SEEN2: Temporal Profile Traces (samples=2 seen=2): # SEEN3: Temporal Profile Traces (samples=2 seen=3): # SEEN4: Temporal Profile Traces (samples=2 seen=4): -# SAMPLE1: Temporal Profile Trace 0 (count=3): +# SAMPLE1: Temporal Profile Trace 0 (weight=1 count=3): # SAMPLE1: a # SAMPLE1: b # SAMPLE1: c -# SAMPLE2: Temporal Profile Trace 1 (count=3): +# SAMPLE2: Temporal Profile Trace 1 (weight=1 count=3): # SAMPLE2: a # SAMPLE2: b # SAMPLE2: c @@ -27,6 +27,8 @@ 1 # Trace Stream Size: 1 +# Weight +1 a, b, c diff --git a/llvm/test/tools/llvm-profdata/read-traces.proftext b/llvm/test/tools/llvm-profdata/read-traces.proftext --- a/llvm/test/tools/llvm-profdata/read-traces.proftext +++ b/llvm/test/tools/llvm-profdata/read-traces.proftext @@ -6,15 +6,15 @@ # RUN: llvm-profdata show --temporal-profile-traces %t.1.proftext | FileCheck %s # CHECK: Temporal Profile Traces (samples=3 seen=3): -# CHECK: Temporal Profile Trace 0 (count=3): +# CHECK: Temporal Profile Trace 0 (weight=1 count=3): # CHECK: foo # CHECK: bar # CHECK: goo -# CHECK: Temporal Profile Trace 1 (count=3): +# CHECK: Temporal Profile Trace 1 (weight=3 count=3): # CHECK: foo # CHECK: goo # CHECK: bar -# CHECK: Temporal Profile Trace 2 (count=1): +# CHECK: Temporal Profile Trace 2 (weight=1 count=1): # CHECK: goo # Header @@ -24,8 +24,14 @@ 3 # Trace Stream Size: 3 +# Weight +1 foo, bar, goo +# Weight +3 foo,goo,bar, +# Weight +1 goo foo diff --git a/llvm/test/tools/llvm-profdata/trace-limit.proftext b/llvm/test/tools/llvm-profdata/trace-limit.proftext --- a/llvm/test/tools/llvm-profdata/trace-limit.proftext +++ b/llvm/test/tools/llvm-profdata/trace-limit.proftext @@ -1,16 +1,16 @@ -# RUN: llvm-profdata merge --max-temporal-profile-trace-length=0 %s -o %t.profdata +# RUN: llvm-profdata merge --temporal-profile-max-trace-length=0 %s -o %t.profdata # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefix=NONE -# RUN: llvm-profdata merge --max-temporal-profile-trace-length=2 %s -o %t.profdata +# RUN: llvm-profdata merge --temporal-profile-max-trace-length=2 %s -o %t.profdata # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=CHECK,SOME -# RUN: llvm-profdata merge --max-temporal-profile-trace-length=1000 %s -o %t.profdata +# RUN: llvm-profdata merge --temporal-profile-max-trace-length=1000 %s -o %t.profdata # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=CHECK,ALL # NONE: Temporal Profile Traces (samples=0 seen=0): # CHECK: Temporal Profile Traces (samples=1 seen=1): -# SOME: Trace 0 (count=2): -# ALL: Trace 0 (count=3): +# SOME: Trace 0 (weight=1 count=2): +# ALL: Trace 0 (weight=1 count=3): # Header :ir @@ -19,6 +19,8 @@ 1 # Trace Stream Size: 1 +# Weight +1 a, b, c diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -344,7 +344,7 @@ } if (Reader->hasTemporalProfile()) { - auto &Traces = Reader->getTemporalProfTraces(); + auto &Traces = Reader->getTemporalProfTraces(Input.Weight); if (!Traces.empty()) WC->Writer.addTemporalProfileTraces( Traces, Reader->getTemporalProfTraceStreamSize()); @@ -1279,8 +1279,8 @@ "temporal-profile-trace-reservoir-size", cl::init(100), cl::desc("The maximum number of stored temporal profile traces (default: " "100)")); - cl::opt MaxTemporalProfTraceLength( - "max-temporal-profile-trace-length", cl::init(10000), + cl::opt TemporalProfMaxTraceLength( + "temporal-profile-max-trace-length", cl::init(10000), cl::desc("The maximum length of a single temporal profile trace " "(default: 10000)")); @@ -1326,7 +1326,7 @@ mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), OutputFilename, OutputFormat, TemporalProfTraceReservoirSize, - MaxTemporalProfTraceLength, OutputSparse, NumThreads, + TemporalProfMaxTraceLength, OutputSparse, NumThreads, FailureMode, ProfiledBinary); else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, @@ -2635,9 +2635,9 @@ OS << "Temporal Profile Traces (samples=" << Traces.size() << " seen=" << Reader->getTemporalProfTraceStreamSize() << "):\n"; for (unsigned i = 0; i < Traces.size(); i++) { - OS << " Temporal Profile Trace " << i << " (count=" << Traces[i].size() - << "):\n"; - for (auto &NameRef : Traces[i]) + OS << " Temporal Profile Trace " << i << " (weight=" << Traces[i].Weight + << " count=" << Traces[i].FunctionNameRefs.size() << "):\n"; + for (auto &NameRef : Traces[i].FunctionNameRefs) OS << " " << Reader->getSymtab().getFuncName(NameRef) << "\n"; } } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -39,6 +39,14 @@ return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n"; } +namespace llvm { +bool operator==(const TemporalProfTraceTy &lhs, + const TemporalProfTraceTy &rhs) { + return lhs.Weight == rhs.Weight && + lhs.FunctionNameRefs == rhs.FunctionNameRefs; +} +} // end namespace llvm + namespace { struct InstrProfTest : ::testing::Test { @@ -234,13 +242,15 @@ ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto LargeTrace = {IndexedInstrProf::ComputeHash("foo"), - IndexedInstrProf::ComputeHash("bar"), - IndexedInstrProf::ComputeHash("goo")}; - auto SmallTrace = {IndexedInstrProf::ComputeHash("foo"), - IndexedInstrProf::ComputeHash("bar")}; + TemporalProfTraceTy LargeTrace, SmallTrace; + LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"), + IndexedInstrProf::ComputeHash("bar"), + IndexedInstrProf::ComputeHash("goo")}; + SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"), + IndexedInstrProf::ComputeHash("bar")}; - Writer.addTemporalProfileTraces({LargeTrace, SmallTrace}, 2); + SmallVector Traces = {LargeTrace, SmallTrace}; + Writer.addTemporalProfileTraces(Traces, 2); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); @@ -261,11 +271,13 @@ ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto FooTrace = {IndexedInstrProf::ComputeHash("foo")}; - auto BarTrace = {IndexedInstrProf::ComputeHash("bar")}; + TemporalProfTraceTy FooTrace, BarTrace; + FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")}; + BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")}; - Writer.addTemporalProfileTraces({FooTrace}, 1); - Writer2.addTemporalProfileTraces({BarTrace}, 1); + SmallVector Traces1({FooTrace}), Traces2({BarTrace}); + Writer.addTemporalProfileTraces(Traces1, 1); + Writer2.addTemporalProfileTraces(Traces2, 1); Writer.mergeRecordsFromWriter(std::move(Writer2), Err); auto Profile = Writer.writeBuffer(); @@ -284,15 +296,19 @@ ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile), Succeeded()); - auto FooTrace = {IndexedInstrProf::ComputeHash("foo")}; - auto BarTrace = {IndexedInstrProf::ComputeHash("bar")}; - auto GooTrace = {IndexedInstrProf::ComputeHash("Goo")}; + TemporalProfTraceTy FooTrace, BarTrace, GooTrace; + FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")}; + BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")}; + GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("Goo")}; // Add some sampled traces - Writer.addTemporalProfileTraces({FooTrace, BarTrace, GooTrace}, 5); + SmallVector SampledTraces = {FooTrace, BarTrace, GooTrace}; + Writer.addTemporalProfileTraces(SampledTraces, 5); // Add some unsampled traces - Writer.addTemporalProfileTraces({BarTrace, GooTrace}, 2); - Writer.addTemporalProfileTraces({FooTrace}, 1); + SmallVector UnsampledTraces = {BarTrace, GooTrace}; + Writer.addTemporalProfileTraces(UnsampledTraces, 2); + UnsampledTraces = {FooTrace}; + Writer.addTemporalProfileTraces(UnsampledTraces, 1); auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile));