Index: llvm/trunk/docs/CommandGuide/llvm-profdata.rst =================================================================== --- llvm/trunk/docs/CommandGuide/llvm-profdata.rst +++ llvm/trunk/docs/CommandGuide/llvm-profdata.rst @@ -90,6 +90,12 @@ Emit the profile using GCC's gcov format (Not yet supported). + .. option:: -sparse[=true|false] + + Do not emit function records with 0 execution count. Can only be used in + conjunction with -instr. Defaults to false, since it can inhibit compiler + optimization during PGO. + EXAMPLES ^^^^^^^^ Basic Usage Index: llvm/trunk/include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- llvm/trunk/include/llvm/ProfileData/InstrProfWriter.h +++ llvm/trunk/include/llvm/ProfileData/InstrProfWriter.h @@ -32,13 +32,14 @@ typedef SmallDenseMap ProfilingData; private: + bool Sparse; StringMap FunctionData; uint64_t MaxFunctionCount; // Use raw pointer here for the incomplete type object. InstrProfRecordWriterTrait *InfoObj; public: - InstrProfWriter(); + InstrProfWriter(bool Sparse = false); ~InstrProfWriter(); /// Add function counts for the given function. If there are already counts @@ -57,8 +58,10 @@ // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); + void setOutputSparse(bool Sparse); private: + bool shouldEncodeData(const ProfilingData &PD); void writeImpl(ProfOStream &OS); }; Index: llvm/trunk/lib/ProfileData/InstrProfWriter.cpp =================================================================== --- llvm/trunk/lib/ProfileData/InstrProfWriter.cpp +++ llvm/trunk/lib/ProfileData/InstrProfWriter.cpp @@ -139,8 +139,8 @@ }; } -InstrProfWriter::InstrProfWriter() - : FunctionData(), MaxFunctionCount(0), +InstrProfWriter::InstrProfWriter(bool Sparse) + : Sparse(Sparse), FunctionData(), MaxFunctionCount(0), InfoObj(new InstrProfRecordWriterTrait()) {} InstrProfWriter::~InstrProfWriter() { delete InfoObj; } @@ -150,6 +150,9 @@ support::endianness Endianness) { InfoObj->ValueProfDataEndianness = Endianness; } +void InstrProfWriter::setOutputSparse(bool Sparse) { + this->Sparse = Sparse; +} std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) { @@ -184,11 +187,24 @@ return Result; } +bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { + if (!Sparse) + return true; + for (const auto &Func : PD) { + const InstrProfRecord &IPR = Func.second; + if (std::any_of(IPR.Counts.begin(), IPR.Counts.end(), + [](uint64_t Count) { return Count > 0; })) + return true; + } + return false; +} + void InstrProfWriter::writeImpl(ProfOStream &OS) { OnDiskChainedHashTableGenerator Generator; // Populate the hash table generator. for (const auto &I : FunctionData) - Generator.insert(I.getKey(), &I.getValue()); + if (shouldEncodeData(I.getValue())) + Generator.insert(I.getKey(), &I.getValue()); // Write the header. IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; @@ -279,10 +295,12 @@ void InstrProfWriter::writeText(raw_fd_ostream &OS) { InstrProfSymtab Symtab; for (const auto &I : FunctionData) - Symtab.addFuncName(I.getKey()); + if (shouldEncodeData(I.getValue())) + Symtab.addFuncName(I.getKey()); Symtab.finalizeSymtab(); for (const auto &I : FunctionData) - for (const auto &Func : I.getValue()) - writeRecordInText(Func.second, Symtab, OS); + if (shouldEncodeData(I.getValue())) + for (const auto &Func : I.getValue()) + writeRecordInText(Func.second, Symtab, OS); } Index: llvm/trunk/test/tools/llvm-profdata/general.proftext =================================================================== --- llvm/trunk/test/tools/llvm-profdata/general.proftext +++ llvm/trunk/test/tools/llvm-profdata/general.proftext @@ -1,6 +1,6 @@ +# RUN: llvm-profdata merge -sparse=true %s -o %t.profdata - -# RUN: llvm-profdata merge %s -o %t.profdata +# RUN: llvm-profdata merge -sparse=false %s -o %t.profdata.dense # RUN: llvm-profdata show %t.profdata --function function_count_only --counts | FileCheck %s -check-prefix=FUNC_COUNT_ONLY function_count_only @@ -12,7 +12,8 @@ # FUNC_COUNT_ONLY-NEXT: Function count: 97531 # FUNC_COUNT_ONLY-NEXT: Block counts: [] -# RUN: llvm-profdata show %t.profdata --function "name with spaces" --counts | FileCheck %s -check-prefix=SPACES +# RUN: llvm-profdata show %t.profdata.dense --function "name with spaces" --counts | FileCheck %s -check-prefix=SPACES +# RUN: llvm-profdata show %t.profdata --function "name with spaces" --counts | FileCheck %s --check-prefix=SPARSE_SPACES name with spaces 1024 2 @@ -22,6 +23,7 @@ # SPACES-NEXT: Counters: 2 # SPACES-NEXT: Function count: 0 # SPACES-NEXT: Block counts: [0] +# SPARSE_SPACES-NOT: Function count: 0 # RUN: llvm-profdata show %t.profdata --function large_numbers --counts | FileCheck %s -check-prefix=LARGENUM large_numbers @@ -38,7 +40,7 @@ # LARGENUM-NEXT: Function count: 2305843009213693952 # LARGENUM-NEXT: Block counts: [1152921504606846976, 576460752303423488, 288230376151711744, 144115188075855872, 72057594037927936] -# RUN: llvm-profdata show %t.profdata --function hex_hash | FileCheck %s -check-prefix=HEX-HASH +# RUN: llvm-profdata show %t.profdata.dense --function hex_hash | FileCheck %s -check-prefix=HEX-HASH hex_hash 0x1234 1 @@ -51,19 +53,21 @@ # NOSUCHFUNC: Functions shown: 0 # RUN: llvm-profdata show %t.profdata --function _ | FileCheck %s -check-prefix=SOMEFUNCS +# RUN: llvm-profdata show %t.profdata.dense --function _ | FileCheck %s -check-prefix=SOMEFUNCS_DENSE # SOMEFUNCS: Counters: # SOMEFUNCS: function_count_only: # SOMEFUNCS: large_numbers: -# SOMEFUNCS: Functions shown: 3 +# SOMEFUNCS: Functions shown: 2 +# SOMEFUNCS_DENSE: Functions shown: 3 -# RUN: llvm-profdata show %t.profdata | FileCheck %s -check-prefix=SUMMARY +# RUN: llvm-profdata show %t.profdata.dense | FileCheck %s -check-prefix=SUMMARY # SUMMARY-NOT: Counters: # SUMMARY-NOT: Functions shown: # SUMMARY: Total functions: 4 # SUMMARY: Maximum function count: 2305843009213693952 # SUMMARY: Maximum internal block count: 1152921504606846976 -# RUN: llvm-profdata show --detailed-summary %t.profdata | FileCheck %s -check-prefix=DETAILED-SUMMARY +# RUN: llvm-profdata show --detailed-summary %t.profdata.dense | FileCheck %s -check-prefix=DETAILED-SUMMARY # DETAILED-SUMMARY: Detailed summary: # DETAILED-SUMMARY: Total number of blocks: 10 # DETAILED-SUMMARY: Total count: 4539628424389557499 Index: llvm/trunk/tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- llvm/trunk/tools/llvm-profdata/llvm-profdata.cpp +++ llvm/trunk/tools/llvm-profdata/llvm-profdata.cpp @@ -107,7 +107,7 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, StringRef OutputFilename, - ProfileFormat OutputFormat) { + ProfileFormat OutputFormat, bool OutputSparse) { if (OutputFilename.compare("-") == 0) exitWithError("Cannot write indexed profdata format to stdout."); @@ -119,7 +119,7 @@ if (EC) exitWithErrorCode(EC, OutputFilename); - InstrProfWriter Writer; + InstrProfWriter Writer(OutputSparse); SmallSet WriterErrorCodes; for (const auto &Input : Inputs) { auto ReaderOrErr = InstrProfReader::create(Input.Filename); @@ -228,6 +228,9 @@ "GCC encoding (only meaningful for -sample)"), clEnumValEnd)); + cl::opt OutputSparse("sparse", cl::init(false), + cl::desc("Generate a sparse profile (only meaningful for -instr)")); + cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); if (InputFilenames.empty() && WeightedInputFilenames.empty()) @@ -241,7 +244,8 @@ WeightedInputs.push_back(parseWeightedFile(WeightedFilename)); if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, OutputFilename, OutputFormat); + mergeInstrProfile(WeightedInputs, OutputFilename, OutputFormat, + OutputSparse); else mergeSampleProfile(WeightedInputs, OutputFilename, OutputFormat); Index: llvm/trunk/unittests/ProfileData/CoverageMappingTest.cpp =================================================================== --- llvm/trunk/unittests/ProfileData/CoverageMappingTest.cpp +++ llvm/trunk/unittests/ProfileData/CoverageMappingTest.cpp @@ -92,6 +92,7 @@ void SetUp() override { NextFile = 0; + ProfileWriter.setOutputSparse(false); } unsigned getFile(StringRef Name) { @@ -154,7 +155,16 @@ } }; -TEST_F(CoverageMappingTest, basic_write_read) { +struct MaybeSparseCoverageMappingTest + : public CoverageMappingTest, + public ::testing::WithParamInterface { + void SetUp() { + CoverageMappingTest::SetUp(); + ProfileWriter.setOutputSparse(GetParam()); + } +}; + +TEST_P(MaybeSparseCoverageMappingTest, basic_write_read) { addCMR(Counter::getCounter(0), "foo", 1, 1, 1, 1); addCMR(Counter::getCounter(1), "foo", 2, 1, 2, 2); addCMR(Counter::getZero(), "foo", 3, 1, 3, 4); @@ -174,7 +184,7 @@ } } -TEST_F(CoverageMappingTest, expansion_gets_first_counter) { +TEST_P(MaybeSparseCoverageMappingTest, expansion_gets_first_counter) { addCMR(Counter::getCounter(1), "foo", 10, 1, 10, 2); // This starts earlier in "foo", so the expansion should get its counter. addCMR(Counter::getCounter(2), "foo", 1, 1, 20, 1); @@ -187,7 +197,7 @@ ASSERT_EQ(3U, OutputCMRs[2].LineStart); } -TEST_F(CoverageMappingTest, basic_coverage_iteration) { +TEST_P(MaybeSparseCoverageMappingTest, basic_coverage_iteration) { InstrProfRecord Record("func", 0x1234, {30, 20, 10, 0}); ProfileWriter.addRecord(std::move(Record)); readProfCounts(); @@ -210,7 +220,7 @@ ASSERT_EQ(CoverageSegment(11, 11, false), Segments[6]); } -TEST_F(CoverageMappingTest, uncovered_function) { +TEST_P(MaybeSparseCoverageMappingTest, uncovered_function) { readProfCounts(); addCMR(Counter::getZero(), "file1", 1, 2, 3, 4); @@ -223,7 +233,7 @@ ASSERT_EQ(CoverageSegment(3, 4, false), Segments[1]); } -TEST_F(CoverageMappingTest, uncovered_function_with_mapping) { +TEST_P(MaybeSparseCoverageMappingTest, uncovered_function_with_mapping) { readProfCounts(); addCMR(Counter::getCounter(0), "file1", 1, 1, 9, 9); @@ -238,7 +248,7 @@ ASSERT_EQ(CoverageSegment(9, 9, false), Segments[2]); } -TEST_F(CoverageMappingTest, combine_regions) { +TEST_P(MaybeSparseCoverageMappingTest, combine_regions) { InstrProfRecord Record("func", 0x1234, {10, 20, 30}); ProfileWriter.addRecord(std::move(Record)); readProfCounts(); @@ -257,9 +267,11 @@ ASSERT_EQ(CoverageSegment(9, 9, false), Segments[3]); } -TEST_F(CoverageMappingTest, dont_combine_expansions) { - InstrProfRecord Record("func", 0x1234, {10, 20}); - ProfileWriter.addRecord(std::move(Record)); +TEST_P(MaybeSparseCoverageMappingTest, dont_combine_expansions) { + InstrProfRecord Record1("func", 0x1234, {10, 20}); + InstrProfRecord Record2("func", 0x1234, {0, 0}); + ProfileWriter.addRecord(std::move(Record1)); + ProfileWriter.addRecord(std::move(Record2)); readProfCounts(); addCMR(Counter::getCounter(0), "file1", 1, 1, 9, 9); @@ -277,8 +289,8 @@ ASSERT_EQ(CoverageSegment(9, 9, false), Segments[3]); } -TEST_F(CoverageMappingTest, strip_filename_prefix) { - InstrProfRecord Record("file1:func", 0x1234, {10}); +TEST_P(MaybeSparseCoverageMappingTest, strip_filename_prefix) { + InstrProfRecord Record("file1:func", 0x1234, {0}); ProfileWriter.addRecord(std::move(Record)); readProfCounts(); @@ -292,4 +304,7 @@ ASSERT_EQ("func", Names[0]); } +INSTANTIATE_TEST_CASE_P(MaybeSparse, MaybeSparseCoverageMappingTest, + ::testing::Bool()); + } // end anonymous namespace Index: llvm/trunk/unittests/ProfileData/InstrProfTest.cpp =================================================================== --- llvm/trunk/unittests/ProfileData/InstrProfTest.cpp +++ llvm/trunk/unittests/ProfileData/InstrProfTest.cpp @@ -39,6 +39,8 @@ InstrProfWriter Writer; std::unique_ptr Reader; + void SetUp() { Writer.setOutputSparse(false); } + void readProfile(std::unique_ptr Profile) { auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile)); ASSERT_TRUE(NoError(ReaderOrErr.getError())); @@ -46,13 +48,24 @@ } }; -TEST_F(InstrProfTest, write_and_read_empty_profile) { +struct SparseInstrProfTest : public InstrProfTest { + void SetUp() { Writer.setOutputSparse(true); } +}; + +struct MaybeSparseInstrProfTest : public InstrProfTest, + public ::testing::WithParamInterface { + void SetUp() { + Writer.setOutputSparse(GetParam()); + } +}; + +TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) { auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); ASSERT_TRUE(Reader->begin() == Reader->end()); } -TEST_F(InstrProfTest, write_and_read_one_function) { +TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) { InstrProfRecord Record("foo", 0x1234, {1, 2, 3, 4}); Writer.addRecord(std::move(Record)); auto Profile = Writer.writeBuffer(); @@ -70,7 +83,7 @@ ASSERT_TRUE(++I == E); } -TEST_F(InstrProfTest, get_instr_prof_record) { +TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) { InstrProfRecord Record1("foo", 0x1234, {1, 2}); InstrProfRecord Record2("foo", 0x1235, {3, 4}); Writer.addRecord(std::move(Record1)); @@ -97,7 +110,7 @@ ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.getError())); } -TEST_F(InstrProfTest, get_function_counts) { +TEST_P(MaybeSparseInstrProfTest, get_function_counts) { InstrProfRecord Record1("foo", 0x1234, {1, 2}); InstrProfRecord Record2("foo", 0x1235, {3, 4}); Writer.addRecord(std::move(Record1)); @@ -124,7 +137,7 @@ ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, EC)); } -TEST_F(InstrProfTest, get_icall_data_read_write) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) { InstrProfRecord Record1("caller", 0x1234, {1, 2}); InstrProfRecord Record2("callee1", 0x1235, {3, 4}); InstrProfRecord Record3("callee2", 0x1235, {3, 4}); @@ -171,7 +184,7 @@ ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); } -TEST_F(InstrProfTest, get_icall_data_read_write_with_weight) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) { InstrProfRecord Record1("caller", 0x1234, {1, 2}); InstrProfRecord Record2("callee1", 0x1235, {3, 4}); InstrProfRecord Record3("callee2", 0x1235, {3, 4}); @@ -217,7 +230,7 @@ ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1")); } -TEST_F(InstrProfTest, get_icall_data_read_write_big_endian) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) { InstrProfRecord Record1("caller", 0x1234, {1, 2}); InstrProfRecord Record2("callee1", 0x1235, {3, 4}); InstrProfRecord Record3("callee2", 0x1235, {3, 4}); @@ -269,7 +282,7 @@ Writer.setValueProfDataEndianness(support::little); } -TEST_F(InstrProfTest, get_icall_data_merge1) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) { static const char caller[] = "caller"; static const char callee1[] = "callee1"; static const char callee2[] = "callee2"; @@ -384,7 +397,7 @@ ASSERT_EQ(2U, VD_4[2].Count); } -TEST_F(InstrProfTest, get_icall_data_merge1_saturation) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) { static const char bar[] = "bar"; const uint64_t Max = std::numeric_limits::max(); @@ -438,7 +451,7 @@ // This test tests that when there are too many values // for a given site, the merged results are properly // truncated. -TEST_F(InstrProfTest, get_icall_data_merge_site_trunc) { +TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) { static const char caller[] = "caller"; InstrProfRecord Record11(caller, 0x1234, {1, 2}); @@ -508,7 +521,7 @@ nullptr}; static uint16_t NumValueSites[IPVK_Last + 1] = {5}; -TEST_F(InstrProfTest, runtime_value_prof_data_read_write) { +TEST_P(MaybeSparseInstrProfTest, runtime_value_prof_data_read_write) { ValueProfRuntimeRecord RTRecord; initializeValueProfRuntimeRecord(&RTRecord, &NumValueSites[0], &ValueProfNodes[0]); @@ -578,7 +591,7 @@ free(VPData); } -TEST_F(InstrProfTest, get_max_function_count) { +TEST_P(MaybeSparseInstrProfTest, get_max_function_count) { InstrProfRecord Record1("foo", 0x1234, {1ULL << 31, 2}); InstrProfRecord Record2("bar", 0, {1ULL << 63}); InstrProfRecord Record3("baz", 0x5678, {0, 0, 0, 0}); @@ -591,7 +604,7 @@ ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount()); } -TEST_F(InstrProfTest, get_weighted_function_counts) { +TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) { InstrProfRecord Record1("foo", 0x1234, {1, 2}); InstrProfRecord Record2("foo", 0x1235, {3, 4}); Writer.addRecord(std::move(Record1), 3); @@ -611,7 +624,7 @@ ASSERT_EQ(20U, Counts[1]); } -TEST_F(InstrProfTest, instr_prof_symtab_test) { +TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_test) { std::vector FuncNames; FuncNames.push_back("func1"); FuncNames.push_back("func2"); @@ -662,7 +675,7 @@ ASSERT_EQ(StringRef("bar3"), R); } -TEST_F(InstrProfTest, instr_prof_symtab_module_test) { +TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_module_test) { LLVMContext Ctx; std::unique_ptr M = llvm::make_unique("MyModule.cpp", Ctx); FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), @@ -697,7 +710,7 @@ } } -TEST_F(InstrProfTest, instr_prof_symtab_compression_test) { +TEST_P(MaybeSparseInstrProfTest, instr_prof_symtab_compression_test) { std::vector FuncNames1; std::vector FuncNames2; for (int I = 0; I < 10 * 1024; I++) { @@ -768,4 +781,22 @@ } } +TEST_F(SparseInstrProfTest, preserve_no_records) { + InstrProfRecord Record1("foo", 0x1234, {0}); + InstrProfRecord Record2("bar", 0x4321, {0, 0}); + InstrProfRecord Record3("bar", 0x4321, {0, 0, 0}); + + Writer.addRecord(std::move(Record1)); + Writer.addRecord(std::move(Record2)); + Writer.addRecord(std::move(Record3)); + auto Profile = Writer.writeBuffer(); + readProfile(std::move(Profile)); + + auto I = Reader->begin(), E = Reader->end(); + ASSERT_TRUE(I == E); +} + +INSTANTIATE_TEST_CASE_P(MaybeSparse, MaybeSparseInstrProfTest, + ::testing::Bool()); + } // end anonymous namespace