Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -678,6 +678,15 @@ /* Raw profile format version. */ #define INSTR_PROF_RAW_VERSION 2 +/* Profile version is always of type uint_64. We set bit 60 to 1 if if this is + * an IR-level instrumentaiton generated profile and 0 if this is a Clang FE + * generated profile. + */ +#define IR_LEVEL_PROFILE_FLAG ((uint64_t)0x1<<60) +#define SET_IR_LEVEL_PROFILE_FLAG(x) ((x | IR_LEVEL_PROFILE_FLAG)) +#define UNSET_IR_LEVEL_PROFILE_FLAG(x) ((x & ~IR_LEVEL_PROFILE_FLAG)) +#define IS_IR_LEVEL_PROFILE(x) (((x& IR_LEVEL_PROFILE_FLAG) != 0)) + /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -64,6 +64,7 @@ /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + virtual uint64_t getVersion() const = 0; protected: /// Set the current std::error_code and return same. @@ -105,6 +106,7 @@ std::unique_ptr DataBuffer; /// Iterator over the profile data. line_iterator Line; + uint64_t Version; TextInstrProfReader(const TextInstrProfReader &) = delete; TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; @@ -115,8 +117,10 @@ /// Return true if the given buffer is in text instrprof format. static bool hasFormat(const MemoryBuffer &Buffer); - /// Read the header. - std::error_code readHeader() override { return success(); } + uint64_t getVersion() const override { return Version; }; + + /// Read the header. Set the Version if it's available. + std::error_code readHeader() override; /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; }; @@ -134,6 +138,7 @@ /// The profile data file contents. std::unique_ptr DataBuffer; bool ShouldSwapBytes; + uint64_t Version; uint64_t CountersDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; @@ -158,6 +163,7 @@ static bool hasFormat(const MemoryBuffer &DataBuffer); std::error_code readHeader() override; std::error_code readNextRecord(InstrProfRecord &Record) override; + uint64_t getVersion() const override { return Version; }; private: std::error_code readNextHeader(const char *CurrentPos); @@ -322,7 +328,7 @@ IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; public: - uint64_t getVersion() const { return Index->getVersion(); } + uint64_t getVersion() const override { return Index->getVersion(); } IndexedInstrProfReader(std::unique_ptr DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} Index: include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- include/llvm/ProfileData/InstrProfWriter.h +++ include/llvm/ProfileData/InstrProfWriter.h @@ -40,11 +40,11 @@ /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is /// summed. - std::error_code addRecord(InstrProfRecord &&I); + std::error_code addRecord(InstrProfRecord &&I, bool IsLIRInstr = false); /// Write the profile to \c OS - void write(raw_fd_ostream &OS); + void write(raw_fd_ostream &OS, bool IsIRInstr = false); /// Write the profile in text format to \c OS - void writeText(raw_fd_ostream &OS); + void writeText(raw_fd_ostream &OS, bool IsIRInstr = false); /// Write \c Record in text format to \c OS static void writeRecordInText(const InstrProfRecord &Record, raw_fd_ostream &OS); @@ -55,7 +55,7 @@ void setValueProfDataEndianness(support::endianness Endianness); private: - std::pair writeImpl(raw_ostream &OS); + std::pair writeImpl(raw_ostream &OS, bool IsIRInstr = false); }; } // end namespace llvm Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -109,6 +109,24 @@ [](char c) { return ::isprint(c) || ::isspace(c); }); } +std::error_code TextInstrProfReader::readHeader() { + bool IsIRInstr = false; + while (!Line.is_at_end() && + (Line->empty() || Line->startswith("#") || Line->startswith(":"))) { + if (Line->startswith(":")) { + StringRef Str = (Line)->substr(1); + int32_t V = 0; + if (!Str.getAsInteger(0, V)) + IsIRInstr = (V == 1); + } + ++Line; + } + if (IsIRInstr) + // Text format does not have a version. So only set the flag. + Version = SET_IR_LEVEL_PROFILE_FLAG(0); + return success(); +} + std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { // Skip empty lines and comments. while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) @@ -202,7 +220,8 @@ template std::error_code RawInstrProfReader::readHeader( const RawInstrProf::Header &Header) { - if (swap(Header.Version) != RawInstrProf::Version) + Version = swap(Header.Version); + if (UNSET_IR_LEVEL_PROFILE_FLAG(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); @@ -501,7 +520,7 @@ // Read the version. uint64_t FormatVersion = endian::byte_swap(Header->Version); - if (FormatVersion > IndexedInstrProf::Version) + if (UNSET_IR_LEVEL_PROFILE_FLAG(FormatVersion) > IndexedInstrProf::Version) return error(instrprof_error::unsupported_version); // Read the maximal function count. Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -98,7 +98,7 @@ I.updateStrings(&StringTable); } -std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I) { +std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I, bool IsIRInstr) { updateStringTableReferences(I); auto &ProfileDataMap = FunctionData[I.Name]; @@ -120,13 +120,20 @@ // We keep track of the max function count as we go for simplicity. // Update this statistic no matter the result of the merge. - if (Dest.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Dest.Counts[0]; + if (!IsIRInstr) { + if (Dest.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Dest.Counts[0]; + } else { + for (auto &IT : I.Counts) + if (IT > MaxFunctionCount) + MaxFunctionCount = IT; + } return Result; } -std::pair InstrProfWriter::writeImpl(raw_ostream &OS) { +std::pair InstrProfWriter::writeImpl(raw_ostream &OS, + bool IsIRInstr) { OnDiskChainedHashTableGenerator Generator; // Populate the hash table generator. @@ -139,7 +146,10 @@ // Write the header. IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; - Header.Version = IndexedInstrProf::Version; + if (IsIRInstr) + Header.Version = SET_IR_LEVEL_PROFILE_FLAG(IndexedInstrProf::Version); + else + Header.Version = IndexedInstrProf::Version; Header.MaxFunctionCount = MaxFunctionCount; Header.HashType = static_cast(IndexedInstrProf::HashType); Header.HashOffset = 0; @@ -161,9 +171,9 @@ return std::make_pair(HashTableStartLoc, HashTableStart); } -void InstrProfWriter::write(raw_fd_ostream &OS) { +void InstrProfWriter::write(raw_fd_ostream &OS, bool IsIRInstr) { // Write the hash table. - auto TableStart = writeImpl(OS); + auto TableStart = writeImpl(OS, IsIRInstr); // Go back and fill in the hash table start. using namespace support; @@ -184,7 +194,9 @@ OS << "\n"; } -void InstrProfWriter::writeText(raw_fd_ostream &OS) { +void InstrProfWriter::writeText(raw_fd_ostream &OS, bool IsIRInstr) { + if (IsIRInstr) + OS << "# IR level Instrumentation Flag:\n:" << 1 << "\n"; for (const auto &I : FunctionData) for (const auto &Func : I.getValue()) writeRecordInText(Func.second, OS); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -666,6 +666,18 @@ } // end anonymous namespace bool PGOInstrumentationGen::runOnModule(Module &M) { + // Create a COMDAT variable __llvm_profile_ir_level to let runtime + // aware this is ir_level profile so it can set the version flag. + Type *IntTy32 = Type::getInt32Ty(M.getContext()); + auto IRLevelFlagVariable = + new GlobalVariable(M, IntTy32, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy32, APInt(32, 1)), + "__llvm_profile_ir_level"); + IRLevelFlagVariable->setVisibility(GlobalValue::DefaultVisibility); + IRLevelFlagVariable->setAlignment(8); + IRLevelFlagVariable->setComdat( + M.getOrInsertComdat(StringRef("__llvm_profile_ir_level"))); + for (auto &F : M) { if (F.isDeclaration()) continue; @@ -703,6 +715,12 @@ "Cannot get PGOReader")); return false; } + if (!IS_IR_LEVEL_PROFILE(PGOReader->getVersion())) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + ProfileFileName.data(), "Not an IR level instrumentation profile")); + return false; + } + for (auto &F : M) { if (F.isDeclaration()) Index: test/Transforms/PGOProfile/Inputs/branch1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch1.proftext +++ test/Transforms/PGOProfile/Inputs/branch1.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_br_1 25571299074 2 Index: test/Transforms/PGOProfile/Inputs/branch2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch2.proftext +++ test/Transforms/PGOProfile/Inputs/branch2.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_br_2 29667547796 2 Index: test/Transforms/PGOProfile/Inputs/criticaledge.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/criticaledge.proftext +++ test/Transforms/PGOProfile/Inputs/criticaledge.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_criticalEdge 82323253069 8 Index: test/Transforms/PGOProfile/Inputs/diag.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/diag.proftext +++ test/Transforms/PGOProfile/Inputs/diag.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 foo 12884999999 1 Index: test/Transforms/PGOProfile/Inputs/landingpad.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/landingpad.proftext +++ test/Transforms/PGOProfile/Inputs/landingpad.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 foo 59130013419 4 Index: test/Transforms/PGOProfile/Inputs/loop1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop1.proftext +++ test/Transforms/PGOProfile/Inputs/loop1.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_simple_for 34137660316 2 Index: test/Transforms/PGOProfile/Inputs/loop2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop2.proftext +++ test/Transforms/PGOProfile/Inputs/loop2.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_nested_for 53929068288 3 Index: test/Transforms/PGOProfile/Inputs/switch.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/switch.proftext +++ test/Transforms/PGOProfile/Inputs/switch.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_switch 46200943743 4 Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -108,6 +108,8 @@ exitWithErrorCode(EC, OutputFilename); InstrProfWriter Writer; + bool FirstInput = true; + bool IsIRInstrMerge; SmallSet WriterErrorCodes; for (const auto &Filename : Inputs) { auto ReaderOrErr = InstrProfReader::create(Filename); @@ -115,8 +117,19 @@ exitWithErrorCode(ec, Filename); auto Reader = std::move(ReaderOrErr.get()); + uint64_t Version = Reader->getVersion(); + bool IsIRProfile = IS_IR_LEVEL_PROFILE(Version); + if (FirstInput) { + IsIRInstrMerge = IsIRProfile; + FirstInput = false; + } else { + if (IsIRProfile != IsIRInstrMerge) + exitWithError("Merge IR generated profile with Clang generated " + "profile."); + } + for (auto &I : *Reader) { - if (std::error_code EC = Writer.addRecord(std::move(I))) { + if (std::error_code EC = Writer.addRecord(std::move(I), IsIRInstrMerge)) { // Only show hint the first time an error occurs. bool firstTime = WriterErrorCodes.insert(EC).second; handleMergeWriterError(EC, Filename, I.Name, firstTime); @@ -126,9 +139,9 @@ exitWithErrorCode(Reader->getError(), Filename); } if (OutputFormat == PF_Text) - Writer.writeText(Output); + Writer.writeText(Output, IsIRInstrMerge); else - Writer.write(Output); + Writer.write(Output, IsIRInstrMerge); } static sampleprof::SampleProfileFormat FormatMap[] = { @@ -215,6 +228,8 @@ exitWithErrorCode(EC, Filename); auto Reader = std::move(ReaderOrErr.get()); + uint64_t Version = Reader->getVersion(); + bool IsIRInstr = IS_IR_LEVEL_PROFILE(Version); uint64_t MaxFunctionCount = 0, MaxBlockCount = 0; size_t ShownFunctions = 0, TotalFunctions = 0; for (const auto &Func : *Reader) { @@ -231,8 +246,10 @@ ++TotalFunctions; assert(Func.Counts.size() > 0 && "function missing entry counter"); - if (Func.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Func.Counts[0]; + if (!IsIRInstr) { + if (Func.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Func.Counts[0]; + } for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { if (Func.Counts[I] > MaxBlockCount) @@ -248,8 +265,10 @@ OS << " " << Func.Name << ":\n" << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" - << " Counters: " << Func.Counts.size() << "\n" - << " Function count: " << Func.Counts[0] << "\n"; + << " Version: " << format("0x%016" PRIx64, Version) << "\n" + << " Counters: " << Func.Counts.size() << "\n"; + if (!IsIRInstr) + OS << " Function count: " << Func.Counts[0] << "\n"; if (ShowIndirectCallTargets) OS << " Indirect Call Site Count: " @@ -257,8 +276,9 @@ if (ShowCounts) { OS << " Block counts: ["; - for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { - OS << (I == 1 ? "" : ", ") << Func.Counts[I]; + size_t Start = (IsIRInstr ? 0 : 1); + for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counts[I]; } OS << "]\n"; } @@ -288,8 +308,12 @@ if (ShowAllFunctions || !ShowFunction.empty()) OS << "Functions shown: " << ShownFunctions << "\n"; OS << "Total functions: " << TotalFunctions << "\n"; - OS << "Maximum function count: " << MaxFunctionCount << "\n"; - OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + if (!IsIRInstr) { + OS << "Maximum function count: " << MaxFunctionCount << "\n"; + OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + } else { + OS << "Maximum count: " << MaxBlockCount << "\n"; + } return 0; }