Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -678,6 +678,18 @@ /* Raw profile format version. */ #define INSTR_PROF_RAW_VERSION 2 +/* Profile version is always of type uint_64_t. Reserve the upper 8 bits in the + * version for other variants of profile. We set the lowest bit of the upper 8 + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * generated profile, and 0 if this is a Clang FE generated profile. + */ +#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define GET_VERSION(V) ((V)&~VARIANT_MASKS_ALL) +#define IR_LEVEL_PROF_VARNAME __llvm_profile_ir_level +#define IR_LEVEL_PROF_STRINGFY(s) #s +#define IR_LEVEL_PROF_STRING(s) IR_LEVEL_PROF_STRINGFY(s) + /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -64,6 +64,7 @@ /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + virtual bool isIRLevelProfile() const = 0; protected: /// Set the current std::error_code and return same. @@ -105,6 +106,7 @@ std::unique_ptr DataBuffer; /// Iterator over the profile data. line_iterator Line; + bool IsIRLevelProfile; // String table for holding a unique copy of all the strings in the profile. InstrProfStringTable StringTable; @@ -120,8 +122,10 @@ /// Return true if the given buffer is in text instrprof format. static bool hasFormat(const MemoryBuffer &Buffer); + bool isIRLevelProfile() const override { return IsIRLevelProfile; }; + /// Read the header. - std::error_code readHeader() override { return success(); } + std::error_code readHeader() override; /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; }; @@ -139,6 +143,8 @@ /// The profile data file contents. std::unique_ptr DataBuffer; bool ShouldSwapBytes; + // Version of the Raw profile. It contains the variant profile information. + uint64_t Version; uint64_t CountersDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; @@ -163,6 +169,9 @@ static bool hasFormat(const MemoryBuffer &DataBuffer); std::error_code readHeader() override; std::error_code readNextRecord(InstrProfRecord &Record) override; + bool isIRLevelProfile() const override { + return (Version & VARIANT_MASK_IR_PROF) != 0; + } private: std::error_code readNextHeader(const char *CurrentPos); @@ -310,7 +319,7 @@ HashTable->getInfoObj().setValueProfDataEndianness(Endianness); } ~InstrProfReaderIndex() override {} - uint64_t getVersion() const override { return FormatVersion; } + uint64_t getVersion() const { return FormatVersion; } }; /// Reader for the indexed binary instrprof format. @@ -327,7 +336,11 @@ IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; public: - uint64_t getVersion() const { return Index->getVersion(); } + /// Return the profile version. Note that is a unmasked version. + uint64_t getVersion() const { return GET_VERSION(Index->getVersion()); } + bool isIRLevelProfile() const override { + return (Index->getVersion() & VARIANT_MASK_IR_PROF) !=0; + } IndexedInstrProfReader(std::unique_ptr DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} Index: include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- include/llvm/ProfileData/InstrProfWriter.h +++ include/llvm/ProfileData/InstrProfWriter.h @@ -27,13 +27,15 @@ class InstrProfWriter { public: typedef SmallDenseMap ProfilingData; + enum ProfKind { PF_None = 0, PF_FE, PF_IRLevel }; private: InstrProfStringTable StringTable; StringMap FunctionData; uint64_t MaxFunctionCount; + ProfKind ProfileKind; public: - InstrProfWriter() : MaxFunctionCount(0) {} + InstrProfWriter() : MaxFunctionCount(0), ProfileKind(PF_None) {} /// Update string entries in profile data with references to StringTable. void updateStringTableReferences(InstrProfRecord &I); @@ -51,6 +53,16 @@ /// Write the profile, returning the raw data. For testing. std::unique_ptr writeBuffer(); + /// Set the ProfileKind. Report error if mixing FE and IR level profiles. + std::error_code setIsIRLevelProfile(bool IsIRLevel) { + if (ProfileKind == PF_None) { + ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + return instrprof_error::success; + } + return (IsIRLevel == (ProfileKind == PF_IRLevel)) ? + instrprof_error::success : instrprof_error::unsupported_version; + } + // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -109,6 +109,31 @@ [](char c) { return ::isprint(c) || ::isspace(c); }); } +// Read the profile variant flag from the header: ":0" means this is a FE +// generated profile. ":1" means this is an IR level profile. Other strings +// with a leading ':' will be reported an error format. +std::error_code TextInstrProfReader::readHeader() { + bool IsIRInstr = false; + while (!Line.is_at_end() && + (Line->empty() || Line->startswith("#") || Line->startswith(":"))) { + if (Line->startswith(":")) { + StringRef Str = (Line)->substr(1); + int32_t V = 0; + if (!Str.getAsInteger(0, V)) { + if (V == 1) + IsIRInstr = true; + else if (V == 0) + IsIRInstr = false; + else + return instrprof_error::bad_header; + } + } + ++Line; + } + IsIRLevelProfile = IsIRInstr; + return success(); +} + std::error_code TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { @@ -268,7 +293,8 @@ template std::error_code RawInstrProfReader::readHeader( const RawInstrProf::Header &Header) { - if (swap(Header.Version) != RawInstrProf::Version) + Version = swap(Header.Version); + if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); @@ -460,10 +486,10 @@ return data_type(); uint64_t Hash = endian::readNext(D); - // Initialize number of counters for FormatVersion == 1. + // Initialize number of counters for GET_VERSION(FormatVersion) == 1. uint64_t CountsSize = N / sizeof(uint64_t) - 1; // If format version is different then read the number of counters. - if (FormatVersion != 1) { + if (GET_VERSION(FormatVersion) != 1) { if (D + sizeof(uint64_t) > End) return data_type(); CountsSize = endian::readNext(D); @@ -480,7 +506,7 @@ DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. - if (FormatVersion > 2 && !readValueProfilingData(D, End)) { + if (GET_VERSION(FormatVersion) > 2 && !readValueProfilingData(D, End)) { DataBuffer.clear(); return data_type(); } @@ -567,7 +593,7 @@ // Read the version. uint64_t FormatVersion = endian::byte_swap(Header->Version); - if (FormatVersion > IndexedInstrProf::Version) + if (GET_VERSION(FormatVersion) > IndexedInstrProf::Version) return error(instrprof_error::unsupported_version); // Read the maximal function count. Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -130,8 +130,14 @@ // We keep track of the max function count as we go for simplicity. // Update this statistic no matter the result of the merge. - if (Dest.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Dest.Counts[0]; + if (ProfileKind != PF_IRLevel) { + if (Dest.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Dest.Counts[0]; + } else { + for (auto &IT : Dest.Counts) + if (IT > MaxFunctionCount) + MaxFunctionCount = IT; + } return Result; } @@ -149,7 +155,10 @@ // Write the header. IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; - Header.Version = IndexedInstrProf::Version; + if (ProfileKind == PF_IRLevel) + Header.Version = IndexedInstrProf::Version | VARIANT_MASK_IR_PROF; + else + Header.Version = IndexedInstrProf::Version; Header.MaxFunctionCount = MaxFunctionCount; Header.HashType = static_cast(IndexedInstrProf::HashType); Header.HashOffset = 0; @@ -227,6 +236,8 @@ } void InstrProfWriter::writeText(raw_fd_ostream &OS) { + if (ProfileKind == PF_IRLevel) + OS << "# IR level Instrumentation Flag:\n:" << 1 << "\n"; for (const auto &I : FunctionData) for (const auto &Func : I.getValue()) writeRecordInText(Func.second, OS); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -665,7 +665,22 @@ } } // end anonymous namespace +// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to let runtime +// aware this is an ir_level profile so it can set the version flag. +static void createIRLevelProfileFlagVariable(Module &M) { + Type *IntTy32 = Type::getInt32Ty(M.getContext()); + auto IRLevelFlagVariable = + new GlobalVariable(M, IntTy32, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy32, APInt(32, 1)), + IR_LEVEL_PROF_STRING(IR_LEVEL_PROF_VARNAME)); + IRLevelFlagVariable->setVisibility(GlobalValue::DefaultVisibility); + IRLevelFlagVariable->setComdat( + M.getOrInsertComdat(StringRef( + IR_LEVEL_PROF_STRING(IR_LEVEL_PROF_VARNAME)))); +} + bool PGOInstrumentationGen::runOnModule(Module &M) { + createIRLevelProfileFlagVariable(M); for (auto &F : M) { if (F.isDeclaration()) continue; @@ -703,6 +718,12 @@ "Cannot get PGOReader")); return false; } + if (!PGOReader->isIRLevelProfile()) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + ProfileFileName.data(), "Not an IR level instrumentation profile")); + return false; + } + for (auto &F : M) { if (F.isDeclaration()) Index: test/Transforms/PGOProfile/Inputs/branch1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch1.proftext +++ test/Transforms/PGOProfile/Inputs/branch1.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_br_1 25571299074 2 Index: test/Transforms/PGOProfile/Inputs/branch2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch2.proftext +++ test/Transforms/PGOProfile/Inputs/branch2.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_br_2 29667547796 2 Index: test/Transforms/PGOProfile/Inputs/criticaledge.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/criticaledge.proftext +++ test/Transforms/PGOProfile/Inputs/criticaledge.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_criticalEdge 82323253069 8 Index: test/Transforms/PGOProfile/Inputs/diag.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/diag.proftext +++ test/Transforms/PGOProfile/Inputs/diag.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 foo 12884999999 1 Index: test/Transforms/PGOProfile/Inputs/landingpad.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/landingpad.proftext +++ test/Transforms/PGOProfile/Inputs/landingpad.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 foo 59130013419 4 Index: test/Transforms/PGOProfile/Inputs/loop1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop1.proftext +++ test/Transforms/PGOProfile/Inputs/loop1.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_simple_for 34137660316 2 Index: test/Transforms/PGOProfile/Inputs/loop2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop2.proftext +++ test/Transforms/PGOProfile/Inputs/loop2.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_nested_for 53929068288 3 Index: test/Transforms/PGOProfile/Inputs/switch.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/switch.proftext +++ test/Transforms/PGOProfile/Inputs/switch.proftext @@ -1,3 +1,5 @@ +# :1 is the flat to indicate this is IR level profile. +:1 test_switch 46200943743 4 Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -127,6 +127,10 @@ exitWithErrorCode(ec, Input.Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRProfile = Reader->isIRLevelProfile(); + if (Writer.setIsIRLevelProfile(IsIRProfile)) + exitWithError("Merge IR generated profile with Clang generated profile."); + for (auto &I : *Reader) { if (std::error_code EC = Writer.addRecord(std::move(I), Input.Weight)) { // Only show hint the first time an error occurs. @@ -257,6 +261,7 @@ exitWithErrorCode(EC, Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRInstr = Reader->isIRLevelProfile(); uint64_t MaxFunctionCount = 0, MaxBlockCount = 0; size_t ShownFunctions = 0, TotalFunctions = 0; for (const auto &Func : *Reader) { @@ -273,8 +278,10 @@ ++TotalFunctions; assert(Func.Counts.size() > 0 && "function missing entry counter"); - if (Func.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Func.Counts[0]; + if (!IsIRInstr) { + if (Func.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Func.Counts[0]; + } for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { if (Func.Counts[I] > MaxBlockCount) @@ -290,8 +297,9 @@ OS << " " << Func.Name << ":\n" << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" - << " Counters: " << Func.Counts.size() << "\n" - << " Function count: " << Func.Counts[0] << "\n"; + << " Counters: " << Func.Counts.size() << "\n"; + if (!IsIRInstr) + OS << " Function count: " << Func.Counts[0] << "\n"; if (ShowIndirectCallTargets) OS << " Indirect Call Site Count: " @@ -299,8 +307,9 @@ if (ShowCounts) { OS << " Block counts: ["; - for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { - OS << (I == 1 ? "" : ", ") << Func.Counts[I]; + size_t Start = (IsIRInstr ? 0 : 1); + for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counts[I]; } OS << "]\n"; } @@ -330,8 +339,12 @@ if (ShowAllFunctions || !ShowFunction.empty()) OS << "Functions shown: " << ShownFunctions << "\n"; OS << "Total functions: " << TotalFunctions << "\n"; - OS << "Maximum function count: " << MaxFunctionCount << "\n"; - OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + if (!IsIRInstr) { + OS << "Maximum function count: " << MaxFunctionCount << "\n"; + OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + } else { + OS << "Maximum count: " << MaxBlockCount << "\n"; + } return 0; }