Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -678,6 +678,16 @@ /* Raw profile format version. */ #define INSTR_PROF_RAW_VERSION 2 +/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the + * version for other variants of profile. We set the lowest bit of the upper 8 + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * generated profile, and 0 if this is a Clang FE generated profile. + */ +#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define GET_VERSION(V) ((V)&~VARIANT_MASKS_ALL) +#define IR_LEVEL_PROF_VARNAME __llvm_profile_ir_level + /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -64,6 +64,7 @@ /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + virtual bool isIRLevelProfile() const = 0; protected: /// Set the current std::error_code and return same. @@ -105,6 +106,7 @@ std::unique_ptr DataBuffer; /// Iterator over the profile data. line_iterator Line; + bool IsIRLevelProfile; // String table for holding a unique copy of all the strings in the profile. InstrProfStringTable StringTable; @@ -115,13 +117,16 @@ public: TextInstrProfReader(std::unique_ptr DataBuffer_) - : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} + : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#'), + IsIRLevelProfile(false) {} /// Return true if the given buffer is in text instrprof format. static bool hasFormat(const MemoryBuffer &Buffer); + bool isIRLevelProfile() const override { return IsIRLevelProfile; }; + /// Read the header. - std::error_code readHeader() override { return success(); } + std::error_code readHeader() override; /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; }; @@ -139,6 +144,8 @@ /// The profile data file contents. std::unique_ptr DataBuffer; bool ShouldSwapBytes; + // Version of the raw profile. It contains the variant profile information. + uint64_t Version; uint64_t CountersDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; @@ -163,6 +170,9 @@ static bool hasFormat(const MemoryBuffer &DataBuffer); std::error_code readHeader() override; std::error_code readNextRecord(InstrProfRecord &Record) override; + bool isIRLevelProfile() const override { + return (Version & VARIANT_MASK_IR_PROF) != 0; + } private: std::error_code readNextHeader(const char *CurrentPos); @@ -277,6 +287,7 @@ virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; virtual ~InstrProfReaderIndexBase() {} virtual uint64_t getVersion() const = 0; + virtual bool isIRLevelProfile() const = 0; }; typedef OnDiskIterableChainedHashTable @@ -310,7 +321,10 @@ HashTable->getInfoObj().setValueProfDataEndianness(Endianness); } ~InstrProfReaderIndex() override {} - uint64_t getVersion() const override { return FormatVersion; } + uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } + bool isIRLevelProfile() const override { + return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; + } }; /// Reader for the indexed binary instrprof format. @@ -327,7 +341,9 @@ IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; public: + /// Return the profile version. uint64_t getVersion() const { return Index->getVersion(); } + bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } IndexedInstrProfReader(std::unique_ptr DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} Index: include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- include/llvm/ProfileData/InstrProfWriter.h +++ include/llvm/ProfileData/InstrProfWriter.h @@ -27,13 +27,15 @@ class InstrProfWriter { public: typedef SmallDenseMap ProfilingData; + enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; private: InstrProfStringTable StringTable; StringMap FunctionData; uint64_t MaxFunctionCount; + ProfKind ProfileKind; public: - InstrProfWriter() : MaxFunctionCount(0) {} + InstrProfWriter() : MaxFunctionCount(0), ProfileKind(PF_Unknown) {} /// Update string entries in profile data with references to StringTable. void updateStringTableReferences(InstrProfRecord &I); @@ -51,6 +53,16 @@ /// Write the profile, returning the raw data. For testing. std::unique_ptr writeBuffer(); + /// Set the ProfileKind. Report error if mixing FE and IR level profiles. + std::error_code setIsIRLevelProfile(bool IsIRLevel) { + if (ProfileKind == PF_Unknown) { + ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + return instrprof_error::success; + } + return (IsIRLevel == (ProfileKind == PF_IRLevel)) ? + instrprof_error::success : instrprof_error::unsupported_version; + } + // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -109,6 +109,32 @@ [](char c) { return ::isprint(c) || ::isspace(c); }); } +// Read the profile variant flag from the header: ":0" means this is a FE +// generated profile. ":1" means this is an IR level profile. Other strings +// with a leading ':' will be reported an error format. +std::error_code TextInstrProfReader::readHeader() { + bool IsIRInstr = false; + if (!Line->startswith(":")) { + IsIRLevelProfile = false; + return success(); + } + StringRef Str = (Line)->substr(1); + int32_t V = 0; + if (Str.getAsInteger(0, V)) + return instrprof_error::bad_header; + + if (V == 1) + IsIRInstr = true; + else if (V == 0) + IsIRInstr = false; + else + return instrprof_error::bad_header; + + ++Line; + IsIRLevelProfile = IsIRInstr; + return success(); +} + std::error_code TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { @@ -268,7 +294,8 @@ template std::error_code RawInstrProfReader::readHeader( const RawInstrProf::Header &Header) { - if (swap(Header.Version) != RawInstrProf::Version) + Version = swap(Header.Version); + if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); @@ -460,10 +487,10 @@ return data_type(); uint64_t Hash = endian::readNext(D); - // Initialize number of counters for FormatVersion == 1. + // Initialize number of counters for GET_VERSION(FormatVersion) == 1. uint64_t CountsSize = N / sizeof(uint64_t) - 1; // If format version is different then read the number of counters. - if (FormatVersion != 1) { + if (GET_VERSION(FormatVersion) != 1) { if (D + sizeof(uint64_t) > End) return data_type(); CountsSize = endian::readNext(D); @@ -480,7 +507,7 @@ DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. - if (FormatVersion > 2 && !readValueProfilingData(D, End)) { + if (GET_VERSION(FormatVersion) > 2 && !readValueProfilingData(D, End)) { DataBuffer.clear(); return data_type(); } @@ -567,7 +594,7 @@ // Read the version. uint64_t FormatVersion = endian::byte_swap(Header->Version); - if (FormatVersion > IndexedInstrProf::Version) + if (GET_VERSION(FormatVersion) > IndexedInstrProf::Version) return error(instrprof_error::unsupported_version); // Read the maximal function count. Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -130,8 +130,14 @@ // We keep track of the max function count as we go for simplicity. // Update this statistic no matter the result of the merge. - if (Dest.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Dest.Counts[0]; + if (ProfileKind != PF_IRLevel) { + if (Dest.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Dest.Counts[0]; + } else { + for (auto &IT : Dest.Counts) + if (IT > MaxFunctionCount) + MaxFunctionCount = IT; + } return Result; } @@ -149,7 +155,10 @@ // Write the header. IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; - Header.Version = IndexedInstrProf::Version; + if (ProfileKind == PF_IRLevel) + Header.Version = IndexedInstrProf::Version | VARIANT_MASK_IR_PROF; + else + Header.Version = IndexedInstrProf::Version; Header.MaxFunctionCount = MaxFunctionCount; Header.HashType = static_cast(IndexedInstrProf::HashType); Header.HashOffset = 0; @@ -227,6 +236,8 @@ } void InstrProfWriter::writeText(raw_fd_ostream &OS) { + if (ProfileKind == PF_IRLevel) + OS << "# IR level Instrumentation Flag:\n:" << 1 << "\n"; for (const auto &I : FunctionData) for (const auto &Func : I.getValue()) writeRecordInText(Func.second, OS); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -665,7 +665,21 @@ } } // end anonymous namespace +// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime +// aware this is an ir_level profile so it can set the version flag. +static void createIRLevelProfileFlagVariable(Module &M) { + Type *IntTy32 = Type::getInt32Ty(M.getContext()); + auto IRLevelFlagVariable = + new GlobalVariable(M, IntTy32, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy32, APInt(32, 1)), + INSTR_PROF_QUOTE(IR_LEVEL_PROF_VARNAME)); + IRLevelFlagVariable->setVisibility(GlobalValue::DefaultVisibility); + IRLevelFlagVariable->setComdat( + M.getOrInsertComdat(StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VARNAME)))); +} + bool PGOInstrumentationGen::runOnModule(Module &M) { + createIRLevelProfileFlagVariable(M); for (auto &F : M) { if (F.isDeclaration()) continue; @@ -703,6 +717,12 @@ "Cannot get PGOReader")); return false; } + if (!PGOReader->isIRLevelProfile()) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + ProfileFileName.data(), "Not an IR level instrumentation profile")); + return false; + } + for (auto &F : M) { if (F.isDeclaration()) Index: test/Transforms/PGOProfile/Inputs/branch1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch1.proftext +++ test/Transforms/PGOProfile/Inputs/branch1.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_br_1 25571299074 2 Index: test/Transforms/PGOProfile/Inputs/branch2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch2.proftext +++ test/Transforms/PGOProfile/Inputs/branch2.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_br_2 29667547796 2 Index: test/Transforms/PGOProfile/Inputs/criticaledge.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/criticaledge.proftext +++ test/Transforms/PGOProfile/Inputs/criticaledge.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_criticalEdge 82323253069 8 Index: test/Transforms/PGOProfile/Inputs/diag.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/diag.proftext +++ test/Transforms/PGOProfile/Inputs/diag.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 foo 12884999999 1 Index: test/Transforms/PGOProfile/Inputs/diag_FE.proftext =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/Inputs/diag_FE.proftext @@ -0,0 +1,5 @@ +foo +12884999999 +1 +1 + Index: test/Transforms/PGOProfile/Inputs/landingpad.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/landingpad.proftext +++ test/Transforms/PGOProfile/Inputs/landingpad.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 foo 59130013419 4 Index: test/Transforms/PGOProfile/Inputs/loop1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop1.proftext +++ test/Transforms/PGOProfile/Inputs/loop1.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_simple_for 34137660316 2 Index: test/Transforms/PGOProfile/Inputs/loop2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop2.proftext +++ test/Transforms/PGOProfile/Inputs/loop2.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_nested_for 53929068288 3 Index: test/Transforms/PGOProfile/Inputs/switch.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/switch.proftext +++ test/Transforms/PGOProfile/Inputs/switch.proftext @@ -1,3 +1,5 @@ +# :1 is the flag to indicate this is IR level profile. +:1 test_switch 46200943743 4 Index: test/Transforms/PGOProfile/branch1.ll =================================================================== --- test/Transforms/PGOProfile/branch1.ll +++ test/Transforms/PGOProfile/branch1.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_br_1 = private constant [9 x i8] c"test_br_1" define i32 @test_br_1(i32 %i) { Index: test/Transforms/PGOProfile/branch2.ll =================================================================== --- test/Transforms/PGOProfile/branch2.ll +++ test/Transforms/PGOProfile/branch2.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2" define i32 @test_br_2(i32 %i) { Index: test/Transforms/PGOProfile/criticaledge.ll =================================================================== --- test/Transforms/PGOProfile/criticaledge.ll +++ test/Transforms/PGOProfile/criticaledge.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge" ; GEN: @__profn__stdin__bar = private constant [11 x i8] c":bar" Index: test/Transforms/PGOProfile/diag_FE_profile.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/diag_FE_profile.ll @@ -0,0 +1,12 @@ +; RUN: llvm-profdata merge %S/Inputs/diag_FE.proftext -o %t.profdata +; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s + +; CHECK: Not an IR level instrumentation profile + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @foo() { +entry: + ret i32 0 +} Index: test/Transforms/PGOProfile/landingpad.ll =================================================================== --- test/Transforms/PGOProfile/landingpad.ll +++ test/Transforms/PGOProfile/landingpad.ll @@ -6,6 +6,8 @@ @val = global i32 0, align 4 @_ZTIi = external constant i8* +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_bar = private constant [3 x i8] c"bar" ; GEN: @__profn_foo = private constant [3 x i8] c"foo" Index: test/Transforms/PGOProfile/loop1.ll =================================================================== --- test/Transforms/PGOProfile/loop1.ll +++ test/Transforms/PGOProfile/loop1.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_simple_for = private constant [15 x i8] c"test_simple_for" define i32 @test_simple_for(i32 %n) { Index: test/Transforms/PGOProfile/loop2.ll =================================================================== --- test/Transforms/PGOProfile/loop2.ll +++ test/Transforms/PGOProfile/loop2.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_nested_for = private constant [15 x i8] c"test_nested_for" define i32 @test_nested_for(i32 %r, i32 %s) { Index: test/Transforms/PGOProfile/single_bb.ll =================================================================== --- test/Transforms/PGOProfile/single_bb.ll +++ test/Transforms/PGOProfile/single_bb.ll @@ -2,6 +2,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_single_bb = private constant [9 x i8] c"single_bb" define i32 @single_bb() { Index: test/Transforms/PGOProfile/switch.ll =================================================================== --- test/Transforms/PGOProfile/switch.ll +++ test/Transforms/PGOProfile/switch.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_ir_level = comdat any +; GEN: @__llvm_profile_ir_level = constant i32 1, comdat ; GEN: @__profn_test_switch = private constant [11 x i8] c"test_switch" define void @test_switch(i32 %i) { Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -127,6 +127,10 @@ exitWithErrorCode(ec, Input.Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRProfile = Reader->isIRLevelProfile(); + if (Writer.setIsIRLevelProfile(IsIRProfile)) + exitWithError("Merge IR generated profile with Clang generated profile."); + for (auto &I : *Reader) { if (std::error_code EC = Writer.addRecord(std::move(I), Input.Weight)) { // Only show hint the first time an error occurs. @@ -257,6 +261,7 @@ exitWithErrorCode(EC, Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRInstr = Reader->isIRLevelProfile(); uint64_t MaxFunctionCount = 0, MaxBlockCount = 0; size_t ShownFunctions = 0, TotalFunctions = 0; for (const auto &Func : *Reader) { @@ -273,8 +278,10 @@ ++TotalFunctions; assert(Func.Counts.size() > 0 && "function missing entry counter"); - if (Func.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Func.Counts[0]; + if (!IsIRInstr) { + if (Func.Counts[0] > MaxFunctionCount) + MaxFunctionCount = Func.Counts[0]; + } for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { if (Func.Counts[I] > MaxBlockCount) @@ -290,8 +297,9 @@ OS << " " << Func.Name << ":\n" << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" - << " Counters: " << Func.Counts.size() << "\n" - << " Function count: " << Func.Counts[0] << "\n"; + << " Counters: " << Func.Counts.size() << "\n"; + if (!IsIRInstr) + OS << " Function count: " << Func.Counts[0] << "\n"; if (ShowIndirectCallTargets) OS << " Indirect Call Site Count: " @@ -299,8 +307,9 @@ if (ShowCounts) { OS << " Block counts: ["; - for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { - OS << (I == 1 ? "" : ", ") << Func.Counts[I]; + size_t Start = (IsIRInstr ? 0 : 1); + for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counts[I]; } OS << "]\n"; } @@ -330,8 +339,12 @@ if (ShowAllFunctions || !ShowFunction.empty()) OS << "Functions shown: " << ShownFunctions << "\n"; OS << "Total functions: " << TotalFunctions << "\n"; - OS << "Maximum function count: " << MaxFunctionCount << "\n"; - OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + if (!IsIRInstr) { + OS << "Maximum function count: " << MaxFunctionCount << "\n"; + OS << "Maximum internal block count: " << MaxBlockCount << "\n"; + } else { + OS << "Maximum count: " << MaxBlockCount << "\n"; + } return 0; }