Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -705,9 +705,11 @@ * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. -*/ + */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define IR_LEVEL_PROF_VERSION_VAR __llvm_profile_raw_version /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -64,6 +64,7 @@ /// Iterator over profile data. InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + virtual bool isIRLevelProfile() const = 0; /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types @@ -118,6 +119,7 @@ std::unique_ptr DataBuffer; /// Iterator over the profile data. line_iterator Line; + bool IsIRLevelProfile; TextInstrProfReader(const TextInstrProfReader &) = delete; TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; @@ -125,11 +127,14 @@ public: TextInstrProfReader(std::unique_ptr DataBuffer_) - : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} + : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#'), + IsIRLevelProfile(false) {} /// Return true if the given buffer is in text instrprof format. static bool hasFormat(const MemoryBuffer &Buffer); + bool isIRLevelProfile() const override { return IsIRLevelProfile; } + /// Read the header. std::error_code readHeader() override; /// Read a single record. @@ -154,6 +159,10 @@ /// The profile data file contents. std::unique_ptr DataBuffer; bool ShouldSwapBytes; + // The value of the version field of the raw profile data header. The lower 56 + // bits specifies the format version and the most significant 8 bits specify + // the variant types of the profile. + uint64_t Version; uint64_t CountersDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; @@ -177,6 +186,9 @@ static bool hasFormat(const MemoryBuffer &DataBuffer); std::error_code readHeader() override; std::error_code readNextRecord(InstrProfRecord &Record) override; + bool isIRLevelProfile() const override { + return (Version & VARIANT_MASK_IR_PROF) != 0; + } InstrProfSymtab &getSymtab() override { assert(Symtab.get()); @@ -292,6 +304,7 @@ virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; virtual ~InstrProfReaderIndexBase() {} virtual uint64_t getVersion() const = 0; + virtual bool isIRLevelProfile() const = 0; virtual void populateSymtab(InstrProfSymtab &) = 0; }; @@ -323,7 +336,10 @@ HashTable->getInfoObj().setValueProfDataEndianness(Endianness); } ~InstrProfReaderIndex() override {} - uint64_t getVersion() const override { return FormatVersion; } + uint64_t getVersion() const override { return GET_VERSION(FormatVersion); } + bool isIRLevelProfile() const override { + return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; + } void populateSymtab(InstrProfSymtab &Symtab) override { Symtab.create(HashTable->keys()); } @@ -348,7 +364,9 @@ const unsigned char *Cur); public: + /// Return the profile version. uint64_t getVersion() const { return Index->getVersion(); } + bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } IndexedInstrProfReader(std::unique_ptr DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} Index: include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- include/llvm/ProfileData/InstrProfWriter.h +++ include/llvm/ProfileData/InstrProfWriter.h @@ -30,10 +30,12 @@ class InstrProfWriter { public: typedef SmallDenseMap ProfilingData; + enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; private: bool Sparse; StringMap FunctionData; + ProfKind ProfileKind; // Use raw pointer here for the incomplete type object. InstrProfRecordWriterTrait *InfoObj; @@ -55,6 +57,16 @@ /// Write the profile, returning the raw data. For testing. std::unique_ptr writeBuffer(); + /// Set the ProfileKind. Report error if mixing FE and IR level profiles. + std::error_code setIsIRLevelProfile(bool IsIRLevel) { + if (ProfileKind == PF_Unknown) { + ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + return instrprof_error::success; + } + return (IsIRLevel == (ProfileKind == PF_IRLevel)) ? + instrprof_error::success : instrprof_error::unsupported_version; + } + // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); void setOutputSparse(bool Sparse); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -109,8 +109,26 @@ [](char c) { return ::isprint(c) || ::isspace(c); }); } +// Read the profile variant flag from the header: ":FE" means this is a FE +// generated profile. ":IR" means this is an IR level profile. Other strings +// with a leading ':' will be reported an error format. std::error_code TextInstrProfReader::readHeader() { Symtab.reset(new InstrProfSymtab()); + bool IsIRInstr = false; + if (!Line->startswith(":")) { + IsIRLevelProfile = false; + return success(); + } + StringRef Str = (Line)->substr(1); + if (Str.equals_lower("ir")) + IsIRInstr = true; + else if (Str.equals_lower("fe")) + IsIRInstr = false; + else + return instrprof_error::bad_header; + + ++Line; + IsIRLevelProfile = IsIRInstr; return success(); } @@ -293,7 +311,8 @@ template std::error_code RawInstrProfReader::readHeader(const RawInstrProf::Header &Header) { - if (swap(Header.Version) != RawInstrProf::Version) + Version = swap(Header.Version); + if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); @@ -470,10 +489,10 @@ return data_type(); uint64_t Hash = endian::readNext(D); - // Initialize number of counters for FormatVersion == 1. + // Initialize number of counters for GET_VERSION(FormatVersion) == 1. uint64_t CountsSize = N / sizeof(uint64_t) - 1; // If format version is different then read the number of counters. - if (FormatVersion != IndexedInstrProf::ProfVersion::Version1) { + if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) { if (D + sizeof(uint64_t) > End) return data_type(); CountsSize = endian::readNext(D); @@ -490,7 +509,7 @@ DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. - if (FormatVersion > IndexedInstrProf::ProfVersion::Version2 && + if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && !readValueProfilingData(D, End)) { DataBuffer.clear(); return data_type(); @@ -603,7 +622,8 @@ // Read the version. uint64_t FormatVersion = endian::byte_swap(Header->Version); - if (FormatVersion > IndexedInstrProf::ProfVersion::CurrentVersion) + if (GET_VERSION(FormatVersion) > + IndexedInstrProf::ProfVersion::CurrentVersion) return error(instrprof_error::unsupported_version); Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur); Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -142,7 +142,7 @@ } InstrProfWriter::InstrProfWriter(bool Sparse) - : Sparse(Sparse), FunctionData(), + : Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown), InfoObj(new InstrProfRecordWriterTrait()) {} InstrProfWriter::~InstrProfWriter() { delete InfoObj; } @@ -230,6 +230,8 @@ IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion; + if (ProfileKind == PF_IRLevel) + Header.Version |= VARIANT_MASK_IR_PROF; Header.Unused = 0; Header.HashType = static_cast(IndexedInstrProf::HashType); Header.HashOffset = 0; @@ -336,6 +338,8 @@ } void InstrProfWriter::writeText(raw_fd_ostream &OS) { + if (ProfileKind == PF_IRLevel) + OS << "# IR level Instrumentation Flag\n:ir\n"; InstrProfSymtab Symtab; for (const auto &I : FunctionData) if (shouldEncodeData(I.getValue())) Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -49,6 +49,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Triple.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" @@ -713,7 +714,27 @@ } } // end anonymous namespace +// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime +// aware this is an ir_level profile so it can set the version flag. +static void createIRLevelProfileFlagVariable(Module &M) { + Type *IntTy64 = Type::getInt64Ty(M.getContext()); + uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); + auto IRLevelVersionVariable = + new GlobalVariable(M, IntTy64, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), + INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR)); + IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); + Triple TT(M.getTargetTriple()); + if (TT.isOSBinFormatMachO()) + IRLevelVersionVariable->setLinkage(GlobalValue::LinkOnceAnyLinkage); + else + IRLevelVersionVariable->setComdat( + M.getOrInsertComdat(StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR)))); +/* IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage); */ +} + bool PGOInstrumentationGen::runOnModule(Module &M) { + createIRLevelProfileFlagVariable(M); for (auto &F : M) { if (F.isDeclaration()) continue; @@ -751,6 +772,13 @@ "Cannot get PGOReader")); return false; } + // TODO: might need to change the warning once the clang option is finalized. + if (!PGOReader->isIRLevelProfile()) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + ProfileFileName.data(), "Not an IR level instrumentation profile")); + return false; + } + for (auto &F : M) { if (F.isDeclaration()) Index: test/Transforms/PGOProfile/Inputs/branch1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch1.proftext +++ test/Transforms/PGOProfile/Inputs/branch1.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_br_1 25571299074 2 Index: test/Transforms/PGOProfile/Inputs/branch2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/branch2.proftext +++ test/Transforms/PGOProfile/Inputs/branch2.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_br_2 29667547796 2 Index: test/Transforms/PGOProfile/Inputs/criticaledge.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/criticaledge.proftext +++ test/Transforms/PGOProfile/Inputs/criticaledge.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_criticalEdge 82323253069 8 Index: test/Transforms/PGOProfile/Inputs/diag.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/diag.proftext +++ test/Transforms/PGOProfile/Inputs/diag.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir foo 12884999999 1 Index: test/Transforms/PGOProfile/Inputs/diag_FE.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/diag_FE.proftext +++ test/Transforms/PGOProfile/Inputs/diag_FE.proftext @@ -0,0 +1,5 @@ +foo +12884999999 +1 +1 + Index: test/Transforms/PGOProfile/Inputs/landingpad.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/landingpad.proftext +++ test/Transforms/PGOProfile/Inputs/landingpad.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir foo 59130013419 4 Index: test/Transforms/PGOProfile/Inputs/loop1.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop1.proftext +++ test/Transforms/PGOProfile/Inputs/loop1.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_simple_for 34137660316 2 Index: test/Transforms/PGOProfile/Inputs/loop2.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/loop2.proftext +++ test/Transforms/PGOProfile/Inputs/loop2.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_nested_for 53929068288 3 Index: test/Transforms/PGOProfile/Inputs/switch.proftext =================================================================== --- test/Transforms/PGOProfile/Inputs/switch.proftext +++ test/Transforms/PGOProfile/Inputs/switch.proftext @@ -1,3 +1,5 @@ +# :ir is the flag to indicate this is IR level profile. +:ir test_switch 46200943743 4 Index: test/Transforms/PGOProfile/branch1.ll =================================================================== --- test/Transforms/PGOProfile/branch1.ll +++ test/Transforms/PGOProfile/branch1.ll @@ -1,9 +1,14 @@ -; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN +; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-COMDAT +; RUN: opt < %s -mtriple=x86_64-apple-darwin -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-DARWIN-LINKONCE ; RUN: llvm-profdata merge %S/Inputs/branch1.proftext -o %t.profdata ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN-DARWIN-LINKONCE: target triple = "x86_64-apple-darwin" +; GEN-COMDAT: $__llvm_profile_raw_version = comdat any +; GEN-COMDAT: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat +; GEN-DARWIN-LINKONCE: @__llvm_profile_raw_version = linkonce constant i64 72057594037927939 ; GEN: @__profn_test_br_1 = private constant [9 x i8] c"test_br_1" define i32 @test_br_1(i32 %i) { Index: test/Transforms/PGOProfile/branch2.ll =================================================================== --- test/Transforms/PGOProfile/branch2.ll +++ test/Transforms/PGOProfile/branch2.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2" define i32 @test_br_2(i32 %i) { Index: test/Transforms/PGOProfile/criticaledge.ll =================================================================== --- test/Transforms/PGOProfile/criticaledge.ll +++ test/Transforms/PGOProfile/criticaledge.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge" ; GEN: @__profn__stdin__bar = private constant [11 x i8] c":bar" Index: test/Transforms/PGOProfile/diag_FE_profile.ll =================================================================== --- test/Transforms/PGOProfile/diag_FE_profile.ll +++ test/Transforms/PGOProfile/diag_FE_profile.ll @@ -0,0 +1,12 @@ +; RUN: llvm-profdata merge %S/Inputs/diag_FE.proftext -o %t.profdata +; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s + +; CHECK: Not an IR level instrumentation profile + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @foo() { +entry: + ret i32 0 +} Index: test/Transforms/PGOProfile/landingpad.ll =================================================================== --- test/Transforms/PGOProfile/landingpad.ll +++ test/Transforms/PGOProfile/landingpad.ll @@ -6,6 +6,8 @@ @val = global i32 0, align 4 @_ZTIi = external constant i8* +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_bar = private constant [3 x i8] c"bar" ; GEN: @__profn_foo = private constant [3 x i8] c"foo" Index: test/Transforms/PGOProfile/loop1.ll =================================================================== --- test/Transforms/PGOProfile/loop1.ll +++ test/Transforms/PGOProfile/loop1.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_test_simple_for = private constant [15 x i8] c"test_simple_for" define i32 @test_simple_for(i32 %n) { Index: test/Transforms/PGOProfile/loop2.ll =================================================================== --- test/Transforms/PGOProfile/loop2.ll +++ test/Transforms/PGOProfile/loop2.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_test_nested_for = private constant [15 x i8] c"test_nested_for" define i32 @test_nested_for(i32 %r, i32 %s) { Index: test/Transforms/PGOProfile/single_bb.ll =================================================================== --- test/Transforms/PGOProfile/single_bb.ll +++ test/Transforms/PGOProfile/single_bb.ll @@ -2,6 +2,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_single_bb = private constant [9 x i8] c"single_bb" define i32 @single_bb() { Index: test/Transforms/PGOProfile/switch.ll =================================================================== --- test/Transforms/PGOProfile/switch.ll +++ test/Transforms/PGOProfile/switch.ll @@ -4,6 +4,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" +; GEN: $__llvm_profile_raw_version = comdat any +; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat ; GEN: @__profn_test_switch = private constant [11 x i8] c"test_switch" define void @test_switch(i32 %i) { Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -128,6 +128,10 @@ exitWithErrorCode(ec, Input.Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRProfile = Reader->isIRLevelProfile(); + if (Writer.setIsIRLevelProfile(IsIRProfile)) + exitWithError("Merge IR generated profile with Clang generated profile."); + for (auto &I : *Reader) { if (std::error_code EC = Writer.addRecord(std::move(I), Input.Weight)) { // Only show hint the first time an error occurs. @@ -269,6 +273,7 @@ exitWithErrorCode(EC, Filename); auto Reader = std::move(ReaderOrErr.get()); + bool IsIRInstr = Reader->isIRLevelProfile(); size_t ShownFunctions = 0; for (const auto &Func : *Reader) { bool Show = @@ -295,8 +300,9 @@ OS << " " << Func.Name << ":\n" << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" - << " Counters: " << Func.Counts.size() << "\n" - << " Function count: " << Func.Counts[0] << "\n"; + << " Counters: " << Func.Counts.size() << "\n"; + if (!IsIRInstr) + OS << " Function count: " << Func.Counts[0] << "\n"; if (ShowIndirectCallTargets) OS << " Indirect Call Site Count: " @@ -304,8 +310,9 @@ if (ShowCounts) { OS << " Block counts: ["; - for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) { - OS << (I == 1 ? "" : ", ") << Func.Counts[I]; + size_t Start = (IsIRInstr ? 0 : 1); + for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counts[I]; } OS << "]\n"; }