diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 5fb1d9486c16..734ab85ba05d 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1,922 +1,922 @@ //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains support for reading profiling data for clang's // instrumentation based PGO and coverage. // //===----------------------------------------------------------------------===// #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SymbolRemappingReader.h" #include "llvm/Support/SwapByteOrder.h" #include #include #include #include #include #include #include #include #include using namespace llvm; static Expected> setupMemoryBuffer(const Twine &Path) { ErrorOr> BufferOrErr = MemoryBuffer::getFileOrSTDIN(Path); if (std::error_code EC = BufferOrErr.getError()) return errorCodeToError(EC); return std::move(BufferOrErr.get()); } static Error initializeReader(InstrProfReader &Reader) { return Reader.readHeader(); } Expected> InstrProfReader::create(const Twine &Path) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); if (Error E = BufferOrError.takeError()) return std::move(E); return InstrProfReader::create(std::move(BufferOrError.get())); } Expected> InstrProfReader::create(std::unique_ptr Buffer) { // Sanity check the buffer. if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits::max()) return make_error(instrprof_error::too_large); if (Buffer->getBufferSize() == 0) return make_error(instrprof_error::empty_raw_profile); std::unique_ptr Result; // Create the reader. if (IndexedInstrProfReader::hasFormat(*Buffer)) Result.reset(new IndexedInstrProfReader(std::move(Buffer))); else if (RawInstrProfReader64::hasFormat(*Buffer)) Result.reset(new RawInstrProfReader64(std::move(Buffer))); else if (RawInstrProfReader32::hasFormat(*Buffer)) Result.reset(new RawInstrProfReader32(std::move(Buffer))); else if (TextInstrProfReader::hasFormat(*Buffer)) Result.reset(new TextInstrProfReader(std::move(Buffer))); else return make_error(instrprof_error::unrecognized_format); // Initialize the reader and return the result. if (Error E = initializeReader(*Result)) return std::move(E); return std::move(Result); } Expected> IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); if (Error E = BufferOrError.takeError()) return std::move(E); // Set up the remapping buffer if requested. std::unique_ptr RemappingBuffer; std::string RemappingPathStr = RemappingPath.str(); if (!RemappingPathStr.empty()) { auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr); if (Error E = RemappingBufferOrError.takeError()) return std::move(E); RemappingBuffer = std::move(RemappingBufferOrError.get()); } return IndexedInstrProfReader::create(std::move(BufferOrError.get()), std::move(RemappingBuffer)); } Expected> IndexedInstrProfReader::create(std::unique_ptr Buffer, std::unique_ptr RemappingBuffer) { // Sanity check the buffer. if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits::max()) return make_error(instrprof_error::too_large); // Create the reader. if (!IndexedInstrProfReader::hasFormat(*Buffer)) return make_error(instrprof_error::bad_magic); auto Result = std::make_unique( std::move(Buffer), std::move(RemappingBuffer)); // Initialize the reader and return the result. if (Error E = initializeReader(*Result)) return std::move(E); return std::move(Result); } void InstrProfIterator::Increment() { if (auto E = Reader->readNextRecord(Record)) { // Handle errors in the reader. InstrProfError::take(std::move(E)); *this = InstrProfIterator(); } } bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { // Verify that this really looks like plain ASCII text by checking a // 'reasonable' number of characters (up to profile magic size). size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t)); StringRef buffer = Buffer.getBufferStart(); return count == 0 || std::all_of(buffer.begin(), buffer.begin() + count, [](char c) { return isPrint(c) || ::isspace(c); }); } // Read the profile variant flag from the header: ":FE" means this is a FE // generated profile. ":IR" means this is an IR level profile. Other strings // with a leading ':' will be reported an error format. Error TextInstrProfReader::readHeader() { Symtab.reset(new InstrProfSymtab()); bool IsIRInstr = false; if (!Line->startswith(":")) { IsIRLevelProfile = false; return success(); } StringRef Str = (Line)->substr(1); if (Str.equals_lower("ir")) IsIRInstr = true; else if (Str.equals_lower("fe")) IsIRInstr = false; else if (Str.equals_lower("csir")) { IsIRInstr = true; HasCSIRLevelProfile = true; } else return error(instrprof_error::bad_header); ++Line; IsIRLevelProfile = IsIRInstr; return success(); } Error TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { #define CHECK_LINE_END(Line) \ if (Line.is_at_end()) \ return error(instrprof_error::truncated); #define READ_NUM(Str, Dst) \ if ((Str).getAsInteger(10, (Dst))) \ return error(instrprof_error::malformed); #define VP_READ_ADVANCE(Val) \ CHECK_LINE_END(Line); \ uint32_t Val; \ READ_NUM((*Line), (Val)); \ Line++; if (Line.is_at_end()) return success(); uint32_t NumValueKinds; if (Line->getAsInteger(10, NumValueKinds)) { // No value profile data return success(); } if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1) return error(instrprof_error::malformed); Line++; for (uint32_t VK = 0; VK < NumValueKinds; VK++) { VP_READ_ADVANCE(ValueKind); if (ValueKind > IPVK_Last) return error(instrprof_error::malformed); VP_READ_ADVANCE(NumValueSites); if (!NumValueSites) continue; Record.reserveSites(VK, NumValueSites); for (uint32_t S = 0; S < NumValueSites; S++) { VP_READ_ADVANCE(NumValueData); std::vector CurrentValues; for (uint32_t V = 0; V < NumValueData; V++) { CHECK_LINE_END(Line); std::pair VD = Line->rsplit(':'); uint64_t TakenCount, Value; if (ValueKind == IPVK_IndirectCallTarget) { if (InstrProfSymtab::isExternalSymbol(VD.first)) { Value = 0; } else { if (Error E = Symtab->addFuncName(VD.first)) return E; Value = IndexedInstrProf::ComputeHash(VD.first); } } else { READ_NUM(VD.first, Value); } READ_NUM(VD.second, TakenCount); CurrentValues.push_back({Value, TakenCount}); Line++; } Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData, nullptr); } } return success(); #undef CHECK_LINE_END #undef READ_NUM #undef VP_READ_ADVANCE } Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { // Skip empty lines and comments. while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) ++Line; // If we hit EOF while looking for a name, we're done. if (Line.is_at_end()) { return error(instrprof_error::eof); } // Read the function name. Record.Name = *Line++; if (Error E = Symtab->addFuncName(Record.Name)) return error(std::move(E)); // Read the function hash. if (Line.is_at_end()) return error(instrprof_error::truncated); if ((Line++)->getAsInteger(0, Record.Hash)) return error(instrprof_error::malformed); // Read the number of counters. uint64_t NumCounters; if (Line.is_at_end()) return error(instrprof_error::truncated); if ((Line++)->getAsInteger(10, NumCounters)) return error(instrprof_error::malformed); if (NumCounters == 0) return error(instrprof_error::malformed); // Read each counter and fill our internal storage with the values. Record.Clear(); Record.Counts.reserve(NumCounters); for (uint64_t I = 0; I < NumCounters; ++I) { if (Line.is_at_end()) return error(instrprof_error::truncated); uint64_t Count; if ((Line++)->getAsInteger(10, Count)) return error(instrprof_error::malformed); Record.Counts.push_back(Count); } // Check if value profile data exists and read it if so. if (Error E = readValueProfileData(Record)) return error(std::move(E)); return success(); } template bool RawInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { if (DataBuffer.getBufferSize() < sizeof(uint64_t)) return false; uint64_t Magic = *reinterpret_cast(DataBuffer.getBufferStart()); return RawInstrProf::getMagic() == Magic || sys::getSwappedBytes(RawInstrProf::getMagic()) == Magic; } template Error RawInstrProfReader::readHeader() { if (!hasFormat(*DataBuffer)) return error(instrprof_error::bad_magic); if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header)) return error(instrprof_error::bad_header); auto *Header = reinterpret_cast( DataBuffer->getBufferStart()); ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic(); return readHeader(*Header); } template Error RawInstrProfReader::readNextHeader(const char *CurrentPos) { const char *End = DataBuffer->getBufferEnd(); // Skip zero padding between profiles. while (CurrentPos != End && *CurrentPos == 0) ++CurrentPos; // If there's nothing left, we're done. if (CurrentPos == End) return make_error(instrprof_error::eof); // If there isn't enough space for another header, this is probably just // garbage at the end of the file. if (CurrentPos + sizeof(RawInstrProf::Header) > End) return make_error(instrprof_error::malformed); // The writer ensures each profile is padded to start at an aligned address. if (reinterpret_cast(CurrentPos) % alignof(uint64_t)) return make_error(instrprof_error::malformed); // The magic should have the same byte order as in the previous header. uint64_t Magic = *reinterpret_cast(CurrentPos); if (Magic != swap(RawInstrProf::getMagic())) return make_error(instrprof_error::bad_magic); // There's another profile to read, so we need to process the header. auto *Header = reinterpret_cast(CurrentPos); return readHeader(*Header); } template Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { if (Error E = Symtab.create(StringRef(NamesStart, NamesSize))) return error(std::move(E)); for (const RawInstrProf::ProfileData *I = Data; I != DataEnd; ++I) { const IntPtrT FPtr = swap(I->FunctionPointer); if (!FPtr) continue; Symtab.mapAddress(FPtr, I->NameRef); } return success(); } template Error RawInstrProfReader::readHeader( const RawInstrProf::Header &Header) { Version = swap(Header.Version); if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); auto DataSize = swap(Header.DataSize); auto CountersSize = swap(Header.CountersSize); NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData); auto PaddingSize = getNumPaddingBytes(NamesSize); ptrdiff_t DataOffset = sizeof(RawInstrProf::Header); ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes; ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); if (Start + ValueDataOffset > DataBuffer->getBufferEnd()) return error(instrprof_error::bad_header); Data = reinterpret_cast *>( Start + DataOffset); DataEnd = Data + DataSize; CountersStart = reinterpret_cast(Start + CountersOffset); NamesStart = Start + NamesOffset; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); std::unique_ptr NewSymtab = std::make_unique(); if (Error E = createSymtab(*NewSymtab.get())) return E; Symtab = std::move(NewSymtab); return success(); } template Error RawInstrProfReader::readName(NamedInstrProfRecord &Record) { Record.Name = getName(Data->NameRef); return success(); } template Error RawInstrProfReader::readFuncHash(NamedInstrProfRecord &Record) { Record.Hash = swap(Data->FuncHash); return success(); } template Error RawInstrProfReader::readRawCounts( InstrProfRecord &Record) { uint32_t NumCounters = swap(Data->NumCounters); IntPtrT CounterPtr = Data->CounterPtr; if (NumCounters == 0) return error(instrprof_error::malformed); auto *NamesStartAsCounter = reinterpret_cast(NamesStart); ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart; // Check bounds. Note that the counter pointer embedded in the data record // may itself be corrupt. if (NumCounters > MaxNumCounters) return error(instrprof_error::malformed); ptrdiff_t CounterOffset = getCounterOffset(CounterPtr); if (CounterOffset < 0 || CounterOffset > MaxNumCounters || (CounterOffset + NumCounters) > MaxNumCounters) return error(instrprof_error::malformed); auto RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters); if (ShouldSwapBytes) { Record.Counts.clear(); Record.Counts.reserve(RawCounts.size()); for (uint64_t Count : RawCounts) Record.Counts.push_back(swap(Count)); } else Record.Counts = RawCounts; return success(); } template Error RawInstrProfReader::readValueProfilingData( InstrProfRecord &Record) { Record.clearValueData(); CurValueDataSize = 0; // Need to match the logic in value profile dumper code in compiler-rt: uint32_t NumValueKinds = 0; for (uint32_t I = 0; I < IPVK_Last + 1; I++) NumValueKinds += (Data->NumValueSites[I] != 0); if (!NumValueKinds) return success(); Expected> VDataPtrOrErr = ValueProfData::getValueProfData( ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(), getDataEndianness()); if (Error E = VDataPtrOrErr.takeError()) return E; // Note that besides deserialization, this also performs the conversion for // indirect call targets. The function pointers from the raw profile are // remapped into function name hashes. VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get()); CurValueDataSize = VDataPtrOrErr.get()->getSize(); return success(); } template Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { if (atEnd()) // At this point, ValueDataStart field points to the next header. if (Error E = readNextHeader(getNextHeaderPos())) return error(std::move(E)); // Read name ad set it in Record. if (Error E = readName(Record)) return error(std::move(E)); // Read FuncHash and set it in Record. if (Error E = readFuncHash(Record)) return error(std::move(E)); // Read raw counts and set Record. if (Error E = readRawCounts(Record)) return error(std::move(E)); // Read value data and set Record. if (Error E = readValueProfilingData(Record)) return error(std::move(E)); // Iterate. advanceData(); return success(); } namespace llvm { template class RawInstrProfReader; template class RawInstrProfReader; } // end namespace llvm InstrProfLookupTrait::hash_value_type InstrProfLookupTrait::ComputeHash(StringRef K) { return IndexedInstrProf::ComputeHash(HashType, K); } using data_type = InstrProfLookupTrait::data_type; using offset_type = InstrProfLookupTrait::offset_type; bool InstrProfLookupTrait::readValueProfilingData( const unsigned char *&D, const unsigned char *const End) { Expected> VDataPtrOrErr = ValueProfData::getValueProfData(D, End, ValueProfDataEndianness); if (VDataPtrOrErr.takeError()) return false; VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr); D += VDataPtrOrErr.get()->TotalSize; return true; } data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, offset_type N) { using namespace support; // Check if the data is corrupt. If so, don't try to read it. if (N % sizeof(uint64_t)) return data_type(); DataBuffer.clear(); std::vector CounterBuffer; const unsigned char *End = D + N; while (D < End) { // Read hash. if (D + sizeof(uint64_t) >= End) return data_type(); uint64_t Hash = endian::readNext(D); // Initialize number of counters for GET_VERSION(FormatVersion) == 1. uint64_t CountsSize = N / sizeof(uint64_t) - 1; // If format version is different then read the number of counters. if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) { if (D + sizeof(uint64_t) > End) return data_type(); CountsSize = endian::readNext(D); } // Read counter values. if (D + CountsSize * sizeof(uint64_t) > End) return data_type(); CounterBuffer.clear(); CounterBuffer.reserve(CountsSize); for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && !readValueProfilingData(D, End)) { DataBuffer.clear(); return data_type(); } } return DataBuffer; } template Error InstrProfReaderIndex::getRecords( StringRef FuncName, ArrayRef &Data) { auto Iter = HashTable->find(FuncName); if (Iter == HashTable->end()) return make_error(instrprof_error::unknown_function); Data = (*Iter); if (Data.empty()) return make_error(instrprof_error::malformed); return Error::success(); } template Error InstrProfReaderIndex::getRecords( ArrayRef &Data) { if (atEnd()) return make_error(instrprof_error::eof); Data = *RecordIterator; if (Data.empty()) return make_error(instrprof_error::malformed); return Error::success(); } template InstrProfReaderIndex::InstrProfReaderIndex( const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version) { FormatVersion = Version; HashTable.reset(HashTableImpl::Create( Buckets, Payload, Base, typename HashTableImpl::InfoType(HashType, Version))); RecordIterator = HashTable->data_begin(); } namespace { /// A remapper that does not apply any remappings. class InstrProfReaderNullRemapper : public InstrProfReaderRemapper { InstrProfReaderIndexBase &Underlying; public: InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying) : Underlying(Underlying) {} Error getRecords(StringRef FuncName, ArrayRef &Data) override { return Underlying.getRecords(FuncName, Data); } }; } /// A remapper that applies remappings based on a symbol remapping file. template class llvm::InstrProfReaderItaniumRemapper : public InstrProfReaderRemapper { public: InstrProfReaderItaniumRemapper( std::unique_ptr RemapBuffer, InstrProfReaderIndex &Underlying) : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) { } /// Extract the original function name from a PGO function name. static StringRef extractName(StringRef Name) { // We can have multiple :-separated pieces; there can be pieces both // before and after the mangled name. Find the first part that starts // with '_Z'; we'll assume that's the mangled name we want. std::pair Parts = {StringRef(), Name}; while (true) { Parts = Parts.second.split(':'); if (Parts.first.startswith("_Z")) return Parts.first; if (Parts.second.empty()) return Name; } } /// Given a mangled name extracted from a PGO function name, and a new /// form for that mangled name, reconstitute the name. static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl &Out) { Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size()); Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin()); Out.insert(Out.end(), Replacement.begin(), Replacement.end()); Out.insert(Out.end(), ExtractedName.end(), OrigName.end()); } Error populateRemappings() override { if (Error E = Remappings.read(*RemapBuffer)) return E; for (StringRef Name : Underlying.HashTable->keys()) { StringRef RealName = extractName(Name); if (auto Key = Remappings.insert(RealName)) { // FIXME: We could theoretically map the same equivalence class to // multiple names in the profile data. If that happens, we should // return NamedInstrProfRecords from all of them. MappedNames.insert({Key, RealName}); } } return Error::success(); } Error getRecords(StringRef FuncName, ArrayRef &Data) override { StringRef RealName = extractName(FuncName); if (auto Key = Remappings.lookup(RealName)) { StringRef Remapped = MappedNames.lookup(Key); if (!Remapped.empty()) { if (RealName.begin() == FuncName.begin() && RealName.end() == FuncName.end()) FuncName = Remapped; else { // Try rebuilding the name from the given remapping. SmallString<256> Reconstituted; reconstituteName(FuncName, RealName, Remapped, Reconstituted); Error E = Underlying.getRecords(Reconstituted, Data); if (!E) return E; // If we failed because the name doesn't exist, fall back to asking // about the original name. if (Error Unhandled = handleErrors( std::move(E), [](std::unique_ptr Err) { return Err->get() == instrprof_error::unknown_function ? Error::success() : Error(std::move(Err)); })) return Unhandled; } } } return Underlying.getRecords(FuncName, Data); } private: /// The memory buffer containing the remapping configuration. Remappings /// holds pointers into this buffer. std::unique_ptr RemapBuffer; /// The mangling remapper. SymbolRemappingReader Remappings; /// Mapping from mangled name keys to the name used for the key in the /// profile data. /// FIXME: Can we store a location within the on-disk hash table instead of /// redoing lookup? DenseMap MappedNames; /// The real profile data reader. InstrProfReaderIndex &Underlying; }; bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { using namespace support; if (DataBuffer.getBufferSize() < 8) return false; uint64_t Magic = endian::read(DataBuffer.getBufferStart()); // Verify that it's magical. return Magic == IndexedInstrProf::Magic; } const unsigned char * IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, const unsigned char *Cur, bool UseCS) { using namespace IndexedInstrProf; using namespace support; if (Version >= IndexedInstrProf::Version4) { const IndexedInstrProf::Summary *SummaryInLE = reinterpret_cast(Cur); uint64_t NFields = endian::byte_swap(SummaryInLE->NumSummaryFields); uint64_t NEntries = endian::byte_swap(SummaryInLE->NumCutoffEntries); uint32_t SummarySize = IndexedInstrProf::Summary::getSize(NFields, NEntries); std::unique_ptr SummaryData = IndexedInstrProf::allocSummary(SummarySize); const uint64_t *Src = reinterpret_cast(SummaryInLE); uint64_t *Dst = reinterpret_cast(SummaryData.get()); for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) Dst[I] = endian::byte_swap(Src[I]); SummaryEntryVector DetailedSummary; for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) { const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I); DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount, Ent.NumBlocks); } std::unique_ptr &Summary = UseCS ? this->CS_Summary : this->Summary; // initialize InstrProfSummary using the SummaryData from disk. Summary = std::make_unique( UseCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr, DetailedSummary, SummaryData->get(Summary::TotalBlockCount), SummaryData->get(Summary::MaxBlockCount), SummaryData->get(Summary::MaxInternalBlockCount), SummaryData->get(Summary::MaxFunctionCount), SummaryData->get(Summary::TotalNumBlocks), SummaryData->get(Summary::TotalNumFunctions)); return Cur + SummarySize; } else { - // For older version of profile data, we need to compute on the fly: - using namespace IndexedInstrProf; - + // The older versions do not support a profile summary. This just computes + // an empty summary, which will not result in accurate hot/cold detection. + // We would need to call addRecord for all NamedInstrProfRecords to get the + // correct summary. However, this version is old (prior to early 2016) and + // has not been supporting an accurate summary for several years. InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); - // FIXME: This only computes an empty summary. Need to call addRecord for - // all NamedInstrProfRecords to get the correct summary. - this->Summary = Builder.getSummary(); + Summary = Builder.getSummary(); return Cur; } } Error IndexedInstrProfReader::readHeader() { using namespace support; const unsigned char *Start = (const unsigned char *)DataBuffer->getBufferStart(); const unsigned char *Cur = Start; if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24) return error(instrprof_error::truncated); auto *Header = reinterpret_cast(Cur); Cur += sizeof(IndexedInstrProf::Header); // Check the magic number. uint64_t Magic = endian::byte_swap(Header->Magic); if (Magic != IndexedInstrProf::Magic) return error(instrprof_error::bad_magic); // Read the version. uint64_t FormatVersion = endian::byte_swap(Header->Version); if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::CurrentVersion) return error(instrprof_error::unsupported_version); Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur, /* UseCS */ false); if (FormatVersion & VARIANT_MASK_CSIR_PROF) Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur, /* UseCS */ true); // Read the hash type and start offset. IndexedInstrProf::HashT HashType = static_cast( endian::byte_swap(Header->HashType)); if (HashType > IndexedInstrProf::HashT::Last) return error(instrprof_error::unsupported_hash_type); uint64_t HashOffset = endian::byte_swap(Header->HashOffset); // The rest of the file is an on disk hash table. auto IndexPtr = std::make_unique>( Start + HashOffset, Cur, Start, HashType, FormatVersion); // Load the remapping table now if requested. if (RemappingBuffer) { Remapper = std::make_unique< InstrProfReaderItaniumRemapper>( std::move(RemappingBuffer), *IndexPtr); if (Error E = Remapper->populateRemappings()) return E; } else { Remapper = std::make_unique(*IndexPtr); } Index = std::move(IndexPtr); return success(); } InstrProfSymtab &IndexedInstrProfReader::getSymtab() { if (Symtab.get()) return *Symtab.get(); std::unique_ptr NewSymtab = std::make_unique(); if (Error E = Index->populateSymtab(*NewSymtab.get())) { consumeError(error(InstrProfError::take(std::move(E)))); } Symtab = std::move(NewSymtab); return *Symtab.get(); } Expected IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, uint64_t FuncHash) { ArrayRef Data; Error Err = Remapper->getRecords(FuncName, Data); if (Err) return std::move(Err); // Found it. Look for counters with the right hash. for (unsigned I = 0, E = Data.size(); I < E; ++I) { // Check for a match and fill the vector if there is one. if (Data[I].Hash == FuncHash) { return std::move(Data[I]); } } return error(instrprof_error::hash_mismatch); } Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts) { Expected Record = getInstrProfRecord(FuncName, FuncHash); if (Error E = Record.takeError()) return error(std::move(E)); Counts = Record.get().Counts; return success(); } Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { ArrayRef Data; Error E = Index->getRecords(Data); if (E) return error(std::move(E)); Record = Data[RecordIndex++]; if (RecordIndex >= Data.size()) { Index->advanceToNextKey(); RecordIndex = 0; } return success(); } void InstrProfReader::accumuateCounts(CountSumOrPercent &Sum, bool IsCS) { uint64_t NumFuncs = 0; for (const auto &Func : *this) { if (isIRLevelProfile()) { bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash); if (FuncIsCS != IsCS) continue; } Func.accumuateCounts(Sum); ++NumFuncs; } Sum.NumEntries = NumFuncs; } diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index c8cf1805c66d..e776d59cccb5 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1,1884 +1,1887 @@ //===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements PGO instrumentation using a minimum spanning tree based // on the following paper: // [1] Donald E. Knuth, Francis R. Stevenson. Optimal measurement of points // for program frequency counts. BIT Numerical Mathematics 1973, Volume 13, // Issue 3, pp 313-322 // The idea of the algorithm based on the fact that for each node (except for // the entry and exit), the sum of incoming edge counts equals the sum of // outgoing edge counts. The count of edge on spanning tree can be derived from // those edges not on the spanning tree. Knuth proves this method instruments // the minimum number of edges. // // The minimal spanning tree here is actually a maximum weight tree -- on-tree // edges have higher frequencies (more likely to execute). The idea is to // instrument those less frequently executed edges to reduce the runtime // overhead of instrumented binaries. // // This file contains two passes: // (1) Pass PGOInstrumentationGen which instruments the IR to generate edge // count profile, and generates the instrumentation for indirect call // profiling. // (2) Pass PGOInstrumentationUse which reads the edge count profile and // annotates the branch weights. It also reads the indirect call value // profiling records and annotate the indirect call instructions. // // To get the precise counter information, These two passes need to invoke at // the same compilation point (so they see the same IR). For pass // PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For // pass PGOInstrumentationUse, the real work in done in class PGOUseFunc and // the profile is opened in module level and passed to each PGOUseFunc instance. // The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put // in class FuncPGOInstrumentation. // // Class PGOEdge represents a CFG edge and some auxiliary information. Class // BBInfo contains auxiliary information for each BB. These two classes are used // in pass PGOInstrumentationGen. Class PGOUseEdge and UseBBInfo are the derived // class of PGOEdge and BBInfo, respectively. They contains extra data structure // used in populating profile counters. // The MST implementation is in Class CFGMST (CFGMST.h). // //===----------------------------------------------------------------------===// #include "CFGMST.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/IndirectCallVisitor.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/JamCRC.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/MisExpect.h" #include #include #include #include #include #include #include #include #include using namespace llvm; using ProfileCount = Function::ProfileCount; #define DEBUG_TYPE "pgo-instrumentation" STATISTIC(NumOfPGOInstrument, "Number of edges instrumented."); STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented."); STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented."); STATISTIC(NumOfPGOEdge, "Number of edges."); STATISTIC(NumOfPGOBB, "Number of basic-blocks."); STATISTIC(NumOfPGOSplit, "Number of critical edge splits."); STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts."); STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO."); STATISTIC(NumOfCSPGOSelectInsts, "Number of select instruction instrumented in CSPGO."); STATISTIC(NumOfCSPGOMemIntrinsics, "Number of mem intrinsics instrumented in CSPGO."); STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO."); STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO."); STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO."); STATISTIC(NumOfCSPGOFunc, "Number of functions having valid profile counts in CSPGO."); STATISTIC(NumOfCSPGOMismatch, "Number of functions having mismatch profile in CSPGO."); STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO."); // Command line option to specify the file to read profile from. This is // mainly used for testing. static cl::opt PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose.")); static cl::opt PGOTestProfileRemappingFile( "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose.")); // Command line option to disable value profiling. The default is false: // i.e. value profiling is enabled by default. This is for debug purpose. static cl::opt DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling")); // Command line option to set the maximum number of VP annotations to write to // the metadata for a single indirect call callsite. static cl::opt MaxNumAnnotations( "icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite")); // Command line option to set the maximum number of value annotations // to write to the metadata for a single memop intrinsic. static cl::opt MaxNumMemOPAnnotations( "memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic")); // Command line option to control appending FunctionHash to the name of a COMDAT // function. This is to avoid the hash mismatch caused by the preinliner. static cl::opt DoComdatRenaming( "do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner")); // Command line option to enable/disable the warning about missing profile // information. static cl::opt PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions.")); // Command line option to enable/disable the warning about a hash mismatch in // the profile data. static cl::opt NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch.")); // Command line option to enable/disable the warning about a hash mismatch in // the profile data for Comdat functions, which often turns out to be false // positive due to the pre-instrumentation inline. static cl::opt NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions.")); // Command line option to enable/disable select instruction instrumentation. static cl::opt PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. ")); // Command line option to turn on CFG dot or text dump of raw profile counts static cl::opt PGOViewRawCounts( "pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text."))); // Command line option to enable/disable memop intrinsic call.size profiling. static cl::opt PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling.")); // Emit branch probability as optimization remarks. static cl::opt EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation")); // Command line option to turn on CFG dot dump after profile annotation. // Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts extern cl::opt PGOViewCounts; // Command line option to specify the name of the function for CFG dump // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; // Return a string describing the branch condition that can be // used in static branch probability heuristics: static std::string getBranchCondString(Instruction *TI) { BranchInst *BI = dyn_cast(TI); if (!BI || !BI->isConditional()) return std::string(); Value *Cond = BI->getCondition(); ICmpInst *CI = dyn_cast(Cond); if (!CI) return std::string(); std::string result; raw_string_ostream OS(result); OS << CmpInst::getPredicateName(CI->getPredicate()) << "_"; CI->getOperand(0)->getType()->print(OS, true); Value *RHS = CI->getOperand(1); ConstantInt *CV = dyn_cast(RHS); if (CV) { if (CV->isZero()) OS << "_Zero"; else if (CV->isOne()) OS << "_One"; else if (CV->isMinusOne()) OS << "_MinusOne"; else OS << "_Const"; } OS.flush(); return result; } namespace { /// The select instruction visitor plays three roles specified /// by the mode. In \c VM_counting mode, it simply counts the number of /// select instructions. In \c VM_instrument mode, it inserts code to count /// the number times TrueValue of select is taken. In \c VM_annotate mode, /// it reads the profile data and annotate the select instruction with metadata. enum VisitMode { VM_counting, VM_instrument, VM_annotate }; class PGOUseFunc; /// Instruction Visitor class to visit select instructions. struct SelectInstVisitor : public InstVisitor { Function &F; unsigned NSIs = 0; // Number of select instructions instrumented. VisitMode Mode = VM_counting; // Visiting mode. unsigned *CurCtrIdx = nullptr; // Pointer to current counter index. unsigned TotalNumCtrs = 0; // Total number of counters GlobalVariable *FuncNameVar = nullptr; uint64_t FuncHash = 0; PGOUseFunc *UseFunc = nullptr; SelectInstVisitor(Function &Func) : F(Func) {} void countSelects(Function &Func) { NSIs = 0; Mode = VM_counting; visit(Func); } // Visit the IR stream and instrument all select instructions. \p // Ind is a pointer to the counter index variable; \p TotalNC // is the total number of counters; \p FNV is the pointer to the // PGO function name var; \p FHash is the function hash. void instrumentSelects(Function &Func, unsigned *Ind, unsigned TotalNC, GlobalVariable *FNV, uint64_t FHash) { Mode = VM_instrument; CurCtrIdx = Ind; TotalNumCtrs = TotalNC; FuncHash = FHash; FuncNameVar = FNV; visit(Func); } // Visit the IR stream and annotate all select instructions. void annotateSelects(Function &Func, PGOUseFunc *UF, unsigned *Ind) { Mode = VM_annotate; UseFunc = UF; CurCtrIdx = Ind; visit(Func); } void instrumentOneSelectInst(SelectInst &SI); void annotateOneSelectInst(SelectInst &SI); // Visit \p SI instruction and perform tasks according to visit mode. void visitSelectInst(SelectInst &SI); // Return the number of select instructions. This needs be called after // countSelects(). unsigned getNumOfSelectInsts() const { return NSIs; } }; /// Instruction Visitor class to visit memory intrinsic calls. struct MemIntrinsicVisitor : public InstVisitor { Function &F; unsigned NMemIs = 0; // Number of memIntrinsics instrumented. VisitMode Mode = VM_counting; // Visiting mode. unsigned CurCtrId = 0; // Current counter index. unsigned TotalNumCtrs = 0; // Total number of counters GlobalVariable *FuncNameVar = nullptr; uint64_t FuncHash = 0; PGOUseFunc *UseFunc = nullptr; std::vector Candidates; MemIntrinsicVisitor(Function &Func) : F(Func) {} void countMemIntrinsics(Function &Func) { NMemIs = 0; Mode = VM_counting; visit(Func); } void instrumentMemIntrinsics(Function &Func, unsigned TotalNC, GlobalVariable *FNV, uint64_t FHash) { Mode = VM_instrument; TotalNumCtrs = TotalNC; FuncHash = FHash; FuncNameVar = FNV; visit(Func); } std::vector findMemIntrinsics(Function &Func) { Candidates.clear(); Mode = VM_annotate; visit(Func); return Candidates; } // Visit the IR stream and annotate all mem intrinsic call instructions. void instrumentOneMemIntrinsic(MemIntrinsic &MI); // Visit \p MI instruction and perform tasks according to visit mode. void visitMemIntrinsic(MemIntrinsic &SI); unsigned getNumOfMemIntrinsics() const { return NMemIs; } }; class PGOInstrumentationGenLegacyPass : public ModulePass { public: static char ID; PGOInstrumentationGenLegacyPass(bool IsCS = false) : ModulePass(ID), IsCS(IsCS) { initializePGOInstrumentationGenLegacyPassPass( *PassRegistry::getPassRegistry()); } StringRef getPassName() const override { return "PGOInstrumentationGenPass"; } private: // Is this is context-sensitive instrumentation. bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); } }; class PGOInstrumentationUseLegacyPass : public ModulePass { public: static char ID; // Provide the profile filename as the parameter. PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false) : ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; initializePGOInstrumentationUseLegacyPassPass( *PassRegistry::getPassRegistry()); } StringRef getPassName() const override { return "PGOInstrumentationUsePass"; } private: std::string ProfileFileName; // Is this is context-sensitive instrumentation use. bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); AU.addRequired(); } }; class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass { public: static char ID; StringRef getPassName() const override { return "PGOInstrumentationGenCreateVarPass"; } PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "") : ModulePass(ID), InstrProfileOutput(CSInstrName) { initializePGOInstrumentationGenCreateVarLegacyPassPass( *PassRegistry::getPassRegistry()); } private: bool runOnModule(Module &M) override { createProfileFileNameVar(M, InstrProfileOutput); createIRLevelProfileFlagVar(M, true); return false; } std::string InstrProfileOutput; }; } // end anonymous namespace char PGOInstrumentationGenLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) { return new PGOInstrumentationGenLegacyPass(IsCS); } char PGOInstrumentationUseLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename, bool IsCS) { return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS); } char PGOInstrumentationGenCreateVarLegacyPass::ID = 0; INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass, "pgo-instr-gen-create-var", "Create PGO instrumentation version variable for CSPGO.", false, false) ModulePass * llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) { return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName); } namespace { /// An MST based instrumentation for PGO /// /// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO /// in the function level. struct PGOEdge { // This class implements the CFG edges. Note the CFG can be a multi-graph. // So there might be multiple edges with same SrcBB and DestBB. const BasicBlock *SrcBB; const BasicBlock *DestBB; uint64_t Weight; bool InMST = false; bool Removed = false; bool IsCritical = false; PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1) : SrcBB(Src), DestBB(Dest), Weight(W) {} // Return the information string of an edge. const std::string infoString() const { return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") + (IsCritical ? "c" : " ") + " W=" + Twine(Weight)).str(); } }; // This class stores the auxiliary information for each BB. struct BBInfo { BBInfo *Group; uint32_t Index; uint32_t Rank = 0; BBInfo(unsigned IX) : Group(this), Index(IX) {} // Return the information string of this object. const std::string infoString() const { return (Twine("Index=") + Twine(Index)).str(); } // Empty function -- only applicable to UseBBInfo. void addOutEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {} // Empty function -- only applicable to UseBBInfo. void addInEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {} }; // This class implements the CFG edges. Note the CFG can be a multi-graph. template class FuncPGOInstrumentation { private: Function &F; // Is this is context-sensitive instrumentation. bool IsCS; // A map that stores the Comdat group in function F. std::unordered_multimap &ComdatMembers; void computeCFGHash(); void renameComdatFunction(); public: std::vector> ValueSites; SelectInstVisitor SIVisitor; MemIntrinsicVisitor MIVisitor; std::string FuncName; GlobalVariable *FuncNameVar; // CFG hash value for this function. uint64_t FunctionHash = 0; // The Minimum Spanning Tree of function CFG. CFGMST MST; // Collect all the BBs that will be instrumented, and store them in // InstrumentBBs. void getInstrumentBBs(std::vector &InstrumentBBs); // Give an edge, find the BB that will be instrumented. // Return nullptr if there is no BB to be instrumented. BasicBlock *getInstrBB(Edge *E); // Return the auxiliary BB information. BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); } // Return the auxiliary BB information if available. BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); } // Dump edges and BB information. void dumpInfo(std::string Str = "") const { MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " + Twine(FunctionHash) + "\t" + Str); } FuncPGOInstrumentation( Function &Func, std::unordered_multimap &ComdatMembers, bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr, bool IsCS = false) : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); MIVisitor.countMemIntrinsics(Func); if (!IsCS) { NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); NumOfPGOBB += MST.BBInfos.size(); ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func); } else { NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); NumOfCSPGOBB += MST.BBInfos.size(); } ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func); FuncName = getPGOFuncName(F); computeCFGHash(); if (!ComdatMembers.empty()) renameComdatFunction(); LLVM_DEBUG(dumpInfo("after CFGMST")); for (auto &E : MST.AllEdges) { if (E->Removed) continue; IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++; if (!E->InMST) IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++; } if (CreateGlobalVar) FuncNameVar = createPGOFuncNameVar(F, FuncName); } }; } // end anonymous namespace // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index // value of each BB in the CFG. The higher 32 bits record the number of edges. template void FuncPGOInstrumentation::computeCFGHash() { std::vector Indexes; JamCRC JC; for (auto &BB : F) { const Instruction *TI = BB.getTerminator(); for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) { BasicBlock *Succ = TI->getSuccessor(I); auto BI = findBBInfo(Succ); if (BI == nullptr) continue; uint32_t Index = BI->Index; for (int J = 0; J < 4; J++) Indexes.push_back((char)(Index >> (J * 8))); } } JC.update(Indexes); // Hash format for context sensitive profile. Reserve 4 bits for other // information. FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 | (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 | //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 | (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC(); // Reserve bit 60-63 for other information purpose. FunctionHash &= 0x0FFFFFFFFFFFFFFF; if (IsCS) NamedInstrProfRecord::setCSFlagInHash(FunctionHash); LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n" << " CRC = " << JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts() << ", Edges = " << MST.AllEdges.size() << ", ICSites = " << ValueSites[IPVK_IndirectCallTarget].size() << ", Hash = " << FunctionHash << "\n";); } // Check if we can safely rename this Comdat function. static bool canRenameComdat( Function &F, std::unordered_multimap &ComdatMembers) { if (!DoComdatRenaming || !canRenameComdatFunc(F, true)) return false; // FIXME: Current only handle those Comdat groups that only containing one // function and function aliases. // (1) For a Comdat group containing multiple functions, we need to have a // unique postfix based on the hashes for each function. There is a // non-trivial code refactoring to do this efficiently. // (2) Variables can not be renamed, so we can not rename Comdat function in a // group including global vars. Comdat *C = F.getComdat(); for (auto &&CM : make_range(ComdatMembers.equal_range(C))) { if (dyn_cast(CM.second)) continue; Function *FM = dyn_cast(CM.second); if (FM != &F) return false; } return true; } // Append the CFGHash to the Comdat function name. template void FuncPGOInstrumentation::renameComdatFunction() { if (!canRenameComdat(F, ComdatMembers)) return; std::string OrigName = F.getName().str(); std::string NewFuncName = Twine(F.getName() + "." + Twine(FunctionHash)).str(); F.setName(Twine(NewFuncName)); GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigName, &F); FuncName = Twine(FuncName + "." + Twine(FunctionHash)).str(); Comdat *NewComdat; Module *M = F.getParent(); // For AvailableExternallyLinkage functions, change the linkage to // LinkOnceODR and put them into comdat. This is because after renaming, there // is no backup external copy available for the function. if (!F.hasComdat()) { assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage); NewComdat = M->getOrInsertComdat(StringRef(NewFuncName)); F.setLinkage(GlobalValue::LinkOnceODRLinkage); F.setComdat(NewComdat); return; } // This function belongs to a single function Comdat group. Comdat *OrigComdat = F.getComdat(); std::string NewComdatName = Twine(OrigComdat->getName() + "." + Twine(FunctionHash)).str(); NewComdat = M->getOrInsertComdat(StringRef(NewComdatName)); NewComdat->setSelectionKind(OrigComdat->getSelectionKind()); for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) { if (GlobalAlias *GA = dyn_cast(CM.second)) { // For aliases, change the name directly. assert(dyn_cast(GA->getAliasee()->stripPointerCasts()) == &F); std::string OrigGAName = GA->getName().str(); GA->setName(Twine(GA->getName() + "." + Twine(FunctionHash))); GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigGAName, GA); continue; } // Must be a function. Function *CF = dyn_cast(CM.second); assert(CF); CF->setComdat(NewComdat); } } // Collect all the BBs that will be instruments and return them in // InstrumentBBs and setup InEdges/OutEdge for UseBBInfo. template void FuncPGOInstrumentation::getInstrumentBBs( std::vector &InstrumentBBs) { // Use a worklist as we will update the vector during the iteration. std::vector EdgeList; EdgeList.reserve(MST.AllEdges.size()); for (auto &E : MST.AllEdges) EdgeList.push_back(E.get()); for (auto &E : EdgeList) { BasicBlock *InstrBB = getInstrBB(E); if (InstrBB) InstrumentBBs.push_back(InstrBB); } // Set up InEdges/OutEdges for all BBs. for (auto &E : MST.AllEdges) { if (E->Removed) continue; const BasicBlock *SrcBB = E->SrcBB; const BasicBlock *DestBB = E->DestBB; BBInfo &SrcInfo = getBBInfo(SrcBB); BBInfo &DestInfo = getBBInfo(DestBB); SrcInfo.addOutEdge(E.get()); DestInfo.addInEdge(E.get()); } } // Given a CFG E to be instrumented, find which BB to place the instrumented // code. The function will split the critical edge if necessary. template BasicBlock *FuncPGOInstrumentation::getInstrBB(Edge *E) { if (E->InMST || E->Removed) return nullptr; BasicBlock *SrcBB = const_cast(E->SrcBB); BasicBlock *DestBB = const_cast(E->DestBB); // For a fake edge, instrument the real BB. if (SrcBB == nullptr) return DestBB; if (DestBB == nullptr) return SrcBB; auto canInstrument = [](BasicBlock *BB) -> BasicBlock * { // There are basic blocks (such as catchswitch) cannot be instrumented. // If the returned first insertion point is the end of BB, skip this BB. if (BB->getFirstInsertionPt() == BB->end()) return nullptr; return BB; }; // Instrument the SrcBB if it has a single successor, // otherwise, the DestBB if this is not a critical edge. Instruction *TI = SrcBB->getTerminator(); if (TI->getNumSuccessors() <= 1) return canInstrument(SrcBB); if (!E->IsCritical) return canInstrument(DestBB); unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum); if (!InstrBB) { LLVM_DEBUG( dbgs() << "Fail to split critical edge: not instrument this edge.\n"); return nullptr; } // For a critical edge, we have to split. Instrument the newly // created BB. IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++; LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> " << getBBInfo(DestBB).Index << "\n"); // Need to add two new edges. First one: Add new edge of SrcBB->InstrBB. MST.addEdge(SrcBB, InstrBB, 0); // Second one: Add new edge of InstrBB->DestBB. Edge &NewEdge1 = MST.addEdge(InstrBB, DestBB, 0); NewEdge1.InMST = true; E->Removed = true; return canInstrument(InstrBB); } // Visit all edge and instrument the edges not in MST, and do value profiling. // Critical edges will be split. static void instrumentOneFunc( Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap &ComdatMembers, bool IsCS) { // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); FuncPGOInstrumentation FuncInfo(F, ComdatMembers, true, BPI, BFI, IsCS); std::vector InstrumentBBs; FuncInfo.getInstrumentBBs(InstrumentBBs); unsigned NumCounters = InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts(); uint32_t I = 0; Type *I8PtrTy = Type::getInt8PtrTy(M->getContext()); for (auto *InstrBB : InstrumentBBs) { IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt()); assert(Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment), {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters), Builder.getInt32(I++)}); } // Now instrument select instructions: FuncInfo.SIVisitor.instrumentSelects(F, &I, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash); assert(I == NumCounters); if (DisableValueProfiling) return; unsigned NumIndirectCalls = 0; for (auto &I : FuncInfo.ValueSites[IPVK_IndirectCallTarget]) { CallSite CS(I); Value *Callee = CS.getCalledValue(); LLVM_DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = " << NumIndirectCalls << "\n"); IRBuilder<> Builder(I); assert(Builder.GetInsertPoint() != I->getParent()->end() && "Cannot get the Instrumentation point"); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()), Builder.getInt32(IPVK_IndirectCallTarget), Builder.getInt32(NumIndirectCalls++)}); } NumOfPGOICall += NumIndirectCalls; // Now instrument memop intrinsic calls. FuncInfo.MIVisitor.instrumentMemIntrinsics( F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash); } namespace { // This class represents a CFG edge in profile use compilation. struct PGOUseEdge : public PGOEdge { bool CountValid = false; uint64_t CountValue = 0; PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1) : PGOEdge(Src, Dest, W) {} // Set edge count value void setEdgeCount(uint64_t Value) { CountValue = Value; CountValid = true; } // Return the information string for this object. const std::string infoString() const { if (!CountValid) return PGOEdge::infoString(); return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue)) .str(); } }; using DirectEdges = SmallVector; // This class stores the auxiliary information for each BB. struct UseBBInfo : public BBInfo { uint64_t CountValue = 0; bool CountValid; int32_t UnknownCountInEdge = 0; int32_t UnknownCountOutEdge = 0; DirectEdges InEdges; DirectEdges OutEdges; UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {} UseBBInfo(unsigned IX, uint64_t C) : BBInfo(IX), CountValue(C), CountValid(true) {} // Set the profile count value for this BB. void setBBInfoCount(uint64_t Value) { CountValue = Value; CountValid = true; } // Return the information string of this object. const std::string infoString() const { if (!CountValid) return BBInfo::infoString(); return (Twine(BBInfo::infoString()) + " Count=" + Twine(CountValue)).str(); } // Add an OutEdge and update the edge count. void addOutEdge(PGOUseEdge *E) { OutEdges.push_back(E); UnknownCountOutEdge++; } // Add an InEdge and update the edge count. void addInEdge(PGOUseEdge *E) { InEdges.push_back(E); UnknownCountInEdge++; } }; } // end anonymous namespace // Sum up the count values for all the edges. static uint64_t sumEdgeCount(const ArrayRef Edges) { uint64_t Total = 0; for (auto &E : Edges) { if (E->Removed) continue; Total += E->CountValue; } return Total; } namespace { class PGOUseFunc { public: PGOUseFunc(Function &Func, Module *Modu, std::unordered_multimap &ComdatMembers, - BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false) - : F(Func), M(Modu), BFI(BFIin), + BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin, + ProfileSummaryInfo *PSI, bool IsCS) + : F(Func), M(Modu), BFI(BFIin), PSI(PSI), FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS), FreqAttr(FFA_Normal), IsCS(IsCS) {} // Read counts for the instrumented BB from profile. bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros); // Populate the counts for all BBs. void populateCounters(); // Set the branch weights based on the count values. void setBranchWeights(); // Annotate the value profile call sites for all value kind. void annotateValueSites(); // Annotate the value profile call sites for one value kind. void annotateValueSites(uint32_t Kind); // Annotate the irreducible loop header weights. void annotateIrrLoopHeaderWeights(); // The hotness of the function from the profile count. enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot }; // Return the function hotness from the profile. FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; } // Return the function hash. uint64_t getFuncHash() const { return FuncInfo.FunctionHash; } // Return the profile record for this function; InstrProfRecord &getProfileRecord() { return ProfileRecord; } // Return the auxiliary BB information. UseBBInfo &getBBInfo(const BasicBlock *BB) const { return FuncInfo.getBBInfo(BB); } // Return the auxiliary BB information if available. UseBBInfo *findBBInfo(const BasicBlock *BB) const { return FuncInfo.findBBInfo(BB); } Function &getFunc() const { return F; } void dumpInfo(std::string Str = "") const { FuncInfo.dumpInfo(Str); } uint64_t getProgramMaxCount() const { return ProgramMaxCount; } private: Function &F; Module *M; BlockFrequencyInfo *BFI; + ProfileSummaryInfo *PSI; // This member stores the shared information with class PGOGenFunc. FuncPGOInstrumentation FuncInfo; // The maximum count value in the profile. This is only used in PGO use // compilation. uint64_t ProgramMaxCount; // Position of counter that remains to be read. uint32_t CountPosition = 0; // Total size of the profile count for this function. uint32_t ProfileCountSize = 0; // ProfileRecord for this function. InstrProfRecord ProfileRecord; // Function hotness info derived from profile. FuncFreqAttr FreqAttr; // Is to use the context sensitive profile. bool IsCS; // Find the Instrumented BB and set the value. Return false on error. bool setInstrumentedCounts(const std::vector &CountFromProfile); // Set the edge counter value for the unknown edge -- there should be only // one unknown edge. void setEdgeCount(DirectEdges &Edges, uint64_t Value); // Return FuncName string; const std::string getFuncName() const { return FuncInfo.FuncName; } // Set the hot/cold inline hints based on the count values. // FIXME: This function should be removed once the functionality in // the inliner is implemented. void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) { - if (ProgramMaxCount == 0) - return; - // Threshold of the hot functions. - const BranchProbability HotFunctionThreshold(1, 100); - // Threshold of the cold functions. - const BranchProbability ColdFunctionThreshold(2, 10000); - if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount)) + if (PSI->isHotCount(EntryCount)) FreqAttr = FFA_Hot; - else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount)) + else if (PSI->isColdCount(MaxCount)) FreqAttr = FFA_Cold; } }; } // end anonymous namespace // Visit all the edges and assign the count value for the instrumented // edges and the BB. Return false on error. bool PGOUseFunc::setInstrumentedCounts( const std::vector &CountFromProfile) { std::vector InstrumentBBs; FuncInfo.getInstrumentBBs(InstrumentBBs); unsigned NumCounters = InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts(); // The number of counters here should match the number of counters // in profile. Return if they mismatch. if (NumCounters != CountFromProfile.size()) { return false; } // Set the profile count to the Instrumented BBs. uint32_t I = 0; for (BasicBlock *InstrBB : InstrumentBBs) { uint64_t CountValue = CountFromProfile[I++]; UseBBInfo &Info = getBBInfo(InstrBB); Info.setBBInfoCount(CountValue); } ProfileCountSize = CountFromProfile.size(); CountPosition = I; // Set the edge count and update the count of unknown edges for BBs. auto setEdgeCount = [this](PGOUseEdge *E, uint64_t Value) -> void { E->setEdgeCount(Value); this->getBBInfo(E->SrcBB).UnknownCountOutEdge--; this->getBBInfo(E->DestBB).UnknownCountInEdge--; }; // Set the profile count the Instrumented edges. There are BBs that not in // MST but not instrumented. Need to set the edge count value so that we can // populate the profile counts later. for (auto &E : FuncInfo.MST.AllEdges) { if (E->Removed || E->InMST) continue; const BasicBlock *SrcBB = E->SrcBB; UseBBInfo &SrcInfo = getBBInfo(SrcBB); // If only one out-edge, the edge profile count should be the same as BB // profile count. if (SrcInfo.CountValid && SrcInfo.OutEdges.size() == 1) setEdgeCount(E.get(), SrcInfo.CountValue); else { const BasicBlock *DestBB = E->DestBB; UseBBInfo &DestInfo = getBBInfo(DestBB); // If only one in-edge, the edge profile count should be the same as BB // profile count. if (DestInfo.CountValid && DestInfo.InEdges.size() == 1) setEdgeCount(E.get(), DestInfo.CountValue); } if (E->CountValid) continue; // E's count should have been set from profile. If not, this meenas E skips // the instrumentation. We set the count to 0. setEdgeCount(E.get(), 0); } return true; } // Set the count value for the unknown edge. There should be one and only one // unknown edge in Edges vector. void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) { for (auto &E : Edges) { if (E->CountValid) continue; E->setEdgeCount(Value); getBBInfo(E->SrcBB).UnknownCountOutEdge--; getBBInfo(E->DestBB).UnknownCountInEdge--; return; } llvm_unreachable("Cannot find the unknown count edge"); } // Read the profile from ProfileFileName and assign the value to the // instrumented BB and the edges. This function also updates ProgramMaxCount. // Return true if the profile are successfully read, and false on errors. bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) { auto &Ctx = M->getContext(); Expected Result = PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash); if (Error E = Result.takeError()) { handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { auto Err = IPE.get(); bool SkipWarning = false; LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncInfo.FuncName << ": "); if (Err == instrprof_error::unknown_function) { IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++; SkipWarning = !PGOWarnMissing; LLVM_DEBUG(dbgs() << "unknown function"); } else if (Err == instrprof_error::hash_mismatch || Err == instrprof_error::malformed) { IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++; SkipWarning = NoPGOWarnMismatch || (NoPGOWarnMismatchComdat && (F.hasComdat() || F.getLinkage() == GlobalValue::AvailableExternallyLinkage)); LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")"); } LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n"); if (SkipWarning) return; std::string Msg = IPE.message() + std::string(" ") + F.getName().str() + std::string(" Hash = ") + std::to_string(FuncInfo.FunctionHash); Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); }); return false; } ProfileRecord = std::move(Result.get()); std::vector &CountFromProfile = ProfileRecord.Counts; IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++; LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n"); uint64_t ValueSum = 0; for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) { LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n"); ValueSum += CountFromProfile[I]; } AllZeros = (ValueSum == 0); LLVM_DEBUG(dbgs() << "SUM = " << ValueSum << "\n"); getBBInfo(nullptr).UnknownCountOutEdge = 2; getBBInfo(nullptr).UnknownCountInEdge = 2; if (!setInstrumentedCounts(CountFromProfile)) { LLVM_DEBUG( dbgs() << "Inconsistent number of counts, skipping this function"); Ctx.diagnose(DiagnosticInfoPGOProfile( M->getName().data(), Twine("Inconsistent number of counts in ") + F.getName().str() + Twine(": the profile may be stale or there is a function name collision."), DS_Warning)); return false; } ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS); return true; } // Populate the counters from instrumented BBs to all BBs. // In the end of this operation, all BBs should have a valid count value. void PGOUseFunc::populateCounters() { bool Changes = true; unsigned NumPasses = 0; while (Changes) { NumPasses++; Changes = false; // For efficient traversal, it's better to start from the end as most // of the instrumented edges are at the end. for (auto &BB : reverse(F)) { UseBBInfo *Count = findBBInfo(&BB); if (Count == nullptr) continue; if (!Count->CountValid) { if (Count->UnknownCountOutEdge == 0) { Count->CountValue = sumEdgeCount(Count->OutEdges); Count->CountValid = true; Changes = true; } else if (Count->UnknownCountInEdge == 0) { Count->CountValue = sumEdgeCount(Count->InEdges); Count->CountValid = true; Changes = true; } } if (Count->CountValid) { if (Count->UnknownCountOutEdge == 1) { uint64_t Total = 0; uint64_t OutSum = sumEdgeCount(Count->OutEdges); // If the one of the successor block can early terminate (no-return), // we can end up with situation where out edge sum count is larger as // the source BB's count is collected by a post-dominated block. if (Count->CountValue > OutSum) Total = Count->CountValue - OutSum; setEdgeCount(Count->OutEdges, Total); Changes = true; } if (Count->UnknownCountInEdge == 1) { uint64_t Total = 0; uint64_t InSum = sumEdgeCount(Count->InEdges); if (Count->CountValue > InSum) Total = Count->CountValue - InSum; setEdgeCount(Count->InEdges, Total); Changes = true; } } } } LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n"); #ifndef NDEBUG // Assert every BB has a valid counter. for (auto &BB : F) { auto BI = findBBInfo(&BB); if (BI == nullptr) continue; assert(BI->CountValid && "BB count is not valid"); } #endif uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue; F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real)); uint64_t FuncMaxCount = FuncEntryCount; for (auto &BB : F) { auto BI = findBBInfo(&BB); if (BI == nullptr) continue; FuncMaxCount = std::max(FuncMaxCount, BI->CountValue); } markFunctionAttributes(FuncEntryCount, FuncMaxCount); // Now annotate select instructions FuncInfo.SIVisitor.annotateSelects(F, this, &CountPosition); assert(CountPosition == ProfileCountSize); LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile.")); } // Assign the scaled count values to the BB with multiple out edges. void PGOUseFunc::setBranchWeights() { // Generate MD_prof metadata for every branch instruction. LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName() << " IsCS=" << IsCS << "\n"); for (auto &BB : F) { Instruction *TI = BB.getTerminator(); if (TI->getNumSuccessors() < 2) continue; if (!(isa(TI) || isa(TI) || isa(TI))) continue; if (getBBInfo(&BB).CountValue == 0) continue; // We have a non-zero Branch BB. const UseBBInfo &BBCountInfo = getBBInfo(&BB); unsigned Size = BBCountInfo.OutEdges.size(); SmallVector EdgeCounts(Size, 0); uint64_t MaxCount = 0; for (unsigned s = 0; s < Size; s++) { const PGOUseEdge *E = BBCountInfo.OutEdges[s]; const BasicBlock *SrcBB = E->SrcBB; const BasicBlock *DestBB = E->DestBB; if (DestBB == nullptr) continue; unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); uint64_t EdgeCount = E->CountValue; if (EdgeCount > MaxCount) MaxCount = EdgeCount; EdgeCounts[SuccNum] = EdgeCount; } setProfMetadata(M, TI, EdgeCounts, MaxCount); } } static bool isIndirectBrTarget(BasicBlock *BB) { for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { if (isa((*PI)->getTerminator())) return true; } return false; } void PGOUseFunc::annotateIrrLoopHeaderWeights() { LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n"); // Find irr loop headers for (auto &BB : F) { // As a heuristic also annotate indrectbr targets as they have a high chance // to become an irreducible loop header after the indirectbr tail // duplication. if (BFI->isIrrLoopHeader(&BB) || isIndirectBrTarget(&BB)) { Instruction *TI = BB.getTerminator(); const UseBBInfo &BBCountInfo = getBBInfo(&BB); setIrrLoopHeaderMetadata(M, TI, BBCountInfo.CountValue); } } } void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) { Module *M = F.getParent(); IRBuilder<> Builder(&SI); Type *Int64Ty = Builder.getInt64Ty(); Type *I8PtrTy = Builder.getInt8PtrTy(); auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step), {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs), Builder.getInt32(*CurCtrIdx), Step}); ++(*CurCtrIdx); } void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) { std::vector &CountFromProfile = UseFunc->getProfileRecord().Counts; assert(*CurCtrIdx < CountFromProfile.size() && "Out of bound access of counters"); uint64_t SCounts[2]; SCounts[0] = CountFromProfile[*CurCtrIdx]; // True count ++(*CurCtrIdx); uint64_t TotalCount = 0; auto BI = UseFunc->findBBInfo(SI.getParent()); if (BI != nullptr) TotalCount = BI->CountValue; // False Count SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0); uint64_t MaxCount = std::max(SCounts[0], SCounts[1]); if (MaxCount) setProfMetadata(F.getParent(), &SI, SCounts, MaxCount); } void SelectInstVisitor::visitSelectInst(SelectInst &SI) { if (!PGOInstrSelect) return; // FIXME: do not handle this yet. if (SI.getCondition()->getType()->isVectorTy()) return; switch (Mode) { case VM_counting: NSIs++; return; case VM_instrument: instrumentOneSelectInst(SI); return; case VM_annotate: annotateOneSelectInst(SI); return; } llvm_unreachable("Unknown visiting mode"); } void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) { Module *M = F.getParent(); IRBuilder<> Builder(&MI); Type *Int64Ty = Builder.getInt64Ty(); Type *I8PtrTy = Builder.getInt8PtrTy(); Value *Length = MI.getLength(); assert(!isa(Length)); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty), Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)}); ++CurCtrId; } void MemIntrinsicVisitor::visitMemIntrinsic(MemIntrinsic &MI) { if (!PGOInstrMemOP) return; Value *Length = MI.getLength(); // Not instrument constant length calls. if (dyn_cast(Length)) return; switch (Mode) { case VM_counting: NMemIs++; return; case VM_instrument: instrumentOneMemIntrinsic(MI); return; case VM_annotate: Candidates.push_back(&MI); return; } llvm_unreachable("Unknown visiting mode"); } // Traverse all valuesites and annotate the instructions for all value kind. void PGOUseFunc::annotateValueSites() { if (DisableValueProfiling) return; // Create the PGOFuncName meta data. createPGOFuncNameMetadata(F, FuncInfo.FuncName); for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) annotateValueSites(Kind); } static const char *ValueProfKindDescr[] = { #define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr, #include "llvm/ProfileData/InstrProfData.inc" }; // Annotate the instructions for a specific value kind. void PGOUseFunc::annotateValueSites(uint32_t Kind) { assert(Kind <= IPVK_Last); unsigned ValueSiteIndex = 0; auto &ValueSites = FuncInfo.ValueSites[Kind]; unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind); if (NumValueSites != ValueSites.size()) { auto &Ctx = M->getContext(); Ctx.diagnose(DiagnosticInfoPGOProfile( M->getName().data(), Twine("Inconsistent number of value sites for ") + Twine(ValueProfKindDescr[Kind]) + Twine(" profiling in \"") + F.getName().str() + Twine("\", possibly due to the use of a stale profile."), DS_Warning)); return; } for (auto &I : ValueSites) { LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kind << "): Index = " << ValueSiteIndex << " out of " << NumValueSites << "\n"); annotateValueSite(*M, *I, ProfileRecord, static_cast(Kind), ValueSiteIndex, Kind == IPVK_MemOPSize ? MaxNumMemOPAnnotations : MaxNumAnnotations); ValueSiteIndex++; } } // Collect the set of members for each Comdat in module M and store // in ComdatMembers. static void collectComdatMembers( Module &M, std::unordered_multimap &ComdatMembers) { if (!DoComdatRenaming) return; for (Function &F : M) if (Comdat *C = F.getComdat()) ComdatMembers.insert(std::make_pair(C, &F)); for (GlobalVariable &GV : M.globals()) if (Comdat *C = GV.getComdat()) ComdatMembers.insert(std::make_pair(C, &GV)); for (GlobalAlias &GA : M.aliases()) if (Comdat *C = GA.getComdat()) ComdatMembers.insert(std::make_pair(C, &GA)); } static bool InstrumentAllFunctions( Module &M, function_ref LookupBPI, function_ref LookupBFI, bool IsCS) { // For the context-sensitve instrumentation, we should have a separated pass // (before LTO/ThinLTO linking) to create these variables. if (!IsCS) createIRLevelProfileFlagVar(M, /* IsCS */ false); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); for (auto &F : M) { if (F.isDeclaration()) continue; auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS); } return true; } PreservedAnalyses PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) { createProfileFileNameVar(M, CSInstrName); createIRLevelProfileFlagVar(M, /* IsCS */ true); return PreservedAnalyses::all(); } bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { if (skipModule(M)) return false; auto LookupBPI = [this](Function &F) { return &this->getAnalysis(F).getBPI(); }; auto LookupBFI = [this](Function &F) { return &this->getAnalysis(F).getBFI(); }; return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS); } PreservedAnalyses PGOInstrumentationGen::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult(M).getManager(); auto LookupBPI = [&FAM](Function &F) { return &FAM.getResult(F); }; auto LookupBFI = [&FAM](Function &F) { return &FAM.getResult(F); }; if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); } static bool annotateAllFunctions( Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref LookupBPI, - function_ref LookupBFI, bool IsCS) { + function_ref LookupBFI, + ProfileSummaryInfo *PSI, bool IsCS) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName); if (Error E = ReaderOrErr.takeError()) { handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { Ctx.diagnose( DiagnosticInfoPGOProfile(ProfileFileName.data(), EI.message())); }); return false; } std::unique_ptr PGOReader = std::move(ReaderOrErr.get()); if (!PGOReader) { Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(), StringRef("Cannot get PGOReader"))); return false; } if (!PGOReader->hasCSIRLevelProfile() && IsCS) return false; // TODO: might need to change the warning once the clang option is finalized. if (!PGOReader->isIRLevelProfile()) { Ctx.diagnose(DiagnosticInfoPGOProfile( ProfileFileName.data(), "Not an IR level instrumentation profile")); return false; } + // Add the profile summary (read from the header of the indexed summary) here + // so that we can use it below when reading counters (which checks if the + // function should be marked with a cold or inlinehint attribute). + M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), + IsCS ? ProfileSummary::PSK_CSInstr + : ProfileSummary::PSK_Instr); + std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); std::vector HotFunctions; std::vector ColdFunctions; for (auto &F : M) { if (F.isDeclaration()) continue; auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS); + PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, PSI, IsCS); bool AllZeros = false; if (!Func.readCounters(PGOReader.get(), AllZeros)) continue; if (AllZeros) { F.setEntryCount(ProfileCount(0, Function::PCT_Real)); if (Func.getProgramMaxCount() != 0) ColdFunctions.push_back(&F); continue; } Func.populateCounters(); Func.setBranchWeights(); Func.annotateValueSites(); Func.annotateIrrLoopHeaderWeights(); PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr(); if (FreqAttr == PGOUseFunc::FFA_Cold) ColdFunctions.push_back(&F); else if (FreqAttr == PGOUseFunc::FFA_Hot) HotFunctions.push_back(&F); if (PGOViewCounts != PGOVCT_None && (ViewBlockFreqFuncName.empty() || F.getName().equals(ViewBlockFreqFuncName))) { LoopInfo LI{DominatorTree(F)}; std::unique_ptr NewBPI = std::make_unique(F, LI); std::unique_ptr NewBFI = std::make_unique(F, *NewBPI, LI); if (PGOViewCounts == PGOVCT_Graph) NewBFI->view(); else if (PGOViewCounts == PGOVCT_Text) { dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n"; NewBFI->print(dbgs()); } } if (PGOViewRawCounts != PGOVCT_None && (ViewBlockFreqFuncName.empty() || F.getName().equals(ViewBlockFreqFuncName))) { if (PGOViewRawCounts == PGOVCT_Graph) if (ViewBlockFreqFuncName.empty()) WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName()); else ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName()); else if (PGOViewRawCounts == PGOVCT_Text) { dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n"; Func.dumpInfo(); } } } - M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), - IsCS ? ProfileSummary::PSK_CSInstr - : ProfileSummary::PSK_Instr); // Set function hotness attribute from the profile. // We have to apply these attributes at the end because their presence // can affect the BranchProbabilityInfo of any callers, resulting in an // inconsistent MST between prof-gen and prof-use. for (auto &F : HotFunctions) { F->addFnAttr(Attribute::InlineHint); LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName() << "\n"); } for (auto &F : ColdFunctions) { F->addFnAttr(Attribute::Cold); LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n"); } return true; } PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename, std::string RemappingFilename, bool IsCS) : ProfileFileName(std::move(Filename)), ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; if (!PGOTestProfileRemappingFile.empty()) ProfileRemappingFileName = PGOTestProfileRemappingFile; } PreservedAnalyses PGOInstrumentationUse::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult(M).getManager(); auto LookupBPI = [&FAM](Function &F) { return &FAM.getResult(F); }; auto LookupBFI = [&FAM](Function &F) { return &FAM.getResult(F); }; + auto *PSI = &AM.getResult(M); + if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, - LookupBPI, LookupBFI, IsCS)) + LookupBPI, LookupBFI, PSI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); } bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { if (skipModule(M)) return false; auto LookupBPI = [this](Function &F) { return &this->getAnalysis(F).getBPI(); }; auto LookupBFI = [this](Function &F) { return &this->getAnalysis(F).getBFI(); }; - return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, + auto *PSI = &getAnalysis().getPSI(); + return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, PSI, IsCS); } static std::string getSimpleNodeName(const BasicBlock *Node) { if (!Node->getName().empty()) return Node->getName(); std::string SimpleNodeName; raw_string_ostream OS(SimpleNodeName); Node->printAsOperand(OS, false); return OS.str(); } void llvm::setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, uint64_t MaxCount) { MDBuilder MDB(M->getContext()); assert(MaxCount > 0 && "Bad max count"); uint64_t Scale = calculateCountScale(MaxCount); SmallVector Weights; for (const auto &ECI : EdgeCounts) Weights.push_back(scaleBranchCount(ECI, Scale)); LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &W : Weights) { dbgs() << W << " "; } dbgs() << "\n";); misexpect::verifyMisExpect(TI, Weights, TI->getContext()); TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); if (EmitBranchProbability) { std::string BrCondStr = getBranchCondString(TI); if (BrCondStr.empty()) return; uint64_t WSum = std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0, [](uint64_t w1, uint64_t w2) { return w1 + w2; }); uint64_t TotalCount = std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0, [](uint64_t c1, uint64_t c2) { return c1 + c2; }); Scale = calculateCountScale(WSum); BranchProbability BP(scaleBranchCount(Weights[0], Scale), scaleBranchCount(WSum, Scale)); std::string BranchProbStr; raw_string_ostream OS(BranchProbStr); OS << BP; OS << " (total count : " << TotalCount << ")"; OS.flush(); Function *F = TI->getParent()->getParent(); OptimizationRemarkEmitter ORE(F); ORE.emit([&]() { return OptimizationRemark(DEBUG_TYPE, "pgo-instrumentation", TI) << BrCondStr << " is true with probability : " << BranchProbStr; }); } } namespace llvm { void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) { MDBuilder MDB(M->getContext()); TI->setMetadata(llvm::LLVMContext::MD_irr_loop, MDB.createIrrLoopHeaderWeight(Count)); } template <> struct GraphTraits { using NodeRef = const BasicBlock *; using ChildIteratorType = succ_const_iterator; using nodes_iterator = pointer_iterator; static NodeRef getEntryNode(const PGOUseFunc *G) { return &G->getFunc().front(); } static ChildIteratorType child_begin(const NodeRef N) { return succ_begin(N); } static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); } static nodes_iterator nodes_begin(const PGOUseFunc *G) { return nodes_iterator(G->getFunc().begin()); } static nodes_iterator nodes_end(const PGOUseFunc *G) { return nodes_iterator(G->getFunc().end()); } }; template <> struct DOTGraphTraits : DefaultDOTGraphTraits { explicit DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} static std::string getGraphName(const PGOUseFunc *G) { return G->getFunc().getName(); } std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph) { std::string Result; raw_string_ostream OS(Result); OS << getSimpleNodeName(Node) << ":\\l"; UseBBInfo *BI = Graph->findBBInfo(Node); OS << "Count : "; if (BI && BI->CountValid) OS << BI->CountValue << "\\l"; else OS << "Unknown\\l"; if (!PGOInstrSelect) return Result; for (auto BI = Node->begin(); BI != Node->end(); ++BI) { auto *I = &*BI; if (!isa(I)) continue; // Display scaled counts for SELECT instruction: OS << "SELECT : { T = "; uint64_t TC, FC; bool HasProf = I->extractProfMetadata(TC, FC); if (!HasProf) OS << "Unknown, F = Unknown }\\l"; else OS << TC << ", F = " << FC << " }\\l"; } return Result; } }; } // end namespace llvm diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext index 2dc2c2ec9f36..230f44ba443d 100644 --- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext +++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext @@ -1,17 +1,25 @@ # IR level Instrumentation Flag :ir -foo +hot # Func Hash: 12884901887 # Num Counters: 1 # Counter Values: -9999 +9000 -bar +cold # Func Hash: 12884901887 # Num Counters: 1 # Counter Values: -0 +10 + +med +# Func Hash: +12884901887 +# Num Counters: +1 +# Counter Values: +50 diff --git a/llvm/test/Transforms/PGOProfile/func_entry.ll b/llvm/test/Transforms/PGOProfile/func_entry.ll index 37fad27f2f2f..fe1b44b0bd71 100644 --- a/llvm/test/Transforms/PGOProfile/func_entry.ll +++ b/llvm/test/Transforms/PGOProfile/func_entry.ll @@ -1,29 +1,44 @@ ; RUN: llvm-profdata merge %S/Inputs/func_entry.proftext -o %t.profdata ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @s = common dso_local local_unnamed_addr global i32 0, align 4 -define void @bar() { -; CHECK-LABEL: @bar +define void @cold() { +; CHECK-LABEL: @cold() +; CHECK-SAME: #[[COLD_ATTR:[0-1]+]] ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_ZERO:[0-9]+]] entry: store i32 1, i32* @s, align 4 ret void } -define void @foo() { -; CHECK-LABEL: @foo +define void @hot() { +; CHECK-LABEL: @hot() +; CHECK-SAME: #[[HOT_ATTR:[0-1]+]] ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_NON_ZERO:[0-9]+]] entry: %0 = load i32, i32* @s, align 4 %add = add nsw i32 %0, 4 store i32 %add, i32* @s, align 4 ret void } -; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 0} -; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9999} +define void @med() { +; CHECK-LABEL: @med +; CHECK-NOT: # +; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_MED:[0-9]+]] + +entry: + store i32 1, i32* @s, align 4 + ret void +} + +; CHECK-DAG: attributes #[[COLD_ATTR]] = { cold } +; CHECK-DAG: attributes #[[HOT_ATTR]] = { inlinehint } +; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 10} +; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9000} +; CHECK-DAG: ![[FUNC_ENTRY_COUNT_MED]] = !{!"function_entry_count", i64 50}