diff --git a/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/profile/Darwin/instrprof-debug-info-correlate.c @@ -0,0 +1,31 @@ +// Value profiling is currently not supported in lightweight mode. +// RUN: %clang_pgogen -o %t.normal -mllvm --disable-vp=true %s +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw + +// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s +// RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t +// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t.dSYM %t.proflite + +// RUN: diff %t.normal.profdata %t.profdata + +int foo(int a) { + if (a % 2) + return 4 * a + 1; + return 0; +} + +int bar(int a) { + while (a > 100) + a /= 2; + return a; +} + +typedef int (*FP)(int); +FP Fps[3] = {foo, bar}; + +int main() { + for (int i = 0; i < 5; i++) + Fps[i % 2](i); + return 0; +} diff --git a/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/profile/Linux/instrprof-debug-info-correlate.c @@ -0,0 +1,31 @@ +// Value profiling is currently not supported in lightweight mode. +// RUN: %clang_pgogen -o %t.normal -mllvm --disable-vp=true %s +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal +// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw + +// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s +// RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t +// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite + +// RUN: diff %t.normal.profdata %t.profdata + +int foo(int a) { + if (a % 2) + return 4 * a + 1; + return 0; +} + +int bar(int a) { + while (a > 100) + a /= 2; + return a; +} + +typedef int (*FP)(int); +FP Fps[3] = {foo, bar}; + +int main() { + for (int i = 0; i < 5; i++) + Fps[i % 2](i); + return 0; +} diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -290,6 +290,10 @@ too_large, truncated, malformed, + missing_debug_info_for_correlation, + unexpected_debug_info_for_correlation, + unable_to_correlate_profile, + unsupported_debug_format, unknown_function, invalid_prof, hash_mismatch, diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -0,0 +1,170 @@ +//===- InstrProfCorrelator.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This file defines InstrProfCorrelator used to generate PGO profiles from +// raw profile data and debug info. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H +#define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H + +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include + +namespace llvm { + +/// InstrProfCorrelator - A base class used to create raw instrumentation data +/// to their functions. +class InstrProfCorrelator { +public: + static llvm::Expected> + get(StringRef DebugInfoFilename); + + /// Construct a ProfileData vector used to correlate raw instrumentation data + /// to their functions. + virtual Error correlateProfileData() = 0; + + static const char *FunctionNameAttributeName; + static const char *CFGHashAttributeName; + static const char *NumCountersAttributeName; + + enum InstrProfCorrelatorKind { CK_32Bit, CK_64Bit }; + InstrProfCorrelatorKind getKind() const { return Kind; } + virtual ~InstrProfCorrelator() {} + +protected: + struct Context { + static llvm::Expected> + get(std::unique_ptr Buffer, const object::ObjectFile &Obj); + std::unique_ptr Buffer; + /// The address range of the __llvm_prf_cnts section. + uint64_t CountersSectionStart; + uint64_t CountersSectionEnd; + /// True if target and host have different endian orders. + bool ShouldSwapBytes; + }; + const std::unique_ptr Ctx; + + InstrProfCorrelator(InstrProfCorrelatorKind K, std::unique_ptr Ctx) + : Ctx(std::move(Ctx)), Kind(K) {} + +private: + static llvm::Expected> + get(std::unique_ptr Buffer); + + const InstrProfCorrelatorKind Kind; +}; + +/// InstrProfCorrelatorImpl - A child of InstrProfCorrelator with a template +/// pointer type so that the ProfileData vector can be materialized. +template +class InstrProfCorrelatorImpl : public InstrProfCorrelator { +public: + InstrProfCorrelatorImpl(std::unique_ptr Ctx); + static bool classof(const InstrProfCorrelator *C); + + /// Return a pointer to the underlying ProfileData vector that this class + /// constructs. + const RawInstrProf::ProfileData *getDataPointer() const { + return Data.empty() ? nullptr : Data.data(); + } + + /// Return the number of ProfileData elements. + size_t getDataSize() const { return Data.size(); } + + /// Return a pointer to the compressed names string that this class + /// constructs. + const char *getCompressedNamesPointer() const { + return CompressedNames.c_str(); + } + + /// Return the number of bytes in the compressed names string. + size_t getCompressedNamesSize() const { return CompressedNames.size(); } + + static llvm::Expected>> + get(std::unique_ptr Ctx, + const object::ObjectFile &Obj); + +protected: + std::vector> Data; + std::string CompressedNames; + + Error correlateProfileData() override; + virtual void correlateProfileDataImpl() = 0; + + void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, + IntPtrT FunctionPtr, uint32_t NumCounters); + +private: + InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind, + std::unique_ptr Ctx) + : InstrProfCorrelator(Kind, std::move(Ctx)){}; + std::vector Names; + + // Byte-swap the value if necessary. + template T maybeSwap(T Value) const { + return Ctx->ShouldSwapBytes ? sys::getSwappedBytes(Value) : Value; + } +}; + +/// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes +/// DWARF debug info as input to correlate profiles. +template +class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl { +public: + DwarfInstrProfCorrelator(std::unique_ptr DICtx, + std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(std::move(Ctx)), + DICtx(std::move(DICtx)) {} + +private: + std::unique_ptr DICtx; + + /// Return the address of the object that the provided DIE symbolizes. + llvm::Optional getLocation(const DWARFDie &Die) const; + + /// Returns true if the provided DIE symbolizes an instrumentation probe + /// symbol. + static bool isDIEOfProbe(const DWARFDie &Die); + + /// Iterate over DWARF DIEs to find those that symbolize instrumentation + /// probes and construct the ProfileData vector and CompressedNames string. + /// + /// Here is some example DWARF for an instrumentation probe we are looking + /// for: + /// \code + /// DW_TAG_subprogram + /// DW_AT_low_pc (0x0000000000000000) + /// DW_AT_high_pc (0x0000000000000014) + /// DW_AT_name ("foo") + /// DW_TAG_variable + /// DW_AT_name ("__profc_foo") + /// DW_AT_location (DW_OP_addr 0x0) + /// DW_TAG_LLVM_annotation + /// DW_AT_name ("Function Name") + /// DW_AT_const_value ("foo") + /// DW_TAG_LLVM_annotation + /// DW_AT_name ("CFG Hash") + /// DW_AT_const_value (12345678) + /// DW_TAG_LLVM_annotation + /// DW_AT_name ("Num Counters") + /// DW_AT_const_value (2) + /// NULL + /// NULL + /// \endcode + void correlateProfileDataImpl() override; +}; + +} // end namespace llvm + +#endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" @@ -96,6 +97,9 @@ virtual bool instrEntryBBEnabled() const = 0; + /// Return true if we must provide debug info to create PGO profiles. + virtual bool useDebugInfoCorrelate() const { return false; } + /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types /// of readers are used only by llvm-profdata tool, while the indexed @@ -150,10 +154,12 @@ /// Factory method to create an appropriately typed reader for the given /// instrprof file. - static Expected> create(const Twine &Path); + static Expected> + create(const Twine &Path, const InstrProfCorrelator *Correlator = nullptr); static Expected> - create(std::unique_ptr Buffer); + create(std::unique_ptr Buffer, + const InstrProfCorrelator *Correlator = nullptr); }; /// Reader for the simple text based instrprof format. @@ -215,6 +221,9 @@ private: /// The profile data file contents. std::unique_ptr DataBuffer; + /// If available, this hold the ProfileData array used to correlate raw + /// instrumentation data to their functions. + const InstrProfCorrelatorImpl *Correlator; bool ShouldSwapBytes; // The value of the version field of the raw profile data header. The lower 56 // bits specifies the format version and the most significant 8 bits specify @@ -226,7 +235,7 @@ const RawInstrProf::ProfileData *DataEnd; const uint64_t *CountersStart; const char *NamesStart; - uint64_t NamesSize; + const char *NamesEnd; // After value profile is all read, this pointer points to // the header of next profile data (if exists) const uint8_t *ValueDataStart; @@ -237,8 +246,11 @@ const uint8_t *BinaryIdsStart; public: - RawInstrProfReader(std::unique_ptr DataBuffer) - : DataBuffer(std::move(DataBuffer)) {} + RawInstrProfReader(std::unique_ptr DataBuffer, + const InstrProfCorrelator *Correlator) + : DataBuffer(std::move(DataBuffer)), + Correlator(dyn_cast_or_null>( + Correlator)) {} RawInstrProfReader(const RawInstrProfReader &) = delete; RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; @@ -259,6 +271,10 @@ return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; } + bool useDebugInfoCorrelate() const override { + return (Version & VARIANT_MASK_DBG_CORRELATE) != 0; + } + InstrProfSymtab &getSymtab() override { assert(Symtab.get()); return *Symtab.get(); diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt --- a/llvm/lib/ProfileData/CMakeLists.txt +++ b/llvm/lib/ProfileData/CMakeLists.txt @@ -1,6 +1,7 @@ add_llvm_component_library(LLVMProfileData GCOV.cpp InstrProf.cpp + InstrProfCorrelator.cpp InstrProfReader.cpp InstrProfWriter.cpp ProfileSummaryBuilder.cpp @@ -19,6 +20,8 @@ Core Support Demangle + Object + DebugInfoDWARF ) add_subdirectory(Coverage) diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -110,6 +110,18 @@ case instrprof_error::malformed: OS << "malformed instrumentation profile data"; break; + case instrprof_error::missing_debug_info_for_correlation: + OS << "debug info for correlation is required"; + break; + case instrprof_error::unexpected_debug_info_for_correlation: + OS << "debug info for correlation is not necessary"; + break; + case instrprof_error::unable_to_correlate_profile: + OS << "unable to correlate profile"; + break; + case instrprof_error::unsupported_debug_format: + OS << "unsupported debug info format (only DWARF is supported)"; + break; case instrprof_error::invalid_prof: OS << "invalid profile created. Please file a bug " "at: " BUG_REPORT_URL diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -0,0 +1,260 @@ +//===-- InstrProfCorrelator.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/InstrProfCorrelator.h" +#include "llvm/Object/MachO.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +#define DEBUG_TYPE "correlator" + +using namespace llvm; + +/// Get the __llvm_prf_cnts section. +Expected getCountersSection(const object::ObjectFile &Obj) { + for (auto &Section : Obj.sections()) + if (auto SectionName = Section.getName()) + if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) + return Section; + return make_error( + instrprof_error::unable_to_correlate_profile); +} + +const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; +const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; +const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; + +llvm::Expected> +InstrProfCorrelator::Context::get(std::unique_ptr Buffer, + const object::ObjectFile &Obj) { + auto CountersSection = getCountersSection(Obj); + if (auto Err = CountersSection.takeError()) + return std::move(Err); + auto C = std::make_unique(); + C->Buffer = std::move(Buffer); + C->CountersSectionStart = CountersSection->getAddress(); + C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); + C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; + return C; +} + +llvm::Expected> +InstrProfCorrelator::get(StringRef DebugInfoFilename) { + auto DsymObjectsOrErr = + object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); + if (auto Err = DsymObjectsOrErr.takeError()) + return std::move(Err); + if (!DsymObjectsOrErr->empty()) { + // TODO: Enable profile correlation when there are multiple objects in a + // dSYM bundle. + if (DsymObjectsOrErr->size() > 1) + return createStringError( + std::error_code(), + "Profile correlation using multiple objects is not yet supported"); + DebugInfoFilename = *DsymObjectsOrErr->begin(); + } + auto BufferOrErr = + errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); + if (auto Err = BufferOrErr.takeError()) + return std::move(Err); + + return get(std::move(*BufferOrErr)); +} + +llvm::Expected> +InstrProfCorrelator::get(std::unique_ptr Buffer) { + auto BinOrErr = object::createBinary(*Buffer); + if (auto Err = BinOrErr.takeError()) + return std::move(Err); + + if (auto *Obj = dyn_cast(BinOrErr->get())) { + auto CtxOrErr = Context::get(std::move(Buffer), *Obj); + if (auto Err = CtxOrErr.takeError()) + return std::move(Err); + auto T = Obj->makeTriple(); + if (T.isArch64Bit()) + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + if (T.isArch32Bit()) + return InstrProfCorrelatorImpl::get(std::move(*CtxOrErr), *Obj); + } + return make_error( + instrprof_error::unable_to_correlate_profile); +} + +template <> +InstrProfCorrelatorImpl::InstrProfCorrelatorImpl( + std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, + std::move(Ctx)) {} +template <> +InstrProfCorrelatorImpl::InstrProfCorrelatorImpl( + std::unique_ptr Ctx) + : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, + std::move(Ctx)) {} +template <> +bool InstrProfCorrelatorImpl::classof(const InstrProfCorrelator *C) { + return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; +} +template <> +bool InstrProfCorrelatorImpl::classof(const InstrProfCorrelator *C) { + return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; +} + +template +llvm::Expected>> +InstrProfCorrelatorImpl::get( + std::unique_ptr Ctx, + const object::ObjectFile &Obj) { + if (Obj.isELF() || Obj.isMachO()) { + auto DICtx = DWARFContext::create(Obj); + return std::make_unique>(std::move(DICtx), + std::move(Ctx)); + } + return make_error(instrprof_error::unsupported_debug_format); +} + +template +Error InstrProfCorrelatorImpl::correlateProfileData() { + assert(Data.empty() && CompressedNames.empty() && Names.empty()); + correlateProfileDataImpl(); + auto Result = + collectPGOFuncNameStrings(Names, /*doCompression=*/true, CompressedNames); + Names.clear(); + return Result; +} + +template +void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, + uint64_t CFGHash, + IntPtrT CounterOffset, + IntPtrT FunctionPtr, + uint32_t NumCounters) { + Data.push_back({ + maybeSwap(IndexedInstrProf::ComputeHash(FunctionName)), + maybeSwap(CFGHash), + // In this mode, CounterPtr actually stores the section relative address + // of the counter. + maybeSwap(CounterOffset), + maybeSwap(FunctionPtr), + // TODO: Value profiling is not yet supported. + /*ValuesPtr=*/maybeSwap(0), + maybeSwap(NumCounters), + /*NumValueSites=*/{maybeSwap(0), maybeSwap(0)}, + }); + Names.push_back(FunctionName.str()); +} + +template +llvm::Optional +DwarfInstrProfCorrelator::getLocation(const DWARFDie &Die) const { + auto Locations = Die.getLocations(dwarf::DW_AT_location); + if (!Locations) { + consumeError(Locations.takeError()); + return {}; + } + auto &DU = *Die.getDwarfUnit(); + for (auto &Location : *Locations) { + auto AddressSize = DU.getAddressByteSize(); + DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); + DWARFExpression Expr(Data, AddressSize); + for (auto &Op : Expr) + if (Op.getCode() == dwarf::DW_OP_addr) + return Op.getRawOperand(0); + } + return {}; +} + +template +bool DwarfInstrProfCorrelator::isDIEOfProbe(const DWARFDie &Die) { + const auto &ParentDie = Die.getParent(); + if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) + return false; + if (Die.getTag() != dwarf::DW_TAG_variable) + return false; + if (!ParentDie.isSubprogramDIE()) + return false; + if (!Die.hasChildren()) + return false; + if (const char *Name = Die.getName(DINameKind::ShortName)) + return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); + return false; +} + +template +void DwarfInstrProfCorrelator::correlateProfileDataImpl() { + auto maybeAddProbe = [&](DWARFDie Die) { + if (!isDIEOfProbe(Die)) + return; + Optional FunctionName; + Optional CFGHash; + Optional CounterPtr = getLocation(Die); + auto FunctionPtr = + dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc)); + Optional NumCounters; + for (const DWARFDie &Child : Die.children()) { + if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) + continue; + auto AnnotationFormName = Child.find(dwarf::DW_AT_name); + auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); + if (!AnnotationFormName || !AnnotationFormValue) + continue; + auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); + if (auto Err = AnnotationNameOrErr.takeError()) { + consumeError(std::move(Err)); + continue; + } + StringRef AnnotationName = *AnnotationNameOrErr; + if (AnnotationName.compare( + InstrProfCorrelator::FunctionNameAttributeName) == 0) { + if (auto EC = + AnnotationFormValue->getAsCString().moveInto(FunctionName)) + consumeError(std::move(EC)); + } else if (AnnotationName.compare( + InstrProfCorrelator::CFGHashAttributeName) == 0) { + CFGHash = AnnotationFormValue->getAsUnsignedConstant(); + } else if (AnnotationName.compare( + InstrProfCorrelator::NumCountersAttributeName) == 0) { + NumCounters = AnnotationFormValue->getAsUnsignedConstant(); + } + } + if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { + LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " + << FunctionName << "\n\tCFGHash: " << CFGHash + << "\n\tCounterPtr: " << CounterPtr + << "\n\tNumCounters: " << NumCounters); + LLVM_DEBUG(Die.dump(dbgs())); + return; + } + uint64_t CountersStart = this->Ctx->CountersSectionStart; + uint64_t CountersEnd = this->Ctx->CountersSectionEnd; + if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { + LLVM_DEBUG( + dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " + << FunctionName << "\n\tExpected: [0x" + << Twine::utohexstr(CountersStart) << ", 0x" + << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" + << Twine::utohexstr(*CounterPtr)); + LLVM_DEBUG(Die.dump(dbgs())); + return; + } + if (!FunctionPtr) { + LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName + << "\n"); + LLVM_DEBUG(Die.dump(dbgs())); + } + this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart, + FunctionPtr.getValueOr(0), *NumCounters); + }; + for (auto &CU : DICtx->normal_units()) + for (const auto &Entry : CU->dies()) + maybeAddProbe(DWARFDie(CU.get(), &Entry)); + for (auto &CU : DICtx->dwo_units()) + for (const auto &Entry : CU->dies()) + maybeAddProbe(DWARFDie(CU.get(), &Entry)); +} diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -52,16 +52,19 @@ } Expected> -InstrProfReader::create(const Twine &Path) { +InstrProfReader::create(const Twine &Path, + const InstrProfCorrelator *Correlator) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); if (Error E = BufferOrError.takeError()) return std::move(E); - return InstrProfReader::create(std::move(BufferOrError.get())); + return InstrProfReader::create(std::move(BufferOrError.get()), Correlator); } Expected> -InstrProfReader::create(std::unique_ptr Buffer) { +InstrProfReader::create(std::unique_ptr Buffer, + const InstrProfCorrelator *Correlator) { + // Sanity check the buffer. if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits::max()) return make_error(instrprof_error::too_large); @@ -73,9 +76,9 @@ if (IndexedInstrProfReader::hasFormat(*Buffer)) Result.reset(new IndexedInstrProfReader(std::move(Buffer))); else if (RawInstrProfReader64::hasFormat(*Buffer)) - Result.reset(new RawInstrProfReader64(std::move(Buffer))); + Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator)); else if (RawInstrProfReader32::hasFormat(*Buffer)) - Result.reset(new RawInstrProfReader32(std::move(Buffer))); + Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator)); else if (TextInstrProfReader::hasFormat(*Buffer)) Result.reset(new TextInstrProfReader(std::move(Buffer))); else @@ -352,7 +355,7 @@ template Error RawInstrProfReader::createSymtab(InstrProfSymtab &Symtab) { - if (Error E = Symtab.create(StringRef(NamesStart, NamesSize))) + if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart))) return error(std::move(E)); for (const RawInstrProf::ProfileData *I = Data; I != DataEnd; ++I) { const IntPtrT FPtr = swap(I->FunctionPointer); @@ -369,6 +372,10 @@ Version = swap(Header.Version); if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); + if (useDebugInfoCorrelate() && !Correlator) + return error(instrprof_error::missing_debug_info_for_correlation); + if (!useDebugInfoCorrelate() && Correlator) + return error(instrprof_error::unexpected_debug_info_for_correlation); BinaryIdsSize = swap(Header.BinaryIdsSize); if (BinaryIdsSize % sizeof(uint64_t)) @@ -380,7 +387,7 @@ auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); auto CountersSize = swap(Header.CountersSize); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); - NamesSize = swap(Header.NamesSize); + auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData); @@ -398,15 +405,27 @@ if (Start + ValueDataOffset > DataBuffer->getBufferEnd()) return error(instrprof_error::bad_header); - Data = reinterpret_cast *>( - Start + DataOffset); - DataEnd = Data + DataSize; + if (Correlator) { + // These sizes in the raw file are zero because we constructed them in the + // Correlator. + assert(DataSize == 0 && NamesSize == 0); + assert(CountersDelta == 0 && NamesDelta == 0); + Data = Correlator->getDataPointer(); + DataEnd = Data + Correlator->getDataSize(); + NamesStart = Correlator->getCompressedNamesPointer(); + NamesEnd = NamesStart + Correlator->getCompressedNamesSize(); + } else { + Data = reinterpret_cast *>( + Start + DataOffset); + DataEnd = Data + DataSize; + NamesStart = Start + NamesOffset; + NamesEnd = NamesStart + NamesSize; + } // Binary ids start just after the header. BinaryIdsStart = reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = reinterpret_cast(Start + CountersOffset); - NamesStart = Start + NamesOffset; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -440,45 +459,50 @@ if (NumCounters == 0) return error(instrprof_error::malformed, "number of counters is zero"); - IntPtrT CounterPtr = Data->CounterPtr; - auto *NamesStartAsCounter = reinterpret_cast(NamesStart); - ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart; - - // Check bounds. Note that the counter pointer embedded in the data record - // may itself be corrupt. - if (MaxNumCounters < 0 || NumCounters > (uint32_t)MaxNumCounters) - return error(instrprof_error::malformed, - "counter pointer is out of bounds"); - - // We need to compute the in-buffer counter offset from the in-memory address - // distance. The initial CountersDelta is the in-memory address difference - // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr - - // CountersDelta computes the offset into the in-buffer counter section. - // - // CountersDelta decreases as we advance to the next data record. - ptrdiff_t CounterOffset = getCounterOffset(CounterPtr); - CountersDelta -= sizeof(*Data); - if (CounterOffset < 0) - return error( - instrprof_error::malformed, - ("counter offset " + Twine(CounterOffset) + " is negative").str()); - - if (CounterOffset > MaxNumCounters) - return error(instrprof_error::malformed, - ("counter offset " + Twine(CounterOffset) + - " is greater than the maximum number of counters " + - Twine((uint32_t)MaxNumCounters)) - .str()); - - if (((uint32_t)CounterOffset + NumCounters) > (uint32_t)MaxNumCounters) - return error(instrprof_error::malformed, - ("number of counters " + - Twine(((uint32_t)CounterOffset + NumCounters)) + - " is greater than the maximum number of counters " + - Twine((uint32_t)MaxNumCounters)) - .str()); - - auto RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters); + ArrayRef RawCounts; + if (Correlator) { + uint64_t CounterOffset = swap(Data->CounterPtr) / sizeof(uint64_t); + RawCounts = + makeArrayRef(CountersStart + CounterOffset, NumCounters); + } else { + IntPtrT CounterPtr = Data->CounterPtr; + ptrdiff_t CounterOffset = getCounterOffset(CounterPtr); + if (CounterOffset < 0) + return error( + instrprof_error::malformed, + ("counter offset " + Twine(CounterOffset) + " is negative").str()); + + // Check bounds. Note that the counter pointer embedded in the data record + // may itself be corrupt. + auto *NamesStartAsCounter = reinterpret_cast(NamesStart); + ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart; + if (MaxNumCounters < 0 || NumCounters > (uint32_t)MaxNumCounters) + return error(instrprof_error::malformed, + "counter pointer is out of bounds"); + // We need to compute the in-buffer counter offset from the in-memory + // address distance. The initial CountersDelta is the in-memory address + // difference start(__llvm_prf_cnts)-start(__llvm_prf_data), so + // SrcData->CounterPtr - CountersDelta computes the offset into the + // in-buffer counter section. + if (CounterOffset > MaxNumCounters) + return error(instrprof_error::malformed, + ("counter offset " + Twine(CounterOffset) + + " is greater than the maximum number of counters " + + Twine((uint32_t)MaxNumCounters)) + .str()); + + if (((uint32_t)CounterOffset + NumCounters) > (uint32_t)MaxNumCounters) + return error(instrprof_error::malformed, + ("number of counters " + + Twine(((uint32_t)CounterOffset + NumCounters)) + + " is greater than the maximum number of counters " + + Twine((uint32_t)MaxNumCounters)) + .str()); + // CountersDelta decreases as we advance to the next data record. + CountersDelta -= sizeof(*Data); + + RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters); + } if (ShouldSwapBytes) { Record.Counts.clear(); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -42,6 +42,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" @@ -941,15 +942,15 @@ if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { - MDString::get(Ctx, "Function Name"), + MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName), MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)), }; Metadata *CFGHashAnnotation[] = { - MDString::get(Ctx, "CFG Hash"), + MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName), ConstantAsMetadata::get(Inc->getHash()), }; Metadata *NumCountersAnnotation[] = { - MDString::get(Ctx, "Num Counters"), + MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName), ConstantAsMetadata::get(Inc->getNumCounters()), }; auto Annotations = DB.getOrCreateArray({ diff --git a/llvm/tools/llvm-profdata/CMakeLists.txt b/llvm/tools/llvm-profdata/CMakeLists.txt --- a/llvm/tools/llvm-profdata/CMakeLists.txt +++ b/llvm/tools/llvm-profdata/CMakeLists.txt @@ -1,6 +1,8 @@ set(LLVM_LINK_COMPONENTS Core ProfileData + Object + DebugInfoDWARF Support ) diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -13,7 +13,10 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/Object/Binary.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/ProfileData/InstrProfWriter.h" #include "llvm/ProfileData/ProfileCommon.h" @@ -233,6 +236,7 @@ /// Load an input into a writer context. static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, + const InstrProfCorrelator *Correlator, WriterContext *WC) { std::unique_lock CtxGuard{WC->Lock}; @@ -241,7 +245,7 @@ // invalid outside of this packaged task. std::string Filename = Input.Filename; - auto ReaderOrErr = InstrProfReader::create(Input.Filename); + auto ReaderOrErr = InstrProfReader::create(Input.Filename, Correlator); if (Error E = ReaderOrErr.takeError()) { // Skip the empty profiles by returning sliently. instrprof_error IPE = InstrProfError::take(std::move(E)); @@ -325,6 +329,7 @@ } static void mergeInstrProfile(const WeightedFileVector &Inputs, + StringRef DebugInfoFilename, SymbolRemapper *Remapper, StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse, @@ -333,6 +338,15 @@ OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text) exitWithError("unknown format is specified"); + std::unique_ptr Correlator; + if (!DebugInfoFilename.empty()) { + if (auto Err = + InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator)) + exitWithError(std::move(Err), DebugInfoFilename); + if (auto Err = Correlator->correlateProfileData()) + exitWithError(std::move(Err), DebugInfoFilename); + } + std::mutex ErrorLock; SmallSet WriterErrorCodes; @@ -352,14 +366,15 @@ if (NumThreads == 1) { for (const auto &Input : Inputs) - loadInput(Input, Remapper, Contexts[0].get()); + loadInput(Input, Remapper, Correlator.get(), Contexts[0].get()); } else { ThreadPool Pool(hardware_concurrency(NumThreads)); // Load the inputs in parallel (N/NumThreads serial steps). unsigned Ctx = 0; for (const auto &Input : Inputs) { - Pool.async(loadInput, Input, Remapper, Contexts[Ctx].get()); + Pool.async(loadInput, Input, Remapper, Correlator.get(), + Contexts[Ctx].get()); Ctx = (Ctx + 1) % NumThreads; } Pool.wait(); @@ -575,7 +590,7 @@ SmallSet WriterErrorCodes; auto WC = std::make_unique(OutputSparse, ErrorLock, WriterErrorCodes); - loadInput(Inputs[0], nullptr, WC.get()); + loadInput(Inputs[0], nullptr, nullptr, WC.get()); if (WC->Errors.size() > 0) exitWithError(std::move(WC->Errors[0].first), InstrFilename); @@ -952,6 +967,9 @@ cl::opt GenCSNestedProfile( "gen-cs-nested-profile", cl::Hidden, cl::init(false), cl::desc("Generate nested function profiles for CSSPGO")); + cl::opt DebugInfoFilename( + "debug-info", cl::init(""), cl::Hidden, + cl::desc("Use the provided debug info to correlate the raw profile.")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); @@ -992,8 +1010,9 @@ } if (ProfileKind == instr) - mergeInstrProfile(WeightedInputs, Remapper.get(), OutputFilename, - OutputFormat, OutputSparse, NumThreads, FailureMode); + mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), + OutputFilename, OutputFormat, OutputSparse, NumThreads, + FailureMode); else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, ProfileSymbolListFile, CompressAllSections, @@ -1024,7 +1043,7 @@ OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; exit(0); } - loadInput(WeightedInput, nullptr, &Context); + loadInput(WeightedInput, nullptr, nullptr, &Context); overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, IsCS); Overlap.dump(OS);