diff --git a/compiler-rt/test/profile/Linux/binary-id-lookup.c b/compiler-rt/test/profile/Linux/binary-id-lookup.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/profile/Linux/binary-id-lookup.c @@ -0,0 +1,26 @@ +// REQUIRES: linux +// RUN: split-file %s %t +// RUN: %clang_profgen -Wl,--build-id=0x12345678 -fcoverage-mapping -O2 -shared %t/foo.c -o %t/libfoo.so +// RUN: %clang_profgen -Wl,--build-id=0xabcd1234 -fcoverage-mapping -O2 %t/main.c -L%t -lfoo -o %t.main +// RUN: env LLVM_PROFILE_FILE=%t.profraw LD_LIBRARY_PATH=%t %run %t.main +// RUN: mkdir -p %t/.build-id/12 %t/.build-id/ab +// RUN: cp %t/libfoo.so %t/.build-id/12/345678.debug +// RUN: echo "bad" > %t/.build-id/ab/cd1234.debug +// RUN: llvm-profdata merge -o %t.profdata %t.profraw +// RUN: llvm-cov show -instr-profile %t.profdata -debug-file-directory %t %t.main | FileCheck %s + +// CHECK: foo +// CHECK: bar +// CHECK: main + +//--- foo.c +void foo(void) {} + +//--- main.c +void foo(void); +void bar(void) {} +int main() { + foo(); + bar(); + return 0; +} diff --git a/llvm/docs/CommandGuide/llvm-cov.rst b/llvm/docs/CommandGuide/llvm-cov.rst --- a/llvm/docs/CommandGuide/llvm-cov.rst +++ b/llvm/docs/CommandGuide/llvm-cov.rst @@ -349,6 +349,18 @@ coverage >= high, red when coverage < low, and yellow otherwise. Both high and low should be between 0-100 and high > low. +.. option:: -debuginfod + +Attempt to look up coverage mapping from objects using debuginfod. This is +attempted by default for binary IDs present in the profile but not provided on +the command line, so long as debuginfod is compiled in and configured via +DEBUGINFOD_URLS. + +.. option:: -debug-file-directory= + +Provides a directory to search for objects corresponding to binary IDs in the +profile. + .. program:: llvm-cov report .. _llvm-cov-report: @@ -418,6 +430,18 @@ when binaries have been compiled with one of `-fcoverage-prefix-map` `-fcoverage-compilation-dir`, or `-ffile-compilation-dir`. +.. option:: -debuginfod + +Attempt to look up coverage mapping from objects using debuginfod. This is +attempted by default for binary IDs present in the profile but not provided on +the command line, so long as debuginfod is compiled in and configured via +DEBUGINFOD_URLS. + +.. option:: -debug-file-directory= + +Provides a directory to search for objects corresponding to binary IDs in the +profile. + .. program:: llvm-cov export .. _llvm-cov-export: @@ -492,3 +516,15 @@ Directory used as a base for relative coverage mapping paths. Only applicable when binaries have been compiled with one of `-fcoverage-prefix-map` `-fcoverage-compilation-dir`, or `-ffile-compilation-dir`. + +.. option:: -debuginfod + +Attempt to look up coverage mapping from objects using debuginfod. This is +attempted by default for binary IDs present in the profile but not provided on +the command line, so long as debuginfod is compiled in and configured via +DEBUGINFOD_URLS. + +.. option:: -debug-file-directory= + +Provides a directory to search for objects corresponding to binary IDs in the +profile. diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h --- a/llvm/include/llvm/Debuginfod/Debuginfod.h +++ b/llvm/include/llvm/Debuginfod/Debuginfod.h @@ -38,9 +38,13 @@ namespace llvm { +/// Returns false if a debuginfod lookup can be determined to have no chance of +/// succeeding. +bool canUseDebuginfod(); + /// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS /// environment variable. -Expected> getDefaultDebuginfodUrls(); +SmallVector getDefaultDebuginfodUrls(); /// Finds a default local file caching directory for the debuginfod client, /// first checking DEBUGINFOD_CACHE_PATH. diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -22,6 +22,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" @@ -43,6 +44,10 @@ class IndexedInstrProfReader; +namespace object { +class BuildIDFetcher; +} // namespace object + namespace coverage { class CoverageMappingReader; @@ -580,6 +585,13 @@ ArrayRef> CoverageReaders, IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage); + // Load coverage records from file. + static Error + loadFromFile(StringRef Filename, StringRef Arch, StringRef CompilationDir, + IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage, + bool &DataFound, + SmallVectorImpl *FoundBinaryIDs = nullptr); + /// Add a function record corresponding to \p Record. Error loadFunctionRecord(const CoverageMappingRecord &Record, IndexedInstrProfReader &ProfileReader); @@ -605,7 +617,8 @@ /// Ignores non-instrumented object files unless all are not instrumented. static Expected> load(ArrayRef ObjectFilenames, StringRef ProfileFilename, - ArrayRef Arches = None, StringRef CompilationDir = ""); + ArrayRef Arches = None, StringRef CompilationDir = "", + const object::BuildIDFetcher* BIDFetcher = nullptr); /// The number of functions that couldn't have their profiles mapped. /// diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -205,7 +205,8 @@ static Expected>> create(MemoryBufferRef ObjectBuffer, StringRef Arch, SmallVectorImpl> &ObjectFileBuffers, - StringRef CompilationDir = ""); + StringRef CompilationDir = "", + SmallVectorImpl *BinaryIDs = nullptr); static Expected> createCoverageReaderFromBuffer(StringRef Coverage, diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp --- a/llvm/lib/Debuginfod/Debuginfod.cpp +++ b/llvm/lib/Debuginfod/Debuginfod.cpp @@ -53,7 +53,11 @@ return llvm::toHex(ID, /*LowerCase=*/true); } -Expected> getDefaultDebuginfodUrls() { +bool canUseDebuginfod() { + return HTTPClient::isAvailable() && !getDefaultDebuginfodUrls().empty(); +} + +SmallVector getDefaultDebuginfodUrls() { const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); if (DebuginfodUrlsEnv == nullptr) return SmallVector(); @@ -124,13 +128,8 @@ return CacheDirOrErr.takeError(); CacheDir = *CacheDirOrErr; - Expected> DebuginfodUrlsOrErr = - getDefaultDebuginfodUrls(); - if (!DebuginfodUrlsOrErr) - return DebuginfodUrlsOrErr.takeError(); - SmallVector &DebuginfodUrls = *DebuginfodUrlsOrErr; return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir, - DebuginfodUrls, + getDefaultDebuginfodUrls(), getDefaultDebuginfodTimeout()); } diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp --- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/Coverage/CoverageMappingReader.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/Debug.h" @@ -343,10 +344,49 @@ }); } +Error CoverageMapping::loadFromFile( + StringRef Filename, StringRef Arch, StringRef CompilationDir, + IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage, + bool &DataFound, SmallVectorImpl *FoundBinaryIDs) { + auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN( + Filename, /*IsText=*/false, /*RequiresNullTerminator=*/false); + if (std::error_code EC = CovMappingBufOrErr.getError()) + return createFileError(Filename, errorCodeToError(EC)); + MemoryBufferRef CovMappingBufRef = + CovMappingBufOrErr.get()->getMemBufferRef(); + SmallVector, 4> Buffers; + + SmallVector BinaryIDs; + auto CoverageReadersOrErr = BinaryCoverageReader::create( + CovMappingBufRef, Arch, Buffers, CompilationDir, + FoundBinaryIDs ? &BinaryIDs : nullptr); + if (Error E = CoverageReadersOrErr.takeError()) { + E = handleMaybeNoDataFoundError(std::move(E)); + if (E) + return createFileError(Filename, std::move(E)); + return E; + } + + SmallVector, 4> Readers; + for (auto &Reader : CoverageReadersOrErr.get()) + Readers.push_back(std::move(Reader)); + if (FoundBinaryIDs && !Readers.empty()) { + llvm::append_range(*FoundBinaryIDs, + llvm::map_range(BinaryIDs, [](object::BuildIDRef BID) { + return object::BuildID(BID); + })); + } + DataFound |= !Readers.empty(); + if (Error E = loadFromReaders(Readers, ProfileReader, Coverage)) + return createFileError(Filename, std::move(E)); + return Error::success(); +} + Expected> CoverageMapping::load(ArrayRef ObjectFilenames, StringRef ProfileFilename, ArrayRef Arches, - StringRef CompilationDir) { + StringRef CompilationDir, + const object::BuildIDFetcher* BIDFetcher) { auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename); if (Error E = ProfileReaderOrErr.takeError()) return createFileError(ProfileFilename, std::move(E)); @@ -354,32 +394,55 @@ auto Coverage = std::unique_ptr(new CoverageMapping()); bool DataFound = false; + auto GetArch = [&](size_t Idx) { + if (Arches.empty()) + return StringRef(); + if (Arches.size() == 1) + return Arches.front(); + return Arches[Idx]; + }; + + SmallVector FoundBinaryIDs; for (const auto &File : llvm::enumerate(ObjectFilenames)) { - auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN( - File.value(), /*IsText=*/false, /*RequiresNullTerminator=*/false); - if (std::error_code EC = CovMappingBufOrErr.getError()) - return createFileError(File.value(), errorCodeToError(EC)); - StringRef Arch = Arches.empty() ? StringRef() : Arches[File.index()]; - MemoryBufferRef CovMappingBufRef = - CovMappingBufOrErr.get()->getMemBufferRef(); - SmallVector, 4> Buffers; - auto CoverageReadersOrErr = BinaryCoverageReader::create( - CovMappingBufRef, Arch, Buffers, CompilationDir); - if (Error E = CoverageReadersOrErr.takeError()) { - E = handleMaybeNoDataFoundError(std::move(E)); - if (E) - return createFileError(File.value(), std::move(E)); - // E == success (originally a no_data_found error). - continue; + if (Error E = + loadFromFile(File.value(), GetArch(File.index()), CompilationDir, + *ProfileReader, *Coverage, DataFound, &FoundBinaryIDs)) + return E; + } + + if (BIDFetcher) { + const auto &Compare = [](object::BuildIDRef A, object::BuildIDRef B) { + return StringRef(reinterpret_cast(A.data()), A.size()) < + StringRef(reinterpret_cast(B.data()), B.size()); + }; + std::vector ProfileBinaryIDs; + if (Error E = ProfileReader->readBinaryIds(ProfileBinaryIDs)) + return createFileError(ProfileFilename, std::move(E)); + llvm::sort(ProfileBinaryIDs, Compare); + std::unique(ProfileBinaryIDs.begin(), ProfileBinaryIDs.end(), Compare); + + SmallVector BinaryIDsToFetch; + if (!ProfileBinaryIDs.empty()) { + llvm::sort(FoundBinaryIDs, Compare); + std::unique(FoundBinaryIDs.begin(), FoundBinaryIDs.end(), Compare); + std::set_difference( + ProfileBinaryIDs.begin(), ProfileBinaryIDs.end(), + FoundBinaryIDs.begin(), FoundBinaryIDs.end(), + std::inserter(BinaryIDsToFetch, BinaryIDsToFetch.end()), Compare); } - SmallVector, 4> Readers; - for (auto &Reader : CoverageReadersOrErr.get()) - Readers.push_back(std::move(Reader)); - DataFound |= !Readers.empty(); - if (Error E = loadFromReaders(Readers, *ProfileReader, *Coverage)) - return createFileError(File.value(), std::move(E)); + for (object::BuildIDRef BinaryID : BinaryIDsToFetch) { + Optional PathOpt = BIDFetcher->fetch(BinaryID); + if (!PathOpt) + continue; + std::string Path = std::move(*PathOpt); + StringRef Arch = Arches.size() == 1 ? Arches.front() : StringRef(); + if (Error E = loadFromFile(Path, Arch, CompilationDir, *ProfileReader, + *Coverage, DataFound)) + return E; + } } + // If no readers were created, either no objects were provided or none of them // had coverage data. Return an error in the latter case. if (!DataFound && !ObjectFilenames.empty()) diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -954,7 +954,8 @@ static Expected> loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, - StringRef CompilationDir = "") { + StringRef CompilationDir = "", + Optional *BinaryID = nullptr) { std::unique_ptr OF; if (auto *Universal = dyn_cast(Bin.get())) { // If we have a universal binary, try to look up the object for the @@ -1052,6 +1053,9 @@ FuncRecords = std::move(WritableBuffer); } + if (BinaryID) + *BinaryID = getBuildID(OF.get()); + return BinaryCoverageReader::createCoverageReaderFromBuffer( CoverageMapping, std::move(FuncRecords), std::move(ProfileNames), BytesInAddress, Endian, CompilationDir); @@ -1074,7 +1078,8 @@ BinaryCoverageReader::create( MemoryBufferRef ObjectBuffer, StringRef Arch, SmallVectorImpl> &ObjectFileBuffers, - StringRef CompilationDir) { + StringRef CompilationDir, + SmallVectorImpl *BinaryIDs) { std::vector> Readers; if (ObjectBuffer.getBuffer().startswith(TestingFormatMagic)) { @@ -1114,7 +1119,7 @@ return BinaryCoverageReader::create( ArchiveOrErr.get()->getMemoryBufferRef(), Arch, ObjectFileBuffers, - CompilationDir); + CompilationDir, BinaryIDs); } } @@ -1127,7 +1132,8 @@ return ChildBufOrErr.takeError(); auto ChildReadersOrErr = BinaryCoverageReader::create( - ChildBufOrErr.get(), Arch, ObjectFileBuffers, CompilationDir); + ChildBufOrErr.get(), Arch, ObjectFileBuffers, CompilationDir, + BinaryIDs); if (!ChildReadersOrErr) return ChildReadersOrErr.takeError(); for (auto &Reader : ChildReadersOrErr.get()) @@ -1146,10 +1152,14 @@ return std::move(Readers); } - auto ReaderOrErr = loadBinaryFormat(std::move(Bin), Arch, CompilationDir); + Optional BinaryID; + auto ReaderOrErr = loadBinaryFormat(std::move(Bin), Arch, CompilationDir, + BinaryIDs ? &BinaryID : nullptr); if (!ReaderOrErr) return ReaderOrErr.takeError(); Readers.push_back(std::move(ReaderOrErr.get())); + if (BinaryID) + BinaryIDs->push_back(*BinaryID); return std::move(Readers); } diff --git a/llvm/tools/llvm-cov/CMakeLists.txt b/llvm/tools/llvm-cov/CMakeLists.txt --- a/llvm/tools/llvm-cov/CMakeLists.txt +++ b/llvm/tools/llvm-cov/CMakeLists.txt @@ -14,3 +14,5 @@ SourceCoverageViewText.cpp TestingSupport.cpp ) + +target_link_libraries(llvm-cov PRIVATE LLVMDebuginfod) diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp --- a/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -23,6 +23,10 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" +#include "llvm/Debuginfod/BuildIDFetcher.h" +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Debuginfod/HTTPClient.h" +#include "llvm/Object/BuildID.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/CommandLine.h" @@ -178,6 +182,8 @@ /// Allowlist from -name-allowlist to be used for filtering. std::unique_ptr NameAllowlist; + + std::unique_ptr BIDFetcher; }; } @@ -434,7 +440,7 @@ ObjectFilename); auto CoverageOrErr = CoverageMapping::load(ObjectFilenames, PGOFilename, CoverageArches, - ViewOpts.CompilationDirectory); + ViewOpts.CompilationDirectory, BIDFetcher.get()); if (Error E = CoverageOrErr.takeError()) { error("Failed to load coverage: " + toString(std::move(E))); return nullptr; @@ -643,6 +649,14 @@ cl::opt DebugDump("dump", cl::Optional, cl::desc("Show internal debug dump")); + cl::list DebugFileDirectory( + "debug-file-directory", cl::Optional, + cl::desc("Directories to search for object files by build ID")); + cl::opt Debuginfod( + "debuginfod", cl::Optional, + cl::desc("Use debuginfod to look up object files from profile"), + cl::init(canUseDebuginfod())); + cl::opt Format( "format", cl::desc("Output format for line-based coverage reports"), cl::values(clEnumValN(CoverageViewOptions::OutputFormat::Text, "text", @@ -745,12 +759,16 @@ auto commandLineParser = [&, this](int argc, const char **argv) -> int { cl::ParseCommandLineOptions(argc, argv, "LLVM code coverage tool\n"); ViewOpts.Debug = DebugDump; + if (Debuginfod) + BIDFetcher = std::make_unique(DebugFileDirectory); + else + BIDFetcher = std::make_unique(DebugFileDirectory); if (!CovFilename.empty()) ObjectFilenames.emplace_back(CovFilename); for (const std::string &Filename : CovFilenames) ObjectFilenames.emplace_back(Filename); - if (ObjectFilenames.empty()) { + if (ObjectFilenames.empty() && !Debuginfod && DebugFileDirectory.empty()) { errs() << "No filenames specified!\n"; ::exit(1); } @@ -863,10 +881,8 @@ } CoverageArches.emplace_back(Arch); } - if (CoverageArches.size() == 1) - CoverageArches.insert(CoverageArches.end(), ObjectFilenames.size() - 1, - CoverageArches[0]); - if (CoverageArches.size() != ObjectFilenames.size()) { + if (CoverageArches.size() != 1 && + CoverageArches.size() != ObjectFilenames.size()) { error("Number of architectures doesn't match the number of objects"); return 1; } diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -3176,9 +3176,7 @@ // Initialize debuginfod. const bool ShouldUseDebuginfodByDefault = - InputArgs.hasArg(OBJDUMP_build_id) || - (HTTPClient::isAvailable() && - !ExitOnErr(getDefaultDebuginfodUrls()).empty()); + InputArgs.hasArg(OBJDUMP_build_id) || canUseDebuginfod(); std::vector DebugFileDirectories = InputArgs.getAllArgValues(OBJDUMP_debug_file_directory); if (InputArgs.hasFlag(OBJDUMP_debuginfod, OBJDUMP_no_debuginfod, diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp --- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp +++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp @@ -440,13 +440,7 @@ LLVMSymbolizer Symbolizer(Opts); - // A debuginfod lookup could succeed if a HTTP client is available and at - // least one backing URL is configured. - bool ShouldUseDebuginfodByDefault = - HTTPClient::isAvailable() && - !ExitOnErr(getDefaultDebuginfodUrls()).empty(); - if (Args.hasFlag(OPT_debuginfod, OPT_no_debuginfod, - ShouldUseDebuginfodByDefault)) + if (Args.hasFlag(OPT_debuginfod, OPT_no_debuginfod, canUseDebuginfod())) enableDebuginfod(Symbolizer, Args); if (Args.hasArg(OPT_filter_markup)) {