Index: llvm/include/llvm/Debuginfod/Debuginfod.h =================================================================== --- /dev/null +++ llvm/include/llvm/Debuginfod/Debuginfod.h @@ -0,0 +1,44 @@ +//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the debuginfod::fetchInfo +/// function and the debuginfod::AssetType enum class +/// +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DEBUGINFOD_H +#define LLVM_SUPPORT_DEBUGINFOD_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +/// DebuginfodAsset - This enum class specifies the types of +/// debugging information that can be requested from a debuginfod +/// server. +enum class DebuginfodAssetType { Executable, Debuginfo, Source }; + +/// Fetch a debuginfod asset to a file in a local cache and return the cached +/// file path. First queries the local cache in the given directory path, +/// followed by the debuginfod servers at the given urls for the specified +/// type of information about the given build ID. Source files are specified +/// by their absolute path provided in the Description. A callback function +/// may optionally be provided for further processing of the fetched asset. +Expected> fetchDebuginfo( + StringRef CacheDirectoryPath, ArrayRef DebuginfodUrls, + StringRef BuildID, DebuginfodAssetType Type, StringRef Description = "", + std::function)> AddBuffer = + [](size_t Task, std::unique_ptr MB) {}); + +} // end namespace llvm + +#endif Index: llvm/lib/CMakeLists.txt =================================================================== --- llvm/lib/CMakeLists.txt +++ llvm/lib/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(Option) add_subdirectory(Remarks) add_subdirectory(DebugInfo) +add_subdirectory(Debuginfod) add_subdirectory(DWP) add_subdirectory(ExecutionEngine) add_subdirectory(Target) Index: llvm/lib/Debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/lib/Debuginfod/CMakeLists.txt @@ -0,0 +1,11 @@ +if(LLVM_ENABLE_CURL) + add_llvm_component_library(LLVMDebuginfod + Debuginfod.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod + + LINK_COMPONENTS + Support + ) +endif() Index: llvm/lib/Debuginfod/Debuginfod.cpp =================================================================== --- /dev/null +++ llvm/lib/Debuginfod/Debuginfod.cpp @@ -0,0 +1,107 @@ +//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the fetchInfo function, which retrieves +/// any of the three supported asset types: (executable, debuginfo, source file) +/// associated with a build-id from debuginfod servers. If a source file is to +/// be fetched, its absolute path must be specified in the Description argument +/// to fetchInfo. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CURLClient.h" +#include "llvm/Support/CachePruning.h" +#include "llvm/Support/Caching.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/xxhash.h" + +using namespace llvm; + +#define DEBUG_TYPE "DEBUGINFOD" + +Expected> llvm::fetchDebuginfo( + StringRef CacheDirectoryPath, ArrayRef DebuginfodUrls, + StringRef BuildID, DebuginfodAssetType Type, StringRef Description, + std::function)> AddBuffer) { + LLVM_DEBUG(dbgs() << "fetching info, debuginfod urls size = " + << DebuginfodUrls.size() << "\n";); + std::string Suffix; + switch (Type) { + case DebuginfodAssetType::Executable: + Suffix = "executable"; + break; + case DebuginfodAssetType::Debuginfo: + Suffix = "debuginfo"; + break; + case DebuginfodAssetType::Source: + // Description is the absolute source path + Suffix = "source" + Description.str(); + break; + } + + std::string UniqueKey = utostr(xxHash64((BuildID + Suffix).str())); + + Expected CacheOrErr = localCache( + "Debuginfod-client", ".debuginfod-client", CacheDirectoryPath, AddBuffer); + if (Error Err = CacheOrErr.takeError()) + return Err; + + NativeObjectCache &Cache = *CacheOrErr; + + // We choose an arbitrary Task parameter as we do not make use of it. + unsigned Task = 0; + AddStreamFn CacheAddStream = Cache(Task, UniqueKey); + + SmallString<64> AbsCachedAssetPath; + sys::path::append(AbsCachedAssetPath, CacheDirectoryPath, + "llvmcache-" + UniqueKey); + + if (!CacheAddStream) { + LLVM_DEBUG(dbgs() << "cache hit\n";); + return AbsCachedAssetPath; + } + + LLVM_DEBUG(dbgs() << "cache miss, UniqueKey = " << UniqueKey << "\n";); + + // The asset was not found in the local cache, so we must query + // the debuginfod servers. + for (const StringRef &ServerUrl : DebuginfodUrls) { + SmallString<64> AssetUrl; + sys::path::append(AssetUrl, ServerUrl, "buildid", BuildID, Suffix); + + BufferedHTTPResponseHandler Handler; + CurlHTTPRequest Request(&Handler); + HTTPRequestConfig Config; + Config.Url = AssetUrl.str(); + if (Error Err = Request.setConfiguration(Config)) + return Err; + if (Error Err = Request.performRequest()) + return Err; + HTTPResponseBuffer &Resp = Handler.ResponseBuffer; + if (Resp.Code != 200) + continue; + + // We have retrieved the asset from this server, + // and now add it to the file cache. + auto Stream = CacheAddStream(Task); + assert(Resp.Body && "Unallocated MemoryBuffer in HTTPResponseBuffer."); + *Stream->OS << StringRef(Resp.Body->getBufferStart(), + Resp.Body->getBufferSize()); + + // Return the path to the asset on disk. + return AbsCachedAssetPath; + } + + return createStringError(errc::argument_out_of_domain, "build id not found"); +} Index: llvm/test/CMakeLists.txt =================================================================== --- llvm/test/CMakeLists.txt +++ llvm/test/CMakeLists.txt @@ -6,6 +6,7 @@ LLVM_ENABLE_DIA_SDK LLVM_ENABLE_FFI LLVM_ENABLE_THREADS + LLVM_ENABLE_CURL LLVM_ENABLE_ZLIB LLVM_ENABLE_LIBXML2 LLVM_INCLUDE_GO_TESTS Index: llvm/test/lit.cfg.py =================================================================== --- llvm/test/lit.cfg.py +++ llvm/test/lit.cfg.py @@ -158,9 +158,9 @@ tools.extend([ 'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as', 'llvm-addr2line', 'llvm-bcanalyzer', 'llvm-bitcode-strip', 'llvm-config', - 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', - 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis', 'llvm-extract', - 'llvm-isel-fuzzer', 'llvm-ifs', + 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find', + 'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis', + 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-ifs', 'llvm-install-name-tool', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib', 'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca', 'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump', 'llvm-otool', Index: llvm/test/lit.site.cfg.py.in =================================================================== --- llvm/test/lit.site.cfg.py.in +++ llvm/test/lit.site.cfg.py.in @@ -37,6 +37,7 @@ config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@ config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.have_zlib = @LLVM_ENABLE_ZLIB@ +config.enable_debuginfod_client = @LLVM_ENABLE_CURL@ config.have_libxar = @LLVM_HAVE_LIBXAR@ config.have_libxml2 = @LLVM_ENABLE_LIBXML2@ config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@ Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo @@ -0,0 +1 @@ +fake_debuginfo Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable @@ -0,0 +1 @@ +fake_executable Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c @@ -0,0 +1 @@ +int foo = 0; Index: llvm/test/tools/llvm-debuginfod/debuginfod-find.py =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/debuginfod-find.py @@ -0,0 +1,63 @@ +# RUN: rm -rf %t && mkdir %t && python %S/debuginfod-find.py %S/Inputs llvm-debuginfod-find %t +import threading +import http.server +import functools +import subprocess +import os + + +test_assets = [ + (['--executable'], '4503653137150120245', 'fake_executable\n'), + (['--source=/directory/file.c'], '163445695080362705', + 'int foo = 0;\n'), + (['--debuginfo'], '9104678335595080419', 'fake_debuginfo\n') +] + +def test_tool(inputs_path, tool_path, cache_directory) -> int: + httpd = http.server.ThreadingHTTPServer( + ('',0), functools.partial( + http.server.SimpleHTTPRequestHandler, + directory=inputs_path)) + port = httpd.server_port + thread = threading.Thread(target=httpd.serve_forever) + + try: + thread.start() + common_args = [tool_path, 'fake_build_id', + '--cache-dir', cache_directory] + for args, xxhash, contents in test_assets: + process = subprocess.Popen( + common_args + args, env={**os.environ, + 'DEBUGINFOD_URLS': f'http://localhost:{port}'}) + if process.wait() != 0: + return 1 + fname = os.path.join(cache_directory, + f'llvmcache-{xxhash}') + print(fname) + if not os.path.exists(fname): + return 1 + with open(fname) as infile: + if infile.read() != contents: + return 1 + + finally: + httpd.shutdown() + thread.join() + + return 0 + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('inputs_path') + parser.add_argument('tool_path') + parser.add_argument('cache_directory') + args = parser.parse_args() + result = test_tool(args.inputs_path, args.tool_path, + args.cache_directory) + os._exit(result) + + +if __name__ == '__main__': + main() Index: llvm/test/tools/llvm-debuginfod/lit.local.cfg =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/lit.local.cfg @@ -0,0 +1 @@ +config.suffixes = ['.py'] Index: llvm/tools/llvm-debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/tools/llvm-debuginfod/CMakeLists.txt @@ -0,0 +1,12 @@ +if (LLVM_ENABLE_CURL) + set(LLVM_LINK_COMPONENTS + Debuginfod + Support + ) + add_llvm_tool(llvm-debuginfod-find + llvm-debuginfod-find.cpp + ) + if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(debuginfod-find llvm-debuginfod-find) + endif() +endif() Index: llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp =================================================================== --- /dev/null +++ llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp @@ -0,0 +1,118 @@ +//===-- llvm-debuginfod-find.cpp - Simple CLI for libdebuginfod-client ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the llvm-debuginfod-find tool. This tool +/// queries the debuginfod servers in the DEBUGINFOD_URLS environment +/// variable (delimited by space (" ")) for the executable, +/// debuginfo, or specified source file of the binary matching the +/// given build-id. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/config.h" +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/COM.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include + +#define DEBUG_TYPE "llvm-debuginfod-find" + +using namespace llvm; + +cl::opt InputBuildID(cl::Positional, cl::Required, + cl::desc(""), cl::init("-")); + +static cl::opt + FetchExecutable("executable", cl::init(false), + cl::desc("fetch the associated executable")); + +static cl::opt FetchDebuginfo("debuginfo", cl::init(false), + cl::desc("fetch associated debuginfo")); + +static cl::opt FetchSource("source", cl::init(""), + cl::desc("/filename")); + +static cl::opt CacheDir("cache-dir", cl::init(""), + cl::desc("Cache Directory"), + cl::value_desc("directory")); + +[[noreturn]] static void helpExit() { + errs() << "Must specify exactly one of --executable, " + "--source=/path/to/file, or --debuginfo."; + exit(1); +} + +ExitOnError ExitOnErr; + +int main(int argc, char **argv) { + InitLLVM X(argc, argv); + + cl::ParseCommandLineOptions(argc, argv); + + const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); + if (DebuginfodUrlsEnv == NULL) { + errs() << "Missing DEBUGINFOD_URLS environment variable.\n"; + return 1; + } + + SmallVector DebuginfodUrls; + StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " "); + + SmallString<64> CacheDirectoryPath = StringRef(CacheDir); + if (CacheDirectoryPath.empty() && + !sys::path::cache_directory(CacheDirectoryPath)) { + errs() << "Unable to determine appropriate cache directory.\n"; + return 1; + } + + assert(CacheDirectoryPath.size() && "CacheDirectoryPath should be nonempty"); + + if (FetchExecutable + FetchDebuginfo + (FetchSource != "") != 1) + helpExit(); + + DebuginfodAssetType Type; + StringRef Description; + if (FetchExecutable) + Type = DebuginfodAssetType::Executable; + else if (FetchDebuginfo) + Type = DebuginfodAssetType::Debuginfo; + else if (FetchSource != "") { + Type = DebuginfodAssetType::Source; + Description = FetchSource; + } else + helpExit(); + + if (Type == DebuginfodAssetType::Source) { + // Print the contents of the source file + ExitOnErr(fetchDebuginfo(CacheDirectoryPath, DebuginfodUrls, InputBuildID, + Type, Description, + [](size_t Task, std::unique_ptr MB) { + outs() << MB->getBuffer(); + })); + } else { + // Print the path to the cached binary file on disk + outs() << ExitOnErr(fetchDebuginfo(CacheDirectoryPath, DebuginfodUrls, + InputBuildID, Type, Description)) + << "\n"; + } +} Index: llvm/unittests/CMakeLists.txt =================================================================== --- llvm/unittests/CMakeLists.txt +++ llvm/unittests/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory(Bitstream) add_subdirectory(CodeGen) add_subdirectory(DebugInfo) +add_subdirectory(Debuginfod) add_subdirectory(Demangle) add_subdirectory(ExecutionEngine) add_subdirectory(FileCheck) Index: llvm/unittests/Debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/unittests/Debuginfod/CMakeLists.txt @@ -0,0 +1,11 @@ +if (LLVM_ENABLE_CURL) + set(LLVM_LINK_COMPONENTS + Debuginfod + ) + + add_llvm_unittest(DebuginfodTests + DebuginfodTests.cpp + ) + + target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport) +endif() Index: llvm/unittests/Debuginfod/DebuginfodTests.cpp =================================================================== --- /dev/null +++ llvm/unittests/Debuginfod/DebuginfodTests.cpp @@ -0,0 +1,18 @@ +//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +TEST(DebuginfodTests, noDebuginfodUrlsFetchInfoTest) { + EXPECT_THAT_EXPECTED(fetchDebuginfo("./", {}, "fakeBuildId", + llvm::DebuginfodAssetType::Executable, + ""), + llvm::Failed()); +}