Index: llvm/CMakeLists.txt =================================================================== --- llvm/CMakeLists.txt +++ llvm/CMakeLists.txt @@ -377,6 +377,8 @@ set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON") +set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl HTTP client if available. Can be ON, OFF, or FORCE_ON") + set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.") option(LLVM_ENABLE_Z3_SOLVER Index: llvm/include/llvm/Debuginfod/Debuginfod.h =================================================================== --- /dev/null +++ llvm/include/llvm/Debuginfod/Debuginfod.h @@ -0,0 +1,44 @@ +//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the debuginfod::fetchInfo +/// function and the debuginfod::AssetType enum class +/// +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DEBUGINFOD_H +#define LLVM_SUPPORT_DEBUGINFOD_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +/// DebuginfodAsset - This enum class specifies the types of +/// debugging information that can be requested from a debuginfod +/// server. +enum class DebuginfodAssetType { Executable, Debuginfo, Source }; + +/// Fetch a debuginfod asset to a file in a local cache and return the cached +/// file path. First queries the local cache in the given directory path, +/// followed by the debuginfod servers at the given urls for the specified +/// type of information about the given build ID. Source files are specified +/// by their absolute path provided in the Description. A callback function +/// may optionally be provided for further processing of the fetched asset. +Expected> fetchDebuginfo( + StringRef CacheDirectoryPath, ArrayRef DebuginfodUrls, + StringRef BuildID, DebuginfodAssetType Type, StringRef Description = "", + std::function)> AddBuffer = + [](size_t Task, std::unique_ptr MB) {}); + +} // end namespace llvm + +#endif Index: llvm/include/llvm/Support/CURLClient.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/CURLClient.h @@ -0,0 +1,50 @@ +//===-- llvm/Support/CURLClient.h - CURL client library ---*- C++ -------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the CurlHTTPRequest class. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CURL_CLIENT_H +#define LLVM_SUPPORT_CURL_CLIENT_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +class CurlHTTPRequest : public HTTPRequest { + template void curlSetOpt(CURLoption Option, T Parameter) { + curl_easy_setopt(this->Curl, Option, Parameter); + } + void curlCleanup(); + +protected: + CURL *Curl = nullptr; + virtual Error curlInit(); + +public: + Error ErrorState = Error::success(); + + size_t curlHandleHeaderLine(char *Contents, size_t Size, size_t NMemb, + void * /*Unused*/ Ptr); + size_t curlHandleBodyChunk(char *Contents, size_t Size, size_t NMemb, + void * /*Unused*/ Ptr); + + using HTTPRequest::HTTPRequest; + ~CurlHTTPRequest(); + HTTPRequestConfig Config; + Error setConfiguration(HTTPRequestConfig Config) override; + virtual Error performRequest() override; +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_CURL_CLIENT_H Index: llvm/include/llvm/Support/HTTPClient.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/HTTPClient.h @@ -0,0 +1,71 @@ +//===-- llvm/Support/HTTPClient.h - HTTP client library ---*- C++ -------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the HTTPRequest, HTTPMethod, +/// HTTPResponseHandler, and BufferedHTTPResponseHandler classes, as well as +/// the HTTPResponseBuffer and HTTPRequestConfig structs. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HTTP_CLIENT_H +#define LLVM_SUPPORT_HTTP_CLIENT_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include + +namespace llvm { + +enum class HTTPMethod { GET }; + +class HTTPResponseHandler { +public: + virtual ~HTTPResponseHandler(){}; + virtual Expected handleHeaderLine(StringRef HeaderLine) = 0; + virtual Expected handleBodyChunk(StringRef BodyChunk) = 0; + virtual void handleResponseCode(unsigned Code) = 0; +}; + +struct HTTPResponseBuffer { + unsigned Code = 0; + std::unique_ptr Body; +}; + +class BufferedHTTPResponseHandler : public HTTPResponseHandler { + size_t BufferOffset = 0; + +public: + HTTPResponseBuffer ResponseBuffer; + Expected handleHeaderLine(StringRef HeaderLine) override; + Expected handleBodyChunk(StringRef BodyChunk) override; + void handleResponseCode(unsigned Code) override; +}; + +struct HTTPRequestConfig { + SmallString<128> Url; + HTTPMethod Method = HTTPMethod::GET; + bool FollowRedirects = true; + bool operator==(const HTTPRequestConfig &O) const { + return Url == O.Url && Method == O.Method && + FollowRedirects == O.FollowRedirects; + } +}; + +class HTTPRequest { +public: + HTTPResponseHandler *Handler; + HTTPRequest(HTTPResponseHandler *Handler) : Handler(Handler){}; + virtual ~HTTPRequest(){}; + virtual Error setConfiguration(const HTTPRequestConfig Config) = 0; + virtual Error performRequest() = 0; +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_HTTP_CLIENT_H Index: llvm/lib/CMakeLists.txt =================================================================== --- llvm/lib/CMakeLists.txt +++ llvm/lib/CMakeLists.txt @@ -12,6 +12,7 @@ add_subdirectory(BinaryFormat) add_subdirectory(Bitcode) add_subdirectory(Bitstream) +add_subdirectory(Debuginfod) add_subdirectory(DWARFLinker) add_subdirectory(Extensions) add_subdirectory(Frontend) Index: llvm/lib/Debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/lib/Debuginfod/CMakeLists.txt @@ -0,0 +1,11 @@ +if(LLVM_ENABLE_CURL) + add_llvm_component_library(LLVMDebuginfod + Debuginfod.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod + + LINK_COMPONENTS + Support + ) +endif() Index: llvm/lib/Debuginfod/Debuginfod.cpp =================================================================== --- /dev/null +++ llvm/lib/Debuginfod/Debuginfod.cpp @@ -0,0 +1,107 @@ +//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the fetchInfo function, which retrieves +/// any of the three supported asset types: (executable, debuginfo, source file) +/// associated with a build-id from debuginfod servers. If a source file is to +/// be fetched, its absolute path must be specified in the Description argument +/// to fetchInfo. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CURLClient.h" +#include "llvm/Support/CachePruning.h" +#include "llvm/Support/Caching.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/xxhash.h" + +using namespace llvm; + +#define DEBUG_TYPE "DEBUGINFOD" + +Expected> llvm::fetchDebuginfo( + StringRef CacheDirectoryPath, ArrayRef DebuginfodUrls, + StringRef BuildID, DebuginfodAssetType Type, StringRef Description, + std::function)> AddBuffer) { + LLVM_DEBUG(dbgs() << "fetching info, debuginfod urls size = " + << DebuginfodUrls.size() << "\n";); + std::string Suffix; + switch (Type) { + case DebuginfodAssetType::Executable: + Suffix = "executable"; + break; + case DebuginfodAssetType::Debuginfo: + Suffix = "debuginfo"; + break; + case DebuginfodAssetType::Source: + // Description is the absolute source path + Suffix = "source" + Description.str(); + break; + } + + std::string UniqueKey = utostr(xxHash64((BuildID + Suffix).str())); + + Expected CacheOrErr = localCache( + "Debuginfod-client", ".debuginfod-client", CacheDirectoryPath, AddBuffer); + if (Error Err = CacheOrErr.takeError()) + return Err; + + NativeObjectCache &Cache = *CacheOrErr; + + // We choose an arbitrary Task parameter as we do not make use of it. + unsigned Task = 0; + AddStreamFn CacheAddStream = Cache(Task, UniqueKey); + + SmallString<64> AbsCachedAssetPath; + sys::path::append(AbsCachedAssetPath, CacheDirectoryPath, + "llvmcache-" + UniqueKey); + + if (!CacheAddStream) { + LLVM_DEBUG(dbgs() << "cache hit\n";); + return AbsCachedAssetPath; + } + + LLVM_DEBUG(dbgs() << "cache miss, UniqueKey = " << UniqueKey << "\n";); + + // The asset was not found in the local cache, so we must query + // the debuginfod servers. + for (const StringRef &ServerUrl : DebuginfodUrls) { + SmallString<64> AssetUrl; + sys::path::append(AssetUrl, ServerUrl, "buildid", BuildID, Suffix); + + BufferedHTTPResponseHandler Handler; + CurlHTTPRequest Request(&Handler); + HTTPRequestConfig Config; + Config.Url = AssetUrl.str(); + if (Error Err = Request.setConfiguration(Config)) + return Err; + if (Error Err = Request.performRequest()) + return Err; + HTTPResponseBuffer &Resp = Handler.ResponseBuffer; + if (Resp.Code != 200) + continue; + + // We have retrieved the asset from this server, + // and now add it to the file cache. + auto Stream = CacheAddStream(Task); + assert(Resp.Body && "Unallocated MemoryBuffer in HTTPResponseBuffer."); + *Stream->OS << StringRef(Resp.Body->getBufferStart(), + Resp.Body->getBufferSize()); + + // Return the path to the asset on disk. + return AbsCachedAssetPath; + } + + return createStringError(errc::argument_out_of_domain, "build id not found"); +} Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -74,6 +74,11 @@ set(system_libs ${system_libs} ${Z3_LIBRARIES}) endif() +# Link LibCURL if the user wants it +if (LLVM_ENABLE_CURL) + set(system_libs ${system_libs} ${CURL_LIBRARIES}) +endif() + # Override the C runtime allocator on Windows and embed it into LLVM tools & libraries if(LLVM_INTEGRATED_CRT_ALLOC) if (CMAKE_BUILD_TYPE AND NOT ${LLVM_USE_CRT_${uppercase_CMAKE_BUILD_TYPE}} MATCHES "^(MT|MTd)$") @@ -134,6 +139,7 @@ ConvertUTF.cpp ConvertUTFWrapper.cpp CrashRecoveryContext.cpp + CURLClient.cpp DataExtractor.cpp Debug.cpp DebugCounter.cpp @@ -155,6 +161,7 @@ GlobPattern.cpp GraphWriter.cpp Hashing.cpp + HTTPClient.cpp InitLLVM.cpp InstructionCost.cpp IntEqClasses.cpp Index: llvm/lib/Support/CURLClient.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/CURLClient.cpp @@ -0,0 +1,108 @@ +//===-- llvm/Support/CURLClient.cpp - CURL client library -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the CurlHTTPRequest class. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/CURLClient.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include + +using namespace llvm; + +#define DEBUG_TYPE "CURLClient" + +size_t CurlHTTPRequest::curlHandleHeaderLine(char *Contents, size_t Size, + size_t NMemb, + void * /*Unused*/ Ptr) { + assert(Size == 1 && "The Size passed by libCURL to CURLOPT_HEADERFUNCTION " + "should always be 1."); + Expected SizeOrErr = + Handler->handleHeaderLine(StringRef(Contents, NMemb)); + + if (Error Err = SizeOrErr.takeError()) { + ErrorState = joinErrors(std::move(ErrorState), std::move(Err)); + return 0; + } + return *SizeOrErr; +} + +size_t CurlHTTPRequest::curlHandleBodyChunk(char *Contents, size_t Size, + size_t NMemb, + void * /*Unused*/ Ptr) { + Expected SizeOrErr = + Handler->handleBodyChunk(StringRef(Contents, Size * NMemb)); + if (Error Err = SizeOrErr.takeError()) { + ErrorState = joinErrors(std::move(ErrorState), std::move(Err)); + return 0; + } + return *SizeOrErr; +} + +Error CurlHTTPRequest::curlInit() { + if (!Curl) + Curl = curl_easy_init(); + if (!Curl) + return createStringError(errc::io_error, "Error initializing curl."); + return std::move(ErrorState); +} + +void CurlHTTPRequest::curlCleanup() { curl_easy_cleanup(Curl); } + +CurlHTTPRequest::~CurlHTTPRequest() { curlCleanup(); } + +static size_t curlHeaderLineMethodHook(char *Contents, size_t Size, + size_t NMemb, CurlHTTPRequest *Request) { + return Request->curlHandleHeaderLine(Contents, Size, NMemb, nullptr); +} + +static size_t curlBodyChunkMethodHook(char *Contents, size_t Size, size_t NMemb, + CurlHTTPRequest *Request) { + return Request->curlHandleBodyChunk(Contents, Size, NMemb, nullptr); +} + +Error CurlHTTPRequest::setConfiguration(HTTPRequestConfig Config) { + this->Config = Config; + if (Config.Method != HTTPMethod::GET) + return createStringError(errc::invalid_argument, + "Unsupported CURL request method."); + if (Error Err = curlInit()) + return joinErrors(std::move(ErrorState), std::move(Err)); + + curlSetOpt(CURLOPT_URL, Config.Url.c_str()); + curlSetOpt(CURLOPT_FOLLOWLOCATION, Config.FollowRedirects); + curlSetOpt(CURLOPT_WRITEFUNCTION, curlBodyChunkMethodHook); + curlSetOpt(CURLOPT_HEADERFUNCTION, curlHeaderLineMethodHook); + curlSetOpt(CURLOPT_WRITEDATA, this); + curlSetOpt(CURLOPT_HEADERDATA, this); + return std::move(ErrorState); +} + +Error CurlHTTPRequest::performRequest() { + if (!Curl) + return createStringError(errc::inappropriate_io_control_operation, + "CURL is uninitialized."); + CURLcode CurlRes = curl_easy_perform(Curl); + if (CurlRes != CURLE_OK) + return joinErrors(std::move(ErrorState), + createStringError(errc::io_error, + "curl_easy_perform() failed: %s\n", + curl_easy_strerror(CurlRes))); + + unsigned Code; + curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &Code); + Handler->handleResponseCode(Code); + return std::move(ErrorState); +} Index: llvm/lib/Support/HTTPClient.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/HTTPClient.cpp @@ -0,0 +1,65 @@ +//===-- llvm/Support/HTTPClient.cpp - HTTP client library -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the BufferedHTTPResponseHandler class. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPClient.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "HTTPClient" + +static bool parseContentLengthHeader(StringRef LineRef, + unsigned long long &ContentLength) { + // Content-Length is a mandatory header, and the only one we handle. + return LineRef.consume_front("Content-Length: ") && + !getAsUnsignedInteger(LineRef.trim(), 10, ContentLength); +} + +Expected +BufferedHTTPResponseHandler::handleHeaderLine(StringRef HeaderLine) { + if (ResponseBuffer.Body) + return HeaderLine.size(); + + unsigned long long ContentLength; + if (parseContentLengthHeader(HeaderLine, ContentLength)) + ResponseBuffer.Body = + WritableMemoryBuffer::getNewUninitMemBuffer(ContentLength); + + return HeaderLine.size(); +} + +Expected +BufferedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) { + if (!ResponseBuffer.Body) + return createStringError(errc::io_error, + "Unallocated response buffer -- HTTP Body data " + "recieved before Content-Length Header."); + + if (BufferOffset + BodyChunk.size() > ResponseBuffer.Body->getBufferSize()) + return createStringError(errc::io_error, + "Content is larger than response buffer."); + + memcpy(ResponseBuffer.Body->getBufferStart() + BufferOffset, BodyChunk.data(), + BodyChunk.size()); + BufferOffset += BodyChunk.size(); + return BodyChunk.size(); +} + +void BufferedHTTPResponseHandler::handleResponseCode(unsigned Code) { + ResponseBuffer.Code = Code; +} Index: llvm/test/CMakeLists.txt =================================================================== --- llvm/test/CMakeLists.txt +++ llvm/test/CMakeLists.txt @@ -6,6 +6,7 @@ LLVM_ENABLE_DIA_SDK LLVM_ENABLE_FFI LLVM_ENABLE_THREADS + LLVM_ENABLE_CURL LLVM_ENABLE_ZLIB LLVM_ENABLE_LIBXML2 LLVM_INCLUDE_GO_TESTS Index: llvm/test/lit.cfg.py =================================================================== --- llvm/test/lit.cfg.py +++ llvm/test/lit.cfg.py @@ -158,9 +158,9 @@ tools.extend([ 'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as', 'llvm-addr2line', 'llvm-bcanalyzer', 'llvm-bitcode-strip', 'llvm-config', - 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', - 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis', 'llvm-extract', - 'llvm-isel-fuzzer', 'llvm-ifs', + 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find', + 'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-dlltool', + 'llvm-exegesis', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-ifs', 'llvm-install-name-tool', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib', 'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca', 'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump', 'llvm-otool', @@ -399,3 +399,6 @@ if config.expensive_checks: config.available_features.add('expensive_checks') + +if getattr(config, 'enable_debuginfod_client', False): + config.available_features.add('debuginfod_client') Index: llvm/test/lit.site.cfg.py.in =================================================================== --- llvm/test/lit.site.cfg.py.in +++ llvm/test/lit.site.cfg.py.in @@ -37,6 +37,7 @@ config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@ config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.have_zlib = @LLVM_ENABLE_ZLIB@ +config.enable_debuginfod_client = @LLVM_ENABLE_CURL@ config.have_libxar = @LLVM_HAVE_LIBXAR@ config.have_libxml2 = @LLVM_ENABLE_LIBXML2@ config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@ Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo @@ -0,0 +1 @@ +fake_debuginfo Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable @@ -0,0 +1 @@ +fake_executable Index: llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c @@ -0,0 +1 @@ +int foo = 0; Index: llvm/test/tools/llvm-debuginfod/debuginfod-find.py =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/debuginfod-find.py @@ -0,0 +1,63 @@ +# RUN: rm -rf %t && mkdir %t && python %S/debuginfod-find.py %S/Inputs llvm-debuginfod-find %t +import threading +import http.server +import functools +import subprocess +import os + + +test_assets = [ + (['--executable'], '4503653137150120245', 'fake_executable\n'), + (['--source=/directory/file.c'], '163445695080362705', + 'int foo = 0;\n'), + (['--debuginfo'], '9104678335595080419', 'fake_debuginfo\n') +] + +def test_tool(inputs_path, tool_path, cache_directory) -> int: + httpd = http.server.ThreadingHTTPServer( + ('',0), functools.partial( + http.server.SimpleHTTPRequestHandler, + directory=inputs_path)) + port = httpd.server_port + thread = threading.Thread(target=httpd.serve_forever) + + try: + thread.start() + common_args = [tool_path, 'fake_build_id', + '--cache-dir', cache_directory] + for args, xxhash, contents in test_assets: + process = subprocess.Popen( + common_args + args, env={**os.environ, + 'DEBUGINFOD_URLS': f'http://localhost:{port}'}) + if process.wait() != 0: + return 1 + fname = os.path.join(cache_directory, + f'llvmcache-{xxhash}') + print(fname) + if not os.path.exists(fname): + return 1 + with open(fname) as infile: + if infile.read() != contents: + return 1 + + finally: + httpd.shutdown() + thread.join() + + return 0 + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('inputs_path') + parser.add_argument('tool_path') + parser.add_argument('cache_directory') + args = parser.parse_args() + result = test_tool(args.inputs_path, args.tool_path, + args.cache_directory) + os._exit(result) + + +if __name__ == '__main__': + main() Index: llvm/test/tools/llvm-debuginfod/lit.local.cfg =================================================================== --- /dev/null +++ llvm/test/tools/llvm-debuginfod/lit.local.cfg @@ -0,0 +1 @@ +config.suffixes = ['.py'] Index: llvm/tools/llvm-debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/tools/llvm-debuginfod/CMakeLists.txt @@ -0,0 +1,12 @@ +if (LLVM_ENABLE_CURL) + set(LLVM_LINK_COMPONENTS + Debuginfod + Support + ) + add_llvm_tool(llvm-debuginfod-find + llvm-debuginfod-find.cpp + ) + if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(debuginfod-find llvm-debuginfod-find) + endif() +endif() Index: llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp =================================================================== --- /dev/null +++ llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp @@ -0,0 +1,118 @@ +//===-- llvm-debuginfod-find.cpp - Simple CLI for libdebuginfod-client ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the llvm-debuginfod-find tool. This tool +/// queries the debuginfod servers in the DEBUGINFOD_URLS environment +/// variable (delimited by space (" ")) for the executable, +/// debuginfo, or specified source file of the binary matching the +/// given build-id. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/config.h" +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/COM.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include + +#define DEBUG_TYPE "llvm-debuginfod-find" + +using namespace llvm; + +cl::opt InputBuildID(cl::Positional, cl::Required, + cl::desc(""), cl::init("-")); + +static cl::opt + FetchExecutable("executable", cl::init(false), + cl::desc("fetch the associated executable")); + +static cl::opt FetchDebuginfo("debuginfo", cl::init(false), + cl::desc("fetch associated debuginfo")); + +static cl::opt FetchSource("source", cl::init(""), + cl::desc("/filename")); + +static cl::opt CacheDir("cache-dir", cl::init(""), + cl::desc("Cache Directory"), + cl::value_desc("directory")); + +[[noreturn]] static void helpExit() { + errs() << "Must specify exactly one of --executable, " + "--source=/path/to/file, or --debuginfo."; + exit(1); +} + +ExitOnError ExitOnErr; + +int main(int argc, char **argv) { + InitLLVM X(argc, argv); + + cl::ParseCommandLineOptions(argc, argv); + + const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); + if (DebuginfodUrlsEnv == NULL) { + errs() << "Missing DEBUGINFOD_URLS environment variable.\n"; + return 1; + } + + SmallVector DebuginfodUrls; + StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " "); + + SmallString<64> CacheDirectoryPath = StringRef(CacheDir); + if (CacheDirectoryPath.empty() && + !sys::path::cache_directory(CacheDirectoryPath)) { + errs() << "Unable to determine appropriate cache directory.\n"; + return 1; + } + + assert(CacheDirectoryPath.size() && "CacheDirectoryPath should be nonempty"); + + if (FetchExecutable + FetchDebuginfo + (FetchSource != "") != 1) + helpExit(); + + DebuginfodAssetType Type; + StringRef Description; + if (FetchExecutable) + Type = DebuginfodAssetType::Executable; + else if (FetchDebuginfo) + Type = DebuginfodAssetType::Debuginfo; + else if (FetchSource != "") { + Type = DebuginfodAssetType::Source; + Description = FetchSource; + } else + helpExit(); + + if (Type == DebuginfodAssetType::Source) { + // Print the contents of the source file + ExitOnErr(fetchDebuginfo(CacheDirectoryPath, DebuginfodUrls, InputBuildID, + Type, Description, + [](size_t Task, std::unique_ptr MB) { + outs() << MB->getBuffer(); + })); + } else { + // Print the path to the cached binary file on disk + outs() << ExitOnErr(fetchDebuginfo(CacheDirectoryPath, DebuginfodUrls, + InputBuildID, Type, Description)) + << "\n"; + } +} Index: llvm/unittests/CMakeLists.txt =================================================================== --- llvm/unittests/CMakeLists.txt +++ llvm/unittests/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory(Bitstream) add_subdirectory(CodeGen) add_subdirectory(DebugInfo) +add_subdirectory(Debuginfod) add_subdirectory(Demangle) add_subdirectory(ExecutionEngine) add_subdirectory(FileCheck) Index: llvm/unittests/Debuginfod/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/unittests/Debuginfod/CMakeLists.txt @@ -0,0 +1,11 @@ +if (LLVM_ENABLE_CURL) + set(LLVM_LINK_COMPONENTS + Debuginfod + ) + + add_llvm_unittest(DebuginfodTests + DebuginfodTests.cpp + ) + + target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport) +endif() Index: llvm/unittests/Debuginfod/DebuginfodTests.cpp =================================================================== --- /dev/null +++ llvm/unittests/Debuginfod/DebuginfodTests.cpp @@ -0,0 +1,18 @@ +//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +TEST(DebuginfodTests, noDebuginfodUrlsFetchInfoTest) { + EXPECT_THAT_EXPECTED(fetchDebuginfo("./", {}, "fakeBuildId", + llvm::DebuginfodAssetType::Executable, + ""), + llvm::Failed()); +} Index: llvm/unittests/Support/CMakeLists.txt =================================================================== --- llvm/unittests/Support/CMakeLists.txt +++ llvm/unittests/Support/CMakeLists.txt @@ -41,6 +41,7 @@ GlobPatternTest.cpp HashBuilderTest.cpp Host.cpp + HTTPClient.cpp IndexedAccessorTest.cpp InstructionCostTest.cpp ItaniumManglingCanonicalizerTest.cpp Index: llvm/unittests/Support/HTTPClient.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/HTTPClient.cpp @@ -0,0 +1,233 @@ +//===-- llvm/unittest/Support/HTTPClient.cpp - unit tests -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/CURLClient.h" +#include "llvm/Support/Errc.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerLifecycleTest) { + BufferedHTTPResponseHandler Handler; + // test initial state + EXPECT_EQ(Handler.ResponseBuffer.Code, 0u); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + // a body chunk passed before the content-length header should return Error + EXPECT_THAT_EXPECTED(Handler.handleBodyChunk("body"), + Failed()); + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("a header line"), HasValue(13)); + EXPECT_THAT_EXPECTED(Handler.handleBodyChunk("body"), + Failed()); + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("Content-Length: 36\r\n"), + HasValue(20)); + EXPECT_THAT_EXPECTED(Handler.handleBodyChunk("body:"), HasValue(5)); + EXPECT_THAT_EXPECTED( + Handler.handleBodyChunk("this puts the total at 36 chars"), HasValue(31)); + EXPECT_EQ(memcmp(Handler.ResponseBuffer.Body->getBufferStart(), + "body:this puts the total at 36 chars", + Handler.ResponseBuffer.Body->getBufferSize()), + 0); + + // additional content should be rejected by the handler + EXPECT_THAT_EXPECTED( + Handler.handleBodyChunk("extra content past the content-length"), + Failed()); + + // response code handling + Handler.handleResponseCode(200u); + EXPECT_EQ(Handler.ResponseBuffer.Code, 200u); +} + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerZeroContentLengthTest) { + BufferedHTTPResponseHandler Handler; + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("Content-Length: 0"), + HasValue(17)); + EXPECT_NE(Handler.ResponseBuffer.Body, nullptr); + EXPECT_EQ(Handler.ResponseBuffer.Body->getBufferSize(), 0u); + + // all content should be rejected by the handler + EXPECT_THAT_EXPECTED(Handler.handleBodyChunk("non-empty body content"), + Failed()); +} + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerMalformedContentLengthTest) { + // confirm getAsUnsignedInteger behaves as expected + // in particular, getAsUnsignedInteger returns false + // for valid inputs + unsigned long long ContentLength; + EXPECT_EQ(llvm::getAsUnsignedInteger("fff", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("-1", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("0111", 10, ContentLength), false); + EXPECT_EQ(llvm::getAsUnsignedInteger("0", 10, ContentLength), false); + EXPECT_EQ(ContentLength, 0u); + + // check several invalid content lengths are ignored + BufferedHTTPResponseHandler Handler; + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("Content-Length: fff"), + HasValue(19)); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("Content-Length: "), + HasValue(19)); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + EXPECT_THAT_EXPECTED( + Handler.handleHeaderLine(StringRef("Content-Length: \0\0\0", 19)), + HasValue(19)); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + EXPECT_THAT_EXPECTED(Handler.handleHeaderLine("Content-Length: -11"), + HasValue(19)); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + // all content should be rejected by the handler because + // no valid content length has been received + EXPECT_THAT_EXPECTED(Handler.handleBodyChunk("non-empty body content"), + Failed()); +} + +class HTTPResponseHandlerSimulator : public HTTPResponseHandler { +public: + struct { + size_t BodyBytesTotal = 0; + size_t HeaderLinesTotal = 0; + unsigned ResponseCode = 0; + } State; + + Expected handleHeaderLine(StringRef HeaderLine) { + State.HeaderLinesTotal++; + return HeaderLine.size(); + }; + + Expected handleBodyChunk(StringRef BodyChunk) { + State.BodyBytesTotal += BodyChunk.size(); + return BodyChunk.size(); + }; + + void handleResponseCode(unsigned Code) { State.ResponseCode = Code; }; +}; + +struct CurlHTTPRequestSimulatorState { + bool Initialized = false; + bool Performed = false; + bool CleanedUp = false; +}; + +class CurlHTTPRequestSimulator : public CurlHTTPRequest { + Error curlInit() override { + State.Initialized = true; + State.CleanedUp = false; + return CurlHTTPRequest::curlInit(); + } + +public: + CurlHTTPRequestSimulatorState &State; + + CurlHTTPRequestSimulator(HTTPResponseHandler *Handler, + CurlHTTPRequestSimulatorState &State) + : CurlHTTPRequest::CurlHTTPRequest(Handler), State(State) {} + + virtual Error performRequest() override { + if (!Curl) + return createStringError(errc::inappropriate_io_control_operation, + "CURL is uninitialized."); + State.Performed = true; + Handler->handleResponseCode(200); + if (Error Err = Handler->handleHeaderLine("Content-Length: 17").takeError()) + return joinErrors(std::move(ErrorState), std::move(Err)); + if (Error Err = Handler->handleBodyChunk("This is the body.").takeError()) + return joinErrors(std::move(ErrorState), std::move(Err)); + return std::move(ErrorState); + } + + ~CurlHTTPRequestSimulator() { + State.Initialized = false; + State.CleanedUp = true; + } +}; + +class CurlHTTPRequestBrokenConnectionSimulator + : public CurlHTTPRequestSimulator { +public: + Error performRequest() override { + if (!Curl) + return createStringError(errc::inappropriate_io_control_operation, + "CURL is uninitialized."); + State.Performed = true; + Handler->handleResponseCode(200); + if (Error Err = Handler->handleHeaderLine("Content-Length: 17").takeError()) + return joinErrors(std::move(ErrorState), std::move(Err)); + if (Error Err = Handler->handleBodyChunk("This is").takeError()) + return joinErrors(std::move(ErrorState), std::move(Err)); + return joinErrors( + std::move(ErrorState), + createStringError(errc::io_error, "Simulated network failure")); + } + using CurlHTTPRequestSimulator::CurlHTTPRequestSimulator; +}; + +TEST(CURLClientTests, curlRequestBrokenConnectionTest) { + HTTPResponseHandlerSimulator Handler; + CurlHTTPRequestSimulatorState State; + CurlHTTPRequestBrokenConnectionSimulator Request(&Handler, State); + + HTTPRequestConfig Config; + EXPECT_THAT_ERROR(Request.setConfiguration(Config), Succeeded()); + EXPECT_THAT_ERROR(Request.performRequest(), Failed()); +} + +TEST(CURLClientTests, curlRequestLifecycleTest) { + HTTPResponseHandlerSimulator Handler; + CurlHTTPRequestSimulatorState State; + { + CurlHTTPRequestSimulator Request(&Handler, State); + EXPECT_EQ(State.Initialized, false); + EXPECT_EQ(State.Performed, false); + EXPECT_EQ(State.CleanedUp, false); + + HTTPRequestConfig Config; + EXPECT_THAT_ERROR(Request.setConfiguration(Config), Succeeded()); + EXPECT_EQ(State.Initialized, true); + EXPECT_EQ(State.Performed, false); + EXPECT_EQ(State.CleanedUp, false); + EXPECT_EQ(Request.Config, Config); + + EXPECT_THAT_ERROR(Request.performRequest(), Succeeded()); + EXPECT_EQ(State.Initialized, true); + EXPECT_EQ(State.Performed, true); + EXPECT_EQ(State.CleanedUp, false); + } + // destructor of CurlHTTPRequest should invoke curlCleanup + EXPECT_EQ(State.Initialized, false); + EXPECT_EQ(State.Performed, true); + EXPECT_EQ(State.CleanedUp, true); + EXPECT_EQ(Handler.State.BodyBytesTotal, 17u); + EXPECT_EQ(Handler.State.HeaderLinesTotal, 1u); + EXPECT_EQ(Handler.State.ResponseCode, 200u); +} + +TEST(CURLClientTests, curlHTTPRequestMultipleInitTest) { + // multiple calls to curlInit should do nothing + HTTPResponseHandlerSimulator Handler; + CurlHTTPRequestSimulatorState State; + HTTPRequestConfig Config; + CurlHTTPRequestSimulator Request(&Handler, State); + EXPECT_EQ(State.Initialized, false); + EXPECT_EQ(State.Performed, false); + EXPECT_EQ(State.CleanedUp, false); + EXPECT_THAT_ERROR(Request.setConfiguration(Config), Succeeded()); + EXPECT_EQ(State.Initialized, true); + EXPECT_EQ(State.Performed, false); + EXPECT_EQ(State.CleanedUp, false); + EXPECT_THAT_ERROR(Request.setConfiguration(Config), Succeeded()); + EXPECT_EQ(State.Initialized, true); + EXPECT_EQ(State.Performed, false); + EXPECT_EQ(State.CleanedUp, false); +}