diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -396,6 +396,12 @@ set(LLVM_ENABLE_Z3_SOLVER_DEFAULT "${Z3_FOUND}") +if (LLVM_ENABLE_DEBUGINFOD_CLIENT) + set(LLVM_WITH_CURL 1) + set(CURL_LIBRARY "-lcurl") + find_package(CURL 7.74.0 REQUIRED) + add_compile_definitions(LLVM_ENABLE_DEBUGINFOD_CLIENT LLVM_WITH_CURL) +endif() if( LLVM_TARGETS_TO_BUILD STREQUAL "all" ) set( LLVM_TARGETS_TO_BUILD ${LLVM_ALL_TARGETS} ) diff --git a/llvm/include/llvm/Support/Debuginfod.h b/llvm/include/llvm/Support/Debuginfod.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/Debuginfod.h @@ -0,0 +1,40 @@ +#ifdef LLVM_ENABLE_DEBUGINFOD_CLIENT + +#ifndef LLVM_SUPPORT_DEBUGINFOD_H +#define LLVM_SUPPORT_DEBUGINFOD_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace debuginfod { + +enum AssetType { Executable, Debuginfo, Source }; + +Expected contentsOfFile(StringRef AbsolutePath); + +class AssetCache { + StringRef DirectoryPath; + +public: + AssetCache(StringRef CacheDirectoryPath); + Expected contains(StringRef RelPath); + Error add(StringRef Key, StringRef Contents); + Error prune(); + std::string absolutePath(StringRef Key); + Expected contentsOf(StringRef Key); +}; + +std::string escapePath(StringRef RelPath); + +Expected fetchInfo(ArrayRef DebuginfodUrls, + StringRef CacheDirectoryPath, StringRef BuildID, + AssetType Type, StringRef Description = ""); + +} // end namespace debuginfod +} // end namespace llvm + +#endif + +#endif \ No newline at end of file diff --git a/llvm/include/llvm/Support/HTTPClient.h b/llvm/include/llvm/Support/HTTPClient.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Support/HTTPClient.h @@ -0,0 +1,18 @@ +#ifndef LLVM_SUPPORT_HTTP_CLIENT_H +#define LLVM_SUPPORT_HTTP_CLIENT_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace http { +struct Response { + long Code = 0; + std::string Body; +}; +Expected get(const Twine &Url); +} // end namespace http +} // end namespace llvm + +#endif // LLVM_HTTP_CLIENT_H diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -74,6 +74,10 @@ set(system_libs ${system_libs} ${Z3_LIBRARIES}) endif() +if (LLVM_WITH_CURL) + set(system_libs ${system_libs} ${CURL_LIBRARIES}) +endif() + # Override the C runtime allocator on Windows and embed it into LLVM tools & libraries if(LLVM_INTEGRATED_CRT_ALLOC) if (CMAKE_BUILD_TYPE AND NOT ${LLVM_USE_CRT_${uppercase_CMAKE_BUILD_TYPE}} MATCHES "^(MT|MTd)$") @@ -136,6 +140,7 @@ DataExtractor.cpp Debug.cpp DebugCounter.cpp + Debuginfod.cpp DeltaAlgorithm.cpp DAGDeltaAlgorithm.cpp DJB.cpp @@ -153,6 +158,7 @@ GlobPattern.cpp GraphWriter.cpp Hashing.cpp + HTTPClient.cpp InitLLVM.cpp InstructionCost.cpp IntEqClasses.cpp diff --git a/llvm/lib/Support/Debuginfod.cpp b/llvm/lib/Support/Debuginfod.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/Debuginfod.cpp @@ -0,0 +1,185 @@ +#ifdef LLVM_ENABLE_DEBUGINFOD_CLIENT + +#include "llvm/Support/Debuginfod.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CachePruning.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/Path.h" + +#include + +namespace llvm { +using namespace sys::fs; +namespace debuginfod { +using namespace sys::path; + +#define DEBUG_TYPE "DEBUGINFOD" + +AssetCache::AssetCache(StringRef CacheDirectoryPath) { + DirectoryPath = CacheDirectoryPath; +} +Expected AssetCache::contains(StringRef Key) { + LLVM_DEBUG(dbgs() << "pruning cache first\n";); + if (Error Err = prune()) + return Err; + + dbgs() << "checking if asset cache contains Key " << Key << "\n"; + auto AbsolutePath = absolutePath(Key); + file_status Status; + auto EC = status(AbsolutePath, Status); + if (EC && EC != errc::no_such_file_or_directory) + return createStringError(EC, "failed to check asset cache entry status"); + else if (EC == errc::no_such_file_or_directory) + return false; + else + return exists(Status); +} +Error AssetCache::add(StringRef Key, StringRef Contents) { + auto AbsolutePath = absolutePath(Key); + StringRef Root = parent_path(AbsolutePath); + dbgs() << "creating Root = " << Root << "\n"; + create_directories(Root); + if (Error Err = prune()) + return Err; + // return Error::success(); + dbgs() << "writing file atomically, Contents.size() = " << Contents.size() + << "\n"; + return writeFileAtomically("llvm-debuginfod-%%%%%%%%%%%%%%%%%%", AbsolutePath, + // "abc1234\n" + Contents); +} +Error AssetCache::prune() { + // see CachePruning.h + // "A default constructed CachePruningPolicy + // provides a reasonable default policy." + pruneCache(DirectoryPath, CachePruningPolicy()); + return Error::success(); +} +Expected contentsOfFile(StringRef AbsolutePath) { + + uint64_t FileSize; + if (auto EC = file_size(AbsolutePath, FileSize)) + return createStringError(EC, "error getting file size"); + + LLVM_DEBUG(dbgs() << "got file size of " << FileSize << " bytes\n";); + + LLVM_DEBUG(dbgs() << "opening native file at " << AbsolutePath << "\n";); + auto FileOrErr = openNativeFileForRead(AbsolutePath); + if (Error Err = FileOrErr.takeError()) + return Err; + auto &File = *FileOrErr; + LLVM_DEBUG(dbgs() << "native file opened\n";); + + LLVM_DEBUG(dbgs() << "reading native file\n";); + std::vector Buf(FileSize); + auto SizeReadOrErr = + readNativeFile(File, MutableArrayRef(Buf.data(), Buf.size())); + if (Error Err = SizeReadOrErr.takeError()) + return Err; + LLVM_DEBUG(dbgs() << "read native file\n";); + + auto &SizeRead = *SizeReadOrErr; + LLVM_DEBUG(dbgs() << "SizeRead = " << SizeRead << " bytes\n";); + + if (SizeRead != FileSize) + return createStringError(errc::result_out_of_range, + "failed to read entire file"); + + if (auto EC = closeFile(File)) + return createStringError(EC, "error closing file"); + + return std::string(Buf.data(), Buf.size()); +} + +Expected AssetCache::contentsOf(StringRef Key) { + auto AbsolutePath = absolutePath(Key); + return contentsOfFile(AbsolutePath); +} + +static std::string hexEncode(std::string String) { + std::string Output = ""; + const std::vector hexDigits = {"0", "1", "2", "3", "4", "5", + "6", "7", "8", "9", "a", "b", + "c", "d", "e", "f"}; + for (auto c : String) { + unsigned int i1 = (int)c & 0b1111; + unsigned int i2 = (int)c >> 4; + Output += hexDigits.at(i1); + Output += hexDigits.at(i2); + } + return Output; +} + +std::string AssetCache::absolutePath(StringRef Key) { + LLVM_DEBUG(dbgs() << "getting absolute path of " << Key << "\n";); + // std::stringstream ss; + // ss << std::hex << Key.str(); + + std::string HexEncodedKey = hexEncode(Key.str()); + auto AbsPath = DirectoryPath + "/llvmcache-debuginfod-" + HexEncodedKey; + dbgs() << "HexEncodedKey = " << HexEncodedKey << "\n"; + LLVM_DEBUG(dbgs() << "got absolute path " << AbsPath << "\n";); + return AbsPath.str(); +} + +Expected fetchInfo(ArrayRef DebuginfodUrls, + StringRef CacheDirectoryPath, StringRef BuildID, + AssetType Type, StringRef Description) { + dbgs() << "fetching info, debuginfod urls size = " << DebuginfodUrls.size() + << "\n"; + std::string Suffix; + if (Type == Executable) { + Suffix = "executable"; + } else if (Type == Debuginfo) { + Suffix = "debuginfo"; + } else if (Type == Source) { + // Description is the absolute source path + Suffix = "source" + Description.str(); + } else { + llvm_unreachable("invalid debuginfo asset type"); + } + + AssetCache Cache(CacheDirectoryPath); + auto UniqueKey = (BuildID + Suffix).str(); + auto CacheHitOrErr = Cache.contains(UniqueKey); + if (auto Err = CacheHitOrErr.takeError()) { + return Err; + } + bool &CacheHit = *CacheHitOrErr; + if (CacheHit) { + dbgs() << "cache hit!\n"; + return Cache.absolutePath(UniqueKey); + } + + dbgs() << "did not find in cache, querying urls\n"; + for (auto &Url : DebuginfodUrls) { + LLVM_DEBUG(dbgs() << "trying Url " + << Url + "buildid/" + BuildID + "/" + Suffix << "\n";); + auto ResponseOrErr = http::get(Url + "buildid/" + BuildID + "/" + Suffix); + if (auto Err = ResponseOrErr.takeError()) { + return Err; + } + auto &Resp = *ResponseOrErr; + if (Resp.Code == 200) { + dbgs() << "caching asset\n"; + if (auto Err = Cache.add(UniqueKey, Resp.Body)) + return Err; + return Cache.absolutePath(UniqueKey); + } else { + LLVM_DEBUG(dbgs() << "debuginfo not found at this server\n";); + } + } + + return createStringError(errc::argument_out_of_domain, "build id not found"); +} + +#undef DEBUG_TYPE +} // end namespace debuginfod +} // end namespace llvm + +#endif \ No newline at end of file diff --git a/llvm/lib/Support/HTTPClient.cpp b/llvm/lib/Support/HTTPClient.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Support/HTTPClient.cpp @@ -0,0 +1,97 @@ +#ifdef LLVM_WITH_CURL + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" + +#include "llvm/Support/HTTPClient.h" + +#include + +namespace llvm { +namespace http { + +#define DEBUG_TYPE "HTTPClient" + +struct MemoryStruct { + char *memory; + size_t size; +}; + +static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, + void *userp) { + size_t realsize = size * nmemb; + struct MemoryStruct *mem = (struct MemoryStruct *)userp; + + char *ptr = (char *)realloc(mem->memory, mem->size + realsize + 1); + if (!ptr) { + /* out of memory! */ + printf("not enough memory (realloc returned NULL)\n"); + return 0; + } + + mem->memory = ptr; + memcpy(&(mem->memory[mem->size]), contents, realsize); + mem->size += realsize; + mem->memory[mem->size] = 0; + + return realsize; +} + +Expected get(const Twine &Url) { + + LLVM_DEBUG(dbgs() << "getting Url " << Url << "\n";); + + struct MemoryStruct chunk; + + chunk.memory = + (char *)malloc(1); /* will be grown as needed by the realloc above */ + chunk.size = 0; /* no data at this point */ + + CURL *Curl = curl_easy_init(); + CURLcode CurlRes; + if (!Curl) { + return createStringError(errc::io_error, "http library error"); + } + + std::string UrlString = Url.str(); + curl_easy_setopt(Curl, CURLOPT_URL, UrlString.c_str()); + /* example.com is redirected, so we tell libcurl to follow redirection */ + curl_easy_setopt(Curl, CURLOPT_FOLLOWLOCATION, 1L); + + /* send all data to this function */ + curl_easy_setopt(Curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(Curl, CURLOPT_WRITEDATA, (void *)&chunk); + + /* Perform the request, res will get the return code */ + LLVM_DEBUG(dbgs() << "performing the curl\n";); + + CurlRes = curl_easy_perform(Curl); + + LLVM_DEBUG(dbgs() << "got the CurlRes\n";); + + /* Check for errors */ + if (CurlRes != CURLE_OK) { + curl_easy_cleanup(Curl); + return createStringError(errc::io_error, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(CurlRes)); + } + + /* always cleanup */ + curl_easy_cleanup(Curl); + std::string Body(chunk.memory, chunk.size); + free(chunk.memory); + Response Resp; + Resp.Body = Body; + curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &Resp.Code); + return Resp; +} + +#undef DEBUG_TYPE + +} // end namespace http +} // end namespace llvm + +#endif \ No newline at end of file diff --git a/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/debuginfo @@ -0,0 +1 @@ +fake_debuginfo diff --git a/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/executable @@ -0,0 +1 @@ +fake_executable diff --git a/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/Inputs/buildid/fake_build_id/source/directory/file.c @@ -0,0 +1 @@ +int foo = 0; diff --git a/llvm/test/tools/llvm-debuginfod/find-test.sh b/llvm/test/tools/llvm-debuginfod/find-test.sh new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/find-test.sh @@ -0,0 +1,15 @@ +set -x + +# start the server +timeout 2s python -m http.server \ + --directory $1/Inputs/ $3 & + +sleep 0.5s + +# test the client +DEBUGINFOD_URLS=http://localhost:$3/ \ + llvm-debuginfod-find fake_build_id \ + $2 \ + -debug + +exit \ No newline at end of file diff --git a/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.sh b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.sh new file mode 100755 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.sh @@ -0,0 +1,3 @@ +timeout 1s python -m http.server & + +wait; \ No newline at end of file diff --git a/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.test b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod-find.test @@ -0,0 +1,3 @@ +# RUN: bash %S/find-test.sh %S --executable 13573 +# RUN: bash %S/find-test.sh %S --debuginfo 13574 +# RUN: bash %S/find-test.sh %S --source=/directory/file.c 13575 diff --git a/llvm/tools/llvm-debuginfod/CMakeLists.txt b/llvm/tools/llvm-debuginfod/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/llvm/tools/llvm-debuginfod/CMakeLists.txt @@ -0,0 +1,12 @@ +if (LLVM_ENABLE_DEBUGINFOD_CLIENT) + set(LLVM_LINK_COMPONENTS + Support + ) + add_llvm_tool(llvm-debuginfod-find + llvm-debuginfod-find.cpp + ) + set_property(TARGET llvm-debuginfod-find PROPERTY LLVM_SYSTEM_LIBS ${imported_libs}) + if(LLVM_INSTALL_BINUTILS_SYMLINKS) + add_llvm_tool_symlink(debuginfod-find llvm-debuginfod-find) + endif() +endif() diff --git a/llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp b/llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp new file mode 100644 --- /dev/null +++ b/llvm/tools/llvm-debuginfod/llvm-debuginfod-find.cpp @@ -0,0 +1,102 @@ + +//===-- llvm-debuginfod-find.cpp - Simple CLI for libdebuginfod-client +//------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tool queries the debuginfod servers in the DEBUGINFOD_URLS +// environment variable (delimited by space (" ")) for debuginfo of +// the binary matching the given build-id +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/config.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/COM.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Debuginfod.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include + +#define DEBUG_TYPE "llvm-debuginfod-find" + +using namespace llvm; +using namespace debuginfod; + +cl::opt InputBuildID(cl::Positional, cl::Required, + cl::desc(""), cl::init("-")); + +static cl::opt + FetchExecutable("executable", cl::init(false), + cl::desc("fetch the associated executable")); + +static cl::opt FetchDebuginfo("debuginfo", cl::init(false), + cl::desc("fetch associated debuginfo")); + +static cl::opt FetchSource("source", cl::init(""), + cl::desc("/filename")); + +ExitOnError ExitOnErr; + +int main(int argc, char **argv) { + InitLLVM X(argc, argv); + + cl::ParseCommandLineOptions(argc, argv); + + const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); + if (DebuginfodUrlsEnv == NULL) { + errs() << "DEBUGINFOD_URLS not set\n"; + return 1; + } + + SmallVector DebuginfodUrls; + StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " "); + + const char *HomeEnv = std::getenv("HOME"); + if (HomeEnv == NULL) { + LLVM_DEBUG(dbgs() << "HOME not set\n";); + return 1; + } + LLVM_DEBUG(dbgs() << "HOME = " << HomeEnv << "\n";); + auto CacheDirectoryPath = + std::string(HomeEnv) + + std::string("/.llvmdebuginfodcache/debuginfod_client"); + LLVM_DEBUG(dbgs() << "CacheDirectoryPath = " << CacheDirectoryPath << "\n";); + + AssetType Type; + StringRef Description; + if (FetchExecutable) { + Type = Executable; + } else if (FetchDebuginfo) { + Type = Debuginfo; + } else if (FetchSource != "") { + Type = Source; + Description = FetchSource; + } else { + llvm_unreachable("invalid asset request"); + } + std::string InfoFilePath = ExitOnErr(fetchInfo( + DebuginfodUrls, CacheDirectoryPath, InputBuildID, Type, Description)); + if (Type == Source) + outs() << ExitOnErr(contentsOfFile(InfoFilePath)); + else + outs() << InfoFilePath << "\n"; +} + +#undef DEBUG_TYPE \ No newline at end of file diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt --- a/llvm/unittests/Support/CMakeLists.txt +++ b/llvm/unittests/Support/CMakeLists.txt @@ -23,6 +23,7 @@ ConvertUTFTest.cpp CRCTest.cpp DataExtractorTest.cpp + Debuginfod.cpp DebugTest.cpp DebugCounterTest.cpp DJBTest.cpp @@ -41,6 +42,7 @@ GlobPatternTest.cpp HashBuilderTest.cpp Host.cpp + HTTPClient.cpp IndexedAccessorTest.cpp InstructionCostTest.cpp ItaniumManglingCanonicalizerTest.cpp diff --git a/llvm/unittests/Support/Debuginfod.cpp b/llvm/unittests/Support/Debuginfod.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/Debuginfod.cpp @@ -0,0 +1,19 @@ +#ifdef LLVM_ENABLE_DEBUGINFOD_CLIENT + +#include "llvm/Support/Debuginfod.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" + +#include "gtest/gtest.h" + +using namespace llvm; + +TEST(DebuginfodTests, noDebuginfodUrlsFetchInfoTest) { + auto InfoOrErr = fetchInfo({}, "./", "fakeBuildId", + llvm::debuginfod::AssetType::Executable, ""); + handleAllErrors(InfoOrErr.takeError(), [](const StringError &SE) { + dbgs() << "got a StringError as expected\n"; + }); +} + +#endif \ No newline at end of file diff --git a/llvm/unittests/Support/HTTPClient.cpp b/llvm/unittests/Support/HTTPClient.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/Support/HTTPClient.cpp @@ -0,0 +1,39 @@ +#ifdef LLVM_WITH_CURL + +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/Error.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/config.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/COM.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Debuginfod.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include + +#include "gtest/gtest.h" + +using namespace llvm; +using namespace http; + +TEST(HTTPClientTests, invalidUrlTest) { + std::string invalidUrl = "llvm is fun"; + handleAllErrors(llvm::http::get(invalidUrl).takeError(), + [](const StringError &SE) { + dbgs() << "got a StringError as expected\n"; + }); +} + +#endif \ No newline at end of file