Index: llvm/include/llvm/Debuginfod/HTTPServer.h =================================================================== --- /dev/null +++ llvm/include/llvm/Debuginfod/HTTPServer.h @@ -0,0 +1,123 @@ +//===-- llvm/Debuginfod/HTTPServer.h - HTTP server library ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the HTTPServer and HTTPServerRequest +/// classes, the HTTPResponse, and StreamingHTTPResponse structs, and the +/// streamFile function. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HTTP_SERVER_H +#define LLVM_SUPPORT_HTTP_SERVER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#ifdef LLVM_ENABLE_HTTPLIB +// forward declarations +namespace httplib { +class Request; +class Response; +class Server; +} // namespace httplib +#endif + +namespace llvm { + +struct HTTPResponse; +struct StreamingHTTPResponse; +class HTTPServer; + +class HTTPServerRequest { + friend HTTPServer; + +#ifdef LLVM_ENABLE_HTTPLIB +private: + HTTPServerRequest(const httplib::Request &HTTPLibRequest, + httplib::Response &HTTPLibResponse); + httplib::Response &HTTPLibResponse; +#endif + +public: + std::string UrlPath; + /// The elements correspond to match groups in the url path matching regex. + SmallVector UrlPathMatches; + + // TODO bring in HTTP headers + + void setResponse(StreamingHTTPResponse Response); + void setResponse(HTTPResponse Response); +}; + +struct HTTPResponse { + unsigned Code; + const char *ContentType; + StringRef Body; +}; + +typedef std::function HTTPRequestHandler; + +/// An HTTPContentProvider is called by the HTTPServer to obtain chunks of the +/// streaming response body. The returned chunk should be located at Offset +/// bytes and have Length bytes. +typedef std::function + HTTPContentProvider; + +/// Wraps the content provider with HTTP Status code and headers. +struct StreamingHTTPResponse { + unsigned Code; + const char *ContentType; + size_t ContentLength; + HTTPContentProvider Provider; + /// Called after the response transfer is complete with the success value of + /// the transfer. + std::function CompletionHandler = [](bool Success) {}; +}; + +/// Sets the response to stream the file at FilePath, if available, and +/// otherwise an HTTP 404 error response. +bool streamFile(HTTPServerRequest &Request, StringRef FilePath); + +/// An HTTP server which can listen on a single TCP/IP port for HTTP +/// requests and delgate them to the appropriate registered handler. +class HTTPServer { +#ifdef LLVM_ENABLE_HTTPLIB + std::unique_ptr Server; + unsigned Port = 0; +#endif +public: + HTTPServer(); + ~HTTPServer(); + + /// Returns true only if LLVM has been compiled with a working HTTPServer. + static bool isAvailable(); + + /// Registers a URL pattern routing rule. When the server is listening, each + /// request is dispatched to the first registered handler whose UrlPathPattern + /// matches the UrlPath. + Error get(StringRef UrlPathPattern, HTTPRequestHandler Handler); + + /// Attempts to assign the requested port and interface, returning an Error + /// upon failure. + Error bind(unsigned Port, const char *HostInterface = "0.0.0.0"); + + /// Attempts to assign any available port and interface, returning either the + /// port number or an Error upon failure. + Expected bind(const char *HostInterface = "0.0.0.0"); + + /// Attempts to listen for requests on the bound port. Returns an Error if + /// called before binding a port. + Error listen(); + + /// If the server is listening, stop and unbind the socket. + void stop(); +}; +} // end namespace llvm + +#endif // LLVM_SUPPORT_HTTP_SERVER_H Index: llvm/lib/Debuginfod/CMakeLists.txt =================================================================== --- llvm/lib/Debuginfod/CMakeLists.txt +++ llvm/lib/Debuginfod/CMakeLists.txt @@ -3,9 +3,15 @@ set(imported_libs CURL::libcurl) endif() +# Link cpp-httplib if the user wants it +if (LLVM_ENABLE_HTTPLIB) + set(imported_libs ${imported_libs} httplib::httplib) +endif() + add_llvm_component_library(LLVMDebuginfod Debuginfod.cpp HTTPClient.cpp + HTTPServer.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod Index: llvm/lib/Debuginfod/HTTPServer.cpp =================================================================== --- /dev/null +++ llvm/lib/Debuginfod/HTTPServer.cpp @@ -0,0 +1,188 @@ +//===-- llvm/Debuginfod/HTTPServer.cpp - HTTP server library --- -*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the methods of the HTTPServer class and the streamFile +/// function. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/HTTPServer.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Regex.h" + +#ifdef LLVM_ENABLE_HTTPLIB +#include "httplib.h" +#endif + +using namespace llvm; + +bool llvm::streamFile(HTTPServerRequest &Request, StringRef FilePath) { + Expected FDOrErr = sys::fs::openNativeFileForRead(FilePath); + if (Error Err = FDOrErr.takeError()) { + consumeError(std::move(Err)); + Request.setResponse({404u, "text/plain", "Could not open file to read.\n"}); + return false; + } + ErrorOr> MBOrErr = + MemoryBuffer::getOpenFile(*FDOrErr, FilePath, + /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + sys::fs::closeFile(*FDOrErr); + if (Error Err = errorCodeToError(MBOrErr.getError())) { + consumeError(std::move(Err)); + Request.setResponse({404u, "text/plain", "Could not memory-map file.\n"}); + return false; + } + // Lambdas are copied on conversion to to std::function, preventing use of + // smart pointers. + MemoryBuffer *MB = MBOrErr->release(); + Request.setResponse({200u, "application/octet-stream", MB->getBufferSize(), + [=](size_t Offset, size_t Length) -> StringRef { + return MB->getBuffer().substr(Offset, Length); + }, + [=](bool Success) { delete MB; }}); + return true; +} + +#ifdef LLVM_ENABLE_HTTPLIB + +bool HTTPServer::isAvailable() { return true; } + +HTTPServer::HTTPServer() { Server = std::make_unique(); } + +HTTPServer::~HTTPServer() { stop(); } + +static void expandUrlPathMatches(const std::smatch &Matches, + HTTPServerRequest &Request) { + bool UrlPathSet = false; + for (const auto &it : Matches) { + if (UrlPathSet) + Request.UrlPathMatches.push_back(it); + else { + Request.UrlPath = it; + UrlPathSet = true; + } + } +} + +HTTPServerRequest::HTTPServerRequest(const httplib::Request &HTTPLibRequest, + httplib::Response &HTTPLibResponse) + : HTTPLibResponse(HTTPLibResponse) { + expandUrlPathMatches(HTTPLibRequest.matches, *this); +} + +void HTTPServerRequest::setResponse(HTTPResponse Response) { + HTTPLibResponse.set_content(Response.Body.begin(), Response.Body.size(), + Response.ContentType); + HTTPLibResponse.status = Response.Code; +} + +void HTTPServerRequest::setResponse(StreamingHTTPResponse Response) { + HTTPLibResponse.set_content_provider( + Response.ContentLength, Response.ContentType, + [=](size_t Offset, size_t Length, httplib::DataSink &Sink) { + if (Offset < Response.ContentLength) { + StringRef Chunk = Response.Provider(Offset, Length); + Sink.write(Chunk.begin(), Chunk.size()); + } + return true; + }, + [=](bool Success) { Response.CompletionHandler(Success); }); + + HTTPLibResponse.status = Response.Code; +} + +Error HTTPServer::get(StringRef UrlPathPattern, HTTPRequestHandler Handler) { + std::string ErrorMessage; + if (!Regex(UrlPathPattern).isValid(ErrorMessage)) + return createStringError(errc::argument_out_of_domain, ErrorMessage); + Server->Get(std::string(UrlPathPattern), + [&](const httplib::Request &HTTPLibRequest, + httplib::Response &HTTPLibResponse) { + HTTPServerRequest Request(HTTPLibRequest, HTTPLibResponse); + Handler(Request); + }); + return Error::success(); +} + +Error HTTPServer::bind(unsigned ListenPort, const char *HostInterface) { + if (!Server->bind_to_port(HostInterface, ListenPort)) + return createStringError(errc::io_error, + "Could not assign requested address."); + Port = ListenPort; + return Error::success(); +} + +Expected HTTPServer::bind(const char *HostInterface) { + int ListenPort = Server->bind_to_any_port(HostInterface); + if (ListenPort < 0) + return createStringError(errc::io_error, + "Could not assign any port on requested address."); + return Port = ListenPort; +} + +Error HTTPServer::listen() { + if (!Port) + return createStringError(errc::io_error, + "Cannot listen without first binding to a port."); + if (!Server->listen_after_bind()) + return createStringError( + errc::io_error, + "An unknown error occurred when cpp-httplib attempted to listen."); + return Error::success(); +} + +void HTTPServer::stop() { + Server->stop(); + Port = 0; +} + +#else + +// TODO: Implement barebones standalone HTTP server implementation. +bool HTTPServer::isAvailable() { return false; } + +HTTPServer::~HTTPServer() = default; + +void HTTPServerRequest::setResponse(HTTPResponse Response) { + llvm_unreachable("No HTTP server implementation available"); +} + +void HTTPServerRequest::setResponse(StreamingHTTPResponse Response) { + llvm_unreachable("No HTTP server implementation available"); +} + +Error HTTPServer::get(StringRef UrlPathPattern, HTTPRequestHandler Handler) { + llvm_unreachable("No HTTP server implementation available"); +} + +Error HTTPServer::bind(unsigned ListenPort, const char *HostInterface) { + llvm_unreachable("No HTTP server implementation available"); +} + +Expected HTTPServer::bind(const char *HostInterface) { + llvm_unreachable("No HTTP server implementation available"); +} + +Error HTTPServer::listen() { + llvm_unreachable("No HTTP server implementation available"); +} + +void HTTPServer::stop() { + llvm_unreachable("No HTTP server implementation available"); +} + +#endif // LLVM_ENABLE_HTTPLIB Index: llvm/unittests/Debuginfod/CMakeLists.txt =================================================================== --- llvm/unittests/Debuginfod/CMakeLists.txt +++ llvm/unittests/Debuginfod/CMakeLists.txt @@ -4,6 +4,7 @@ add_llvm_unittest(DebuginfodTests HTTPClientTests.cpp + HTTPServerTests.cpp DebuginfodTests.cpp ) Index: llvm/unittests/Debuginfod/HTTPServerTests.cpp =================================================================== --- /dev/null +++ llvm/unittests/Debuginfod/HTTPServerTests.cpp @@ -0,0 +1,265 @@ +//===-- llvm/unittest/Support/HTTPServer.cpp - unit tests -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/HTTPClient.h" +#include "llvm/Debuginfod/HTTPServer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Testing/Support/Error.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace llvm; + +#ifdef LLVM_ENABLE_HTTPLIB + +TEST(HTTPServer, IsAvailable) { EXPECT_TRUE(HTTPServer::isAvailable()); } + +HTTPResponse Response = {200u, "text/plain", "hello, world\n"}; +std::string UrlPathPattern = R"(/(.*))"; +std::string InvalidUrlPathPattern = R"(/(.*)"; + +HTTPRequestHandler Handler = [](HTTPServerRequest &Request) { + Request.setResponse(Response); +}; + +HTTPRequestHandler DelayHandler = [](HTTPServerRequest &Request) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + Request.setResponse(Response); +}; + +HTTPRequestHandler StreamingHandler = [](HTTPServerRequest &Request) { + Request.setResponse({200, "text/plain", Response.Body.size(), + [=](size_t Offset, size_t Length) -> StringRef { + return Response.Body.substr(Offset, Length); + }}); +}; + +TEST(HTTPServer, InvalidUrlPath) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(InvalidUrlPathPattern, Handler), + Failed()); + EXPECT_THAT_EXPECTED(Server.bind(), Succeeded()); +} + +TEST(HTTPServer, bind) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + EXPECT_THAT_EXPECTED(Server.bind(), Succeeded()); +} + +TEST(HTTPServer, ListenBeforeBind) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + EXPECT_THAT_ERROR(Server.listen(), Failed()); +} + +#ifdef LLVM_ENABLE_CURL +// Test the client and server against each other. + +TEST(HTTPClientServer, Hello) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Test the streaming response. +TEST(HTTPClientServer, StreamingHello) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, StreamingHandler), Succeeded()); + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Writes a temporary file and streams it back using streamFile. +HTTPRequestHandler TempFileStreamingHandler = [](HTTPServerRequest Request) { + int FD; + SmallString<64> TempFilePath; + sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD, + TempFilePath); + raw_fd_ostream OS(FD, true, /*unbuffered=*/true); + OS << Response.Body; + OS.close(); + streamFile(Request, TempFilePath); +}; + +// Test streaming back chunks of a file. +TEST(HTTPClientServer, StreamingFileResponse) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, TempFileStreamingHandler), + Succeeded()); + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Deletes the temporary file before streaming it back, should give a 404 not +// found status code. +HTTPRequestHandler MissingTempFileStreamingHandler = + [](HTTPServerRequest Request) { + int FD; + SmallString<64> TempFilePath; + sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD, + TempFilePath); + raw_fd_ostream OS(FD, true, /*unbuffered=*/true); + OS << Response.Body; + OS.close(); + // delete the file + sys::fs::remove(TempFilePath); + streamFile(Request, TempFilePath); + }; + +// Streaming a missing file should give a 404. +TEST(HTTPClientServer, StreamingMissingFileResponse) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, MissingTempFileStreamingHandler), + Succeeded()); + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, 404u); + Server.stop(); +} + +TEST(HTTPClientServer, ClientTimeout) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, DelayHandler), Succeeded()); + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + HTTPClient Client; + // Timeout below 50ms, request should fail + Client.setTimeout(std::chrono::milliseconds(40)); + Expected BufferOrErr = Client.get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Failed()); + Server.stop(); +} + +// Check that Url paths are dispatched to the first matching handler and provide +// the correct path pattern match components. +TEST(HTTPClientServer, PathMatching) { + HTTPServer Server; + + EXPECT_THAT_ERROR( + Server.get(R"(/abc/(.*)/(.*))", + [&](HTTPServerRequest &Request) { + EXPECT_EQ(Request.UrlPath, "/abc/1/2"); + ASSERT_THAT(Request.UrlPathMatches, + testing::ElementsAre("1", "2")); + Request.setResponse({200u, "text/plain", Request.UrlPath}); + }), + Succeeded()); + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, + [&](HTTPServerRequest &Request) { + llvm_unreachable( + "Should not reach this handler"); + Handler(Request); + }), + Succeeded()); + + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), "/abc/1/2"); + Server.stop(); +} + +TEST(HTTPClientServer, FirstPathMatched) { + HTTPServer Server; + + EXPECT_THAT_ERROR( + Server.get(UrlPathPattern, + [&](HTTPServerRequest Request) { Handler(Request); }), + Succeeded()); + + EXPECT_THAT_ERROR( + Server.get(R"(/abc/(.*)/(.*))", + [&](HTTPServerRequest Request) { + EXPECT_EQ(Request.UrlPathMatches.size(), 2u); + llvm_unreachable("Should not reach this handler"); + Request.setResponse({200u, "text/plain", Request.UrlPath}); + }), + Succeeded()); + + Expected PortOrErr = Server.bind(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +#endif + +#else + +TEST(HTTPServer, IsAvailable) { EXPECT_FALSE(HTTPServer::isAvailable()); } + +#endif // LLVM_ENABLE_HTTPLIB