Index: llvm/include/llvm/Support/HTTPServer.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/HTTPServer.h @@ -0,0 +1,104 @@ +//===-- llvm/Support/HTTPServer.h - HTTP server library ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the HTTPServer class, the +/// HTTPServerRequest, HTTPResponse, and StreamingHTTPResponse structs, and the +/// streamFileResponse function. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HTTP_SERVER_H +#define LLVM_SUPPORT_HTTP_SERVER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#ifdef LLVM_ENABLE_HTTPLIB +#include "httplib.h" +#endif + +namespace llvm { + +struct HTTPServerRequest { + /// The first element is the entire url path, and the rest of the elements + /// correspond to match groups in the url path matching regex. + SmallVector UrlPathMatches; + + // TODO bring in HTTP headers +}; + +struct HTTPResponse { + unsigned Code; + std::string ContentType; + std::string Body; +}; + +typedef std::function HTTPRequestHandler; + +/// An HTTPContentProvider is called by the HTTPServer to obtain chunks of the +/// streaming response body. The returned chunk should be located at Offset +/// bytes and have Length bytes. +typedef std::function + HTTPContentProvider; + +/// Wraps the content provider with HTTP Status code and headers. +struct StreamingHTTPResponse { + unsigned Code; + std::string ContentType; + size_t ContentLength; + HTTPContentProvider Provider; + /// Called after the response transfer is complete with the success value of + /// the transfer. + std::function CompletionHandler = [](bool Success) {}; +}; + +/// Returns a simple streaming response which streams chunks of the file at +/// FilePath, if available, and otherwise returns an HTTP 404 error status code. +StreamingHTTPResponse streamFileResponse(StringRef FilePath); + +typedef std::function + StreamingHTTPRequestHandler; + +/// An HTTP server which can listen on a single TCP/IP port for HTTP requests +/// and delgate them to the appropriate registered handler. +class HTTPServer { +#ifdef LLVM_ENABLE_HTTPLIB + httplib::Server Server; + unsigned Port = 0; +#endif +public: + ~HTTPServer(); + + /// Returns true only if LLVM has been compiled with a working HTTPServer. + static bool isAvailable(); + + /// Registers a URL pattern routing rule. When the server is listening, each + /// request is dispatched to the first registered handler whose UrlPathPattern + /// matches the UrlPath. + Error get(StringRef UrlPathPattern, HTTPRequestHandler Handler); + Error get(StringRef UrlPathPattern, StreamingHTTPRequestHandler Handler); + + /// Attempts to assign the requested port and interface, returning an Error + /// upon failure. + Error bind(unsigned Port, StringRef HostInterface = "0.0.0.0"); + + /// Attempts to assign any available port and interface, returning either the + /// port number or an Error upon failure. + Expected bindAny(StringRef HostInterface = "0.0.0.0"); + + /// Attempts to listen for requests on the bound port. Returns an Error if + /// called before binding a port. + Error listen(); + + /// If the server is listening, stop and unbind the socket. + void stop(); +}; +} // end namespace llvm + +#endif // LLVM_SUPPORT_HTTP_SERVER_H Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -161,6 +161,7 @@ GraphWriter.cpp Hashing.cpp HTTPClient.cpp + HTTPServer.cpp InitLLVM.cpp InstructionCost.cpp IntEqClasses.cpp Index: llvm/lib/Support/HTTPServer.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/HTTPServer.cpp @@ -0,0 +1,163 @@ +//===-- llvm/Support/HTTPServer.cpp - HTTP server library -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the methods of the HTTPServer class. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPServer.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Regex.h" + +using namespace llvm; + +StreamingHTTPResponse llvm::streamFileResponse(StringRef FilePath) { + Expected FDOrErr = sys::fs::openNativeFileForRead(FilePath); + if (Error Err = FDOrErr.takeError()) { + std::string ResponseStr = "Could not open file to read.\n"; + consumeError(std::move(Err)); + return {404u, "text/plain", ResponseStr.size(), + [=](size_t Offset, size_t Length) { + return ResponseStr.substr(Offset, Length); + }}; + } + ErrorOr> MBOrErr = + MemoryBuffer::getOpenFile(*FDOrErr, FilePath, + /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + sys::fs::closeFile(*FDOrErr); + if (Error Err = errorCodeToError(MBOrErr.getError())) { + std::string ResponseStr = "Could not memory-map file.\n"; + consumeError(std::move(Err)); + return {404u, "text/plain", ResponseStr.size(), + [=](size_t Offset, size_t Length) { + return ResponseStr.substr(Offset, Length); + }}; + } + // Lambdas are copied on conversion to to std::function, preventing use of + // smart pointers. + MemoryBuffer *MB = MBOrErr->release(); + return {200u, "application/octet-stream", MB->getBufferSize(), + [=](size_t Offset, size_t Length) -> std::string { + std::string Chunk = + std::string(MB->getBuffer().substr(Offset, Length)); + return Chunk; + }, + [=](bool Success) { delete MB; }}; +} + +#ifdef LLVM_ENABLE_HTTPLIB + +bool HTTPServer::isAvailable() { return true; } + +HTTPServer::~HTTPServer() { stop(); } + +Error HTTPServer::get(StringRef UrlPathPattern, HTTPRequestHandler Handler) { + std::string ErrorMessage; + if (!Regex(UrlPathPattern).isValid(ErrorMessage)) + return createStringError(errc::argument_out_of_domain, ErrorMessage); + Server.Get(std::string(UrlPathPattern), + [&](const httplib::Request &HTTPLibRequest, + httplib::Response &HTTPLibResponse) { + HTTPServerRequest Request; + for (auto PathComponent : HTTPLibRequest.matches) + Request.UrlPathMatches.push_back(std::string(PathComponent)); + HTTPResponse Response = Handler(Request); + HTTPLibResponse.set_content(Response.Body.c_str(), + Response.ContentType.c_str()); + return HTTPLibResponse.status = Response.Code; + }); + return Error::success(); +} + +Error HTTPServer::get(StringRef UrlPathPattern, + StreamingHTTPRequestHandler Handler) { + std::string ErrorMessage; + if (!Regex(UrlPathPattern).isValid(ErrorMessage)) + return createStringError(errc::argument_out_of_domain, ErrorMessage); + Server.Get(std::string(UrlPathPattern), + [&](const httplib::Request &HTTPLibRequest, + httplib::Response &HTTPLibResponse) { + HTTPServerRequest Request; + for (auto PathComponent : HTTPLibRequest.matches) + Request.UrlPathMatches.push_back(std::string(PathComponent)); + StreamingHTTPResponse Response = Handler(Request); + HTTPLibResponse.set_content_provider( + Response.ContentLength, Response.ContentType.c_str(), + [=](size_t Offset, size_t Length, httplib::DataSink &Sink) { + if (Offset < Response.ContentLength) { + std::string Chunk = Response.Provider(Offset, Length); + Sink.write(Chunk.c_str(), Chunk.size()); + } + return true; + }, + [=](bool Success) { Response.CompletionHandler(Success); }); + return HTTPLibResponse.status = Response.Code; + }); + return Error::success(); +} + +Error HTTPServer::bind(unsigned ListenPort, StringRef HostInterface) { + SmallString<16> HostInterfaceStorage; + StringRef S = + Twine(HostInterface).toNullTerminatedStringRef(HostInterfaceStorage); + + if (!Server.bind_to_port(S.begin(), ListenPort)) + return createStringError(errc::io_error, + "Could not assign requested address."); + + Port = ListenPort; + return Error::success(); +} + +Expected HTTPServer::bindAny(StringRef HostInterface) { + SmallString<16> HostInterfaceStorage; + StringRef S = + Twine(HostInterface).toNullTerminatedStringRef(HostInterfaceStorage); + + int ListenPort = Server.bind_to_any_port(S.begin()); + if (ListenPort < 0) + return createStringError(errc::io_error, + "Could not assign any port on requested address."); + return Port = ListenPort; +} + +Error HTTPServer::listen() { + if (!Port) + return createStringError(errc::io_error, + "Cannot listen without first binding to a port."); + if (!Server.listen_after_bind()) + return createStringError( + errc::io_error, + "An unknown error occurred when cpp-httplib attempted to listen."); + return Error::success(); +} + +void HTTPServer::stop() { + Server.stop(); + Port = 0; +} + +#else + +// TODO: Implement barebones standalone HTTP server implementation. +bool HTTPServer::isAvailable() { return false; } + +HTTPServer::HTTPServer( + std::function StreamingRequestHandler) { + llvm_unreachable( + "Attempt to instantiate HTTPServer when no implementation is available."); +} +#endif Index: llvm/unittests/Support/CMakeLists.txt =================================================================== --- llvm/unittests/Support/CMakeLists.txt +++ llvm/unittests/Support/CMakeLists.txt @@ -42,6 +42,7 @@ HashBuilderTest.cpp Host.cpp HTTPClient.cpp + HTTPServer.cpp IndexedAccessorTest.cpp InstructionCostTest.cpp ItaniumManglingCanonicalizerTest.cpp Index: llvm/unittests/Support/HTTPServer.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/HTTPServer.cpp @@ -0,0 +1,271 @@ +//===-- llvm/unittest/Support/HTTPServer.cpp - unit tests -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPServer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; + +#ifdef LLVM_ENABLE_HTTPLIB + +TEST(HTTPServer, IsAvailable) { EXPECT_TRUE(HTTPServer::isAvailable()); } + +HTTPResponse Response = {200u, "text/plain", "hello, world\n"}; +std::string UrlPathPattern = R"(/(.*))"; +std::string InvalidUrlPathPattern = R"(/(.*)"; + +HTTPRequestHandler Handler = [](HTTPServerRequest) -> HTTPResponse { + return Response; +}; + +HTTPRequestHandler DelayHandler = [](HTTPServerRequest) -> HTTPResponse { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + return Response; +}; + +StreamingHTTPRequestHandler StreamingHandler = + [](HTTPServerRequest Request) -> StreamingHTTPResponse { + return {200, "text/plain", Response.Body.size(), + [=](size_t Offset, size_t Length) -> std::string { + return Response.Body.substr(Offset, Length); + }}; +}; + +TEST(HTTPServer, InvalidUrlPath) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(InvalidUrlPathPattern, Handler), + Failed()); + EXPECT_THAT_EXPECTED(Server.bindAny(), Succeeded()); +} + +TEST(HTTPServer, BindAny) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + EXPECT_THAT_EXPECTED(Server.bindAny(), Succeeded()); +} + +TEST(HTTPServer, ListenBeforeBind) { + // test that we can bind to any address + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + EXPECT_THAT_ERROR(Server.listen(), Failed()); +} + +#ifdef LLVM_ENABLE_CURL +// Test the client and server against each other. + +TEST(HTTPClientServer, Hello) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded()); + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Test the streaming response. +TEST(HTTPClientServer, StreamingHello) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, StreamingHandler), Succeeded()); + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Writes a temporary file and streams it back using streamFileResponse. +StreamingHTTPRequestHandler TempFileStreamingHandler = + [](HTTPServerRequest Request) -> StreamingHTTPResponse { + int FD; + SmallString<64> TempFilePath; + sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD, + TempFilePath); + raw_fd_ostream OF(FD, true, /*unbuffered=*/true); + OF << Response.Body; + OF.close(); + return streamFileResponse(TempFilePath); +}; + +// Test streaming back chunks of a file. +TEST(HTTPClientServer, StreamingFileResponse) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, TempFileStreamingHandler), + Succeeded()); + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Deletes the temporary file before streaming it back, should give a 404 not +// found status code. +StreamingHTTPRequestHandler MissingTempFileStreamingHandler = + [](HTTPServerRequest Request) -> StreamingHTTPResponse { + int FD; + SmallString<64> TempFilePath; + sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD, + TempFilePath); + raw_fd_ostream OF(FD, true, /*unbuffered=*/true); + OF << Response.Body; + OF.close(); + // delete the file + sys::fs::remove(TempFilePath); + return streamFileResponse(TempFilePath); +}; + +// Streaming a missing file should give a 404. +TEST(HTTPClientServer, StreamingMissingFileResponse) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, MissingTempFileStreamingHandler), + Succeeded()); + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, 404u); + Server.stop(); +} + +TEST(HTTPClientServer, ClientTimeout) { + HTTPServer Server; + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, DelayHandler), Succeeded()); + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port); + HTTPClient::initialize(); + HTTPClient Client; + // Timeout below 50ms, request should fail + Client.setTimeout(40); + Expected BufferOrErr = Client.get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Failed()); + // Timeout above 50ms, request should succeed + Client.setTimeout(60); + BufferOrErr = Client.get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +// Check that Url paths are dispatched to the first matching handler and provide +// the correct path pattern match components. +TEST(HTTPClientServer, PathMatching) { + HTTPServer Server; + + EXPECT_THAT_ERROR( + Server.get(R"(/abc/(.*)/(.*))", + [&](HTTPServerRequest Request) -> HTTPResponse { + EXPECT_EQ(Request.UrlPathMatches.size(), 3u); + EXPECT_EQ(Request.UrlPathMatches[0], "/abc/1/2"); + EXPECT_EQ(Request.UrlPathMatches[1], "1"); + EXPECT_EQ(Request.UrlPathMatches[2], "2"); + return {200u, "text/plain", Request.UrlPathMatches[0]}; + }), + Succeeded()); + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, + [&](HTTPServerRequest Request) -> HTTPResponse { + assert(false && + "Should not reach this handler"); + return Handler(Request); + }), + Succeeded()); + + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), "/abc/1/2"); + Server.stop(); +} + +TEST(HTTPClientServer, FirstPathMatched) { + HTTPServer Server; + + EXPECT_THAT_ERROR(Server.get(UrlPathPattern, + [&](HTTPServerRequest Request) -> HTTPResponse { + return Handler(Request); + }), + Succeeded()); + + EXPECT_THAT_ERROR( + Server.get(R"(/abc/(.*)/(.*))", + [&](HTTPServerRequest Request) -> HTTPResponse { + EXPECT_EQ(Request.UrlPathMatches.size(), 3u); + assert(false && "Should not reach this handler"); + return {200u, "text/plain", Request.UrlPathMatches[0]}; + }), + Succeeded()); + + Expected PortOrErr = Server.bindAny(); + EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); + unsigned &Port = *PortOrErr; + ThreadPool Pool(hardware_concurrency(1)); + Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); + std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; + HTTPClient::initialize(); + Expected BufferOrErr = HTTPClient().get(Url); + EXPECT_THAT_EXPECTED(BufferOrErr, Succeeded()); + HTTPResponseBuffer &Buffer = *BufferOrErr; + EXPECT_EQ(Buffer.Code, Response.Code); + EXPECT_EQ(Buffer.Body->MemoryBuffer::getBuffer(), Response.Body); + Server.stop(); +} + +#endif + +#endif