Index: llvm/include/llvm/Support/HTTPClient.h =================================================================== --- /dev/null +++ llvm/include/llvm/Support/HTTPClient.h @@ -0,0 +1,110 @@ +//===-- llvm/Support/HTTPClient.h - HTTP client library ---*- C++ -------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of the HTTPClient, HTTPMethod, +/// HTTPResponseHandler, and BufferedHTTPResponseHandler classes, as well as +/// the HTTPResponseBuffer and HTTPRequest structs. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HTTP_CLIENT_H +#define LLVM_SUPPORT_HTTP_CLIENT_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +enum class HTTPMethod { GET }; + +/// A stateless description of the configuration of an outbound HTTP Request. +/// An HTTPRequest can be performed any number of times by HTTPClient::perform. +struct HTTPRequest { + SmallString<128> Url; + HTTPMethod Method = HTTPMethod::GET; + bool FollowRedirects = true; + bool operator==(const HTTPRequest &O) const { + return Url == O.Url && Method == O.Method && + FollowRedirects == O.FollowRedirects; + } + HTTPRequest(const Twine &Url); +}; + +/// A handler for state updates during the lifecycle of a request performed +/// by HTTPClient::perform. An HTTP ResponseHandler cannot fail, +class HTTPResponseHandler { +public: + /// Processes one line of HTTP response headers and returns the number of + /// bytes handled. Returning 0 causes the request to abort. + virtual size_t handleHeaderLine(StringRef HeaderLine) = 0; + + /// Processes an additional chunk of bytes a position Offset in the HTTP + /// response body and returns the number of bytes handled. + /// Returning 0 causes the request to abort. + virtual size_t handleBodyChunk(StringRef BodyChunk, size_t Offset) = 0; + + /// Processes the HTTP response status code. + virtual void handleStatusCode(unsigned Code) = 0; + + virtual ~HTTPResponseHandler(); +}; + +/// An HTTP response status code bundled with a buffer which can store the +/// body. +struct HTTPResponseBuffer { + unsigned Code = 0; + std::unique_ptr Body; +}; + +/// A simple handler which writes returned data to an HTTPResponseBuffer. +/// Discards all headers except the Content-Length, which it uses to +/// allocate an appropriately-sized Body buffer. +class BufferedHTTPResponseHandler : public HTTPResponseHandler { +public: + /// Tracks any errors which have occurred during request handling, such as + /// if handleBodyChunk is called before a Content-Length header. + Error Err = Error::success(); + + /// Stores the data received from the HTTP server. + HTTPResponseBuffer ResponseBuffer; + + /// Implementations of the HTTPResponseHandler functions to store the + /// HTTP response in a buffer allocated based on Content-Length. + /// Errors during handling are placed in this->Err. + size_t handleHeaderLine(StringRef HeaderLine) override; + size_t handleBodyChunk(StringRef BodyChunk, size_t Offset) override; + void handleStatusCode(unsigned Code) override; +}; + +/// A reusable client for performing outbound HTTP requests. +class HTTPClient { +public: + /// Children of HTTPClient are expected to clean up their resources + /// upon destruction. + virtual ~HTTPClient(); + + /// Performs the Request and passes response data to the Handler. + /// Returns any errors which occur during request. + /// If any of the callbacks to Handler return an Error, a Client + /// should stop performing the request and return an Error. + virtual Error perform(const HTTPRequest Request, + HTTPResponseHandler &Handler) = 0; + + /// Performs the Request with the default BufferedHTTPResponseHandler, + /// and returns its HTTPResponseBuffer or an Error. + Expected perform(const HTTPRequest Request); + + /// Performs an HTTPRequest with the default configuration to make a GET + /// request to the given Url. Returns an HTTPResponseBuffer or an Error. + Expected get(const Twine &Url); +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_HTTP_CLIENT_H Index: llvm/lib/Support/CMakeLists.txt =================================================================== --- llvm/lib/Support/CMakeLists.txt +++ llvm/lib/Support/CMakeLists.txt @@ -155,6 +155,7 @@ GlobPattern.cpp GraphWriter.cpp Hashing.cpp + HTTPClient.cpp InitLLVM.cpp InstructionCost.cpp IntEqClasses.cpp Index: llvm/lib/Support/HTTPClient.cpp =================================================================== --- /dev/null +++ llvm/lib/Support/HTTPClient.cpp @@ -0,0 +1,91 @@ +//===-- llvm/Support/HTTPClient.cpp - HTTP client library -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the methods of the HTTPRequest, HTTPClient, and +/// BufferedHTTPResponseHandler classes. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPClient.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "HTTPClient" + +HTTPRequest::HTTPRequest(const Twine &Url) { this->Url = Url.str(); } + +HTTPResponseHandler::~HTTPResponseHandler() {} + +HTTPClient::~HTTPClient() {} + +Expected HTTPClient::perform(const HTTPRequest Request) { + BufferedHTTPResponseHandler Handler; + if (Error Err = joinErrors(perform(Request, Handler), std::move(Handler.Err))) + return Err; + + return std::move(Handler.ResponseBuffer); +} + +Expected HTTPClient::get(const Twine &Url) { + HTTPRequest Request(Url); + return perform(Request); +} + +static bool parseContentLengthHeader(StringRef LineRef, + unsigned long long &ContentLength) { + // Content-Length is a mandatory header, and the only one we handle. + return LineRef.consume_front("Content-Length: ") && + !getAsUnsignedInteger(LineRef.trim(), 10, ContentLength); +} + +size_t BufferedHTTPResponseHandler::handleHeaderLine(StringRef HeaderLine) { + if (ResponseBuffer.Body) + return HeaderLine.size(); + + unsigned long long ContentLength; + if (parseContentLengthHeader(HeaderLine, ContentLength)) + ResponseBuffer.Body = + WritableMemoryBuffer::getNewUninitMemBuffer(ContentLength); + + return HeaderLine.size(); +} + +size_t BufferedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk, + size_t Offset) { + if (!ResponseBuffer.Body) { + Err = joinErrors( + std::move(Err), + createStringError(errc::io_error, + "Unallocated response buffer -- HTTP Body data " + "recieved before Content-Length Header.")); + return 0; + } + + if (Offset + BodyChunk.size() > ResponseBuffer.Body->getBufferSize()) { + Err = joinErrors( + std::move(Err), + createStringError(errc::io_error, + "Content is larger than response buffer.")); + return 0; + } + + memcpy(ResponseBuffer.Body->getBufferStart() + Offset, BodyChunk.data(), + BodyChunk.size()); + return BodyChunk.size(); +} + +void BufferedHTTPResponseHandler::handleStatusCode(unsigned Code) { + ResponseBuffer.Code = Code; +} Index: llvm/unittests/Support/CMakeLists.txt =================================================================== --- llvm/unittests/Support/CMakeLists.txt +++ llvm/unittests/Support/CMakeLists.txt @@ -41,6 +41,7 @@ GlobPatternTest.cpp HashBuilderTest.cpp Host.cpp + HTTPClient.cpp IndexedAccessorTest.cpp InstructionCostTest.cpp ItaniumManglingCanonicalizerTest.cpp Index: llvm/unittests/Support/HTTPClient.cpp =================================================================== --- /dev/null +++ llvm/unittests/Support/HTTPClient.cpp @@ -0,0 +1,188 @@ +//===-- llvm/unittest/Support/HTTPClient.cpp - unit tests -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/Errc.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerLifecycleTest) { + { + BufferedHTTPResponseHandler Handler; + // test initial state + EXPECT_EQ(Handler.ResponseBuffer.Code, 0u); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + // A body chunk passed before the content-length header is an error. + // The handler should return 0 and set its Err. + EXPECT_EQ(Handler.handleBodyChunk("body", 0u), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Failed()); + + EXPECT_EQ(Handler.handleHeaderLine("a header line"), 13u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + + EXPECT_EQ(Handler.handleBodyChunk("body", 4u), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Failed()); + } + + BufferedHTTPResponseHandler Handler; + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: 36\r\n"), 20u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(Handler.handleBodyChunk("body:", 0), 5u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(Handler.handleBodyChunk("this puts the total at 36 chars", 5), 31u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(memcmp(Handler.ResponseBuffer.Body->getBufferStart(), + "body:this puts the total at 36 chars", + Handler.ResponseBuffer.Body->getBufferSize()), + 0); + + // additional content should be rejected by the handler + EXPECT_EQ( + Handler.handleBodyChunk("extra content past the content-length", 36), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Failed()); + + // response code handling + Handler.handleStatusCode(200u); + EXPECT_EQ(Handler.ResponseBuffer.Code, 200u); +} + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerZeroContentLengthTest) { + BufferedHTTPResponseHandler Handler; + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: 0"), 17u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_NE(Handler.ResponseBuffer.Body, nullptr); + EXPECT_EQ(Handler.ResponseBuffer.Body->getBufferSize(), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + + // all content should be rejected by the handler + EXPECT_EQ(Handler.handleBodyChunk("non-empty body content", 0), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Failed()); +} + +TEST(HTTPClientTests, bufferedHTTPResponseHandlerMalformedContentLengthTest) { + // confirm getAsUnsignedInteger behaves as expected + // in particular, getAsUnsignedInteger returns false + // for valid inputs + unsigned long long ContentLength; + EXPECT_EQ(llvm::getAsUnsignedInteger("fff", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("-1", 10, ContentLength), true); + EXPECT_EQ(llvm::getAsUnsignedInteger("0111", 10, ContentLength), false); + EXPECT_EQ(llvm::getAsUnsignedInteger("0", 10, ContentLength), false); + EXPECT_EQ(ContentLength, 0u); + + // check several invalid content lengths are ignored + BufferedHTTPResponseHandler Handler; + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: fff"), 19u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: "), 19u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + EXPECT_EQ(Handler.handleHeaderLine(StringRef("Content-Length: \0\0\0", 19)), + 19u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Succeeded()); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: -11"), 19u); + EXPECT_EQ(Handler.ResponseBuffer.Body, nullptr); + + // all content should be rejected by the handler because + // no valid content length has been received + EXPECT_EQ(Handler.handleBodyChunk("non-empty body content", 0), 0u); + EXPECT_THAT_ERROR(std::move(Handler.Err), Failed()); +} + +class MockHTTPResponseHandler : public HTTPResponseHandler { +public: + size_t BodyBytesTotal = 0; + size_t HeaderLinesTotal = 0; + unsigned StatusCode = 0; + + size_t handleHeaderLine(StringRef HeaderLine) { + HeaderLinesTotal++; + return HeaderLine.size(); + }; + + size_t handleBodyChunk(StringRef BodyChunk, size_t Offset) { + BodyBytesTotal += BodyChunk.size(); + return BodyChunk.size(); + }; + + void handleStatusCode(unsigned Code) { StatusCode = Code; }; +}; + +TEST(HTTPClientTests, simulatedHTTPResponseHandlerTest) { + MockHTTPResponseHandler Handler; + EXPECT_EQ(Handler.HeaderLinesTotal, 0u); + EXPECT_EQ(Handler.BodyBytesTotal, 0u); + EXPECT_EQ(Handler.StatusCode, 0u); + Handler.handleStatusCode(200); + EXPECT_EQ(Handler.handleHeaderLine("Content-Length: 0"), 17u); + EXPECT_EQ(Handler.handleHeaderLine("Another-Header: foo"), 19u); + EXPECT_EQ(Handler.handleBodyChunk("1234567", 0), 7u); + EXPECT_EQ(Handler.HeaderLinesTotal, 2u); + EXPECT_EQ(Handler.BodyBytesTotal, 7u); + EXPECT_EQ(Handler.StatusCode, 200u); +} + +class MockHTTPClient : public HTTPClient { +public: + using HTTPClient::perform; + Error perform(const HTTPRequest Request, + HTTPResponseHandler &Handler) override { + Handler.handleStatusCode(200); + if (Handler.handleHeaderLine("Content-Length: 7") != 17) + return createStringError(errc::io_error, + "Handler failed handling header"); + if (Handler.handleBodyChunk("abcdefg", 0) != 7) + return createStringError(errc::io_error, "Handler failed to handle body"); + + return Error::success(); + } +}; + +TEST(HTTPClientTests, mockHTTPClientTest) { + MockHTTPResponseHandler Handler; + MockHTTPClient Client; + HTTPRequest Request(""); + EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded()); + EXPECT_EQ(Handler.HeaderLinesTotal, 1u); + EXPECT_EQ(Handler.BodyBytesTotal, 7u); +} + +class MockHTTPClientBrokenConnection : public HTTPClient { +public: + using HTTPClient::perform; + Error perform(const HTTPRequest Request, + HTTPResponseHandler &Handler) override { + Handler.handleStatusCode(200); + if (Handler.handleHeaderLine("Content-Length: 17") != 18) + return createStringError(errc::io_error, + "Handler failed handling header"); + if (Handler.handleBodyChunk("abcdefg", 0) != 7) + return createStringError(errc::io_error, "Handler failed to handle body"); + return createStringError(errc::io_error, "Simulated network failure"); + } +}; + +TEST(HTTPClientTests, mockHTTPClientBrokenConnectionTest) { + MockHTTPResponseHandler Handler; + MockHTTPClientBrokenConnection Client; + HTTPRequest Request(""); + EXPECT_THAT_ERROR(Client.perform(Request, Handler), + Failed()); + EXPECT_THAT_EXPECTED(Client.perform(Request), Failed()); + EXPECT_THAT_EXPECTED(Client.get(""), Failed()); +}