[clangd] Transport abstraction WIP

[sammccall]

Summary

Ideas about abstracting JSON transport away to allow XPC and improve layering.

clangd/ClangdLSPServer.h

 
 #include "ClangdServer.h"
 #include "DraftStore.h"
 #include "FindSymbols.h"
 #include "GlobalCompilationDatabase.h"
 #include "Path.h"
 #include "Protocol.h"
 #include "ProtocolHandlers.h"
+#include "Transport.h"
 #include "clang/Tooling/Core/Replacement.h"
 #include "llvm/ADT/Optional.h"
 
 namespace clang {
 namespace clangd {
 
-class JSONOutput;
 class SymbolIndex;
 
 /// This class provides implementation of an LSP server, glueing the JSON
 /// dispatch and ClangdServer together.
-class ClangdLSPServer : private DiagnosticsConsumer, private ProtocolCallbacks {
+class ClangdLSPServer : private DiagnosticsConsumer {
 public:
   /// If \p CompileCommandsDir has a value, compile_commands.json will be
   /// loaded only from \p CompileCommandsDir. Otherwise, clangd will look
   /// for compile_commands.json in all parent directories of each file.
-  ClangdLSPServer(JSONOutput &Out, const clangd::CodeCompleteOptions &CCOpts,
+  ClangdLSPServer(Transport &Transport,
+                  const clangd::CodeCompleteOptions &CCOpts,
                   llvm::Optional<Path> CompileCommandsDir,
                   const ClangdServer::Options &Opts);
 
-  /// Run LSP server loop, receiving input for it from \p In. \p In must be
+  /// Run LSP server loop, receiving input for it from the transport.
   /// opened in binary mode. Output will be written using Out variable passed to
   /// class constructor. This method must not be executed more than once for
   /// each instance of ClangdLSPServer.
   ///
   /// \return Whether we received a 'shutdown' request before an 'exit' request.
-  bool run(std::FILE *In,
-           JSONStreamStyle InputStyle = JSONStreamStyle::Standard);
+  bool run();
 
 private:
   // Implement DiagnosticsConsumer.
   void onDiagnosticsReady(PathRef File, std::vector<Diag> Diagnostics) override;
 
-  // Implement ProtocolCallbacks.
-  void onInitialize(InitializeParams &Params) override;
-  void onShutdown(ShutdownParams &Params) override;
-  void onExit(ExitParams &Params) override;
-  void onDocumentDidOpen(DidOpenTextDocumentParams &Params) override;
-  void onDocumentDidChange(DidChangeTextDocumentParams &Params) override;
-  void onDocumentDidClose(DidCloseTextDocumentParams &Params) override;
+  // Implement LSP methods.
+  void onInitialize(InitializeParams &Params);
+  void onShutdown(ShutdownParams &Params);
+  void onExit(ExitParams &Params);
+  void onDocumentDidOpen(DidOpenTextDocumentParams &Params);
+  void onDocumentDidChange(DidChangeTextDocumentParams &Params);
+  void onDocumentDidClose(DidCloseTextDocumentParams &Params);
   void
-  onDocumentOnTypeFormatting(DocumentOnTypeFormattingParams &Params) override;
+  onDocumentOnTypeFormatting(DocumentOnTypeFormattingParams &Params);
   void
-  onDocumentRangeFormatting(DocumentRangeFormattingParams &Params) override;
-  void onDocumentFormatting(DocumentFormattingParams &Params) override;
-  void onDocumentSymbol(DocumentSymbolParams &Params) override;
-  void onCodeAction(CodeActionParams &Params) override;
-  void onCompletion(TextDocumentPositionParams &Params) override;
-  void onSignatureHelp(TextDocumentPositionParams &Params) override;
-  void onGoToDefinition(TextDocumentPositionParams &Params) override;
-  void onSwitchSourceHeader(TextDocumentIdentifier &Params) override;
-  void onDocumentHighlight(TextDocumentPositionParams &Params) override;
-  void onFileEvent(DidChangeWatchedFilesParams &Params) override;
-  void onCommand(ExecuteCommandParams &Params) override;
-  void onWorkspaceSymbol(WorkspaceSymbolParams &Params) override;
-  void onRename(RenameParams &Parames) override;
-  void onHover(TextDocumentPositionParams &Params) override;
-  void onChangeConfiguration(DidChangeConfigurationParams &Params) override;
+  onDocumentRangeFormatting(DocumentRangeFormattingParams &Params);
+  void onDocumentFormatting(DocumentFormattingParams &Params);
+  void onDocumentSymbol(DocumentSymbolParams &Params);
+  void onCodeAction(CodeActionParams &Params);
+  void onCompletion(TextDocumentPositionParams &Params);
+  void onSignatureHelp(TextDocumentPositionParams &Params);
+  void onGoToDefinition(TextDocumentPositionParams &Params);
+  void onSwitchSourceHeader(TextDocumentIdentifier &Params);
+  void onDocumentHighlight(TextDocumentPositionParams &Params);
+  void onFileEvent(DidChangeWatchedFilesParams &Params);
+  void onCommand(ExecuteCommandParams &Params);
+  void onWorkspaceSymbol(WorkspaceSymbolParams &Params);
+  void onRename(RenameParams &Parames);
+  void onHover(TextDocumentPositionParams &Params);
+  void onChangeConfiguration(DidChangeConfigurationParams &Params);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
   /// Forces a reparse of all currently opened files.  As a result, this method
   /// may be very expensive.  This method is normally called when the
   /// compilation database is changed.
   void reparseOpenedFiles();
 
-  JSONOutput &Out;
+  Transport &Transport;
   /// Used to indicate that the 'shutdown' request was received from the
   /// Language Server client.
   bool ShutdownRequestReceived = false;
 
-  /// Used to indicate that the 'exit' notification was received from the
-  /// Language Server client.
-  /// It's used to break out of the LSP parsing loop.
-  bool IsDone = false;
-
   std::mutex FixItsMutex;
   typedef std::map<clangd::Diagnostic, std::vector<Fix>, LSPDiagnosticCompare>
       DiagnosticToReplacementMap;
   /// Caches FixIts per file and diagnostics
   llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
 
   // Various ClangdServer parameters go here. It's important they're created
   // before ClangdServer.

clangd/ClangdLSPServer.cpp

if (Settings.compilationDatabasePath.hasValue()) {
     NonCachedCDB.setCompileCommandsDir(
         Settings.compilationDatabasePath.getValue());
     CDB.clear();
 
     reparseOpenedFiles();
   }
 }
 
-ClangdLSPServer::ClangdLSPServer(JSONOutput &Out,
+ClangdLSPServer::ClangdLSPServer(Transport &Transport,
                                  const clangd::CodeCompleteOptions &CCOpts,
                                  llvm::Optional<Path> CompileCommandsDir,
                                  const ClangdServer::Options &Opts)
-    : Out(Out), NonCachedCDB(std::move(CompileCommandsDir)), CDB(NonCachedCDB),
+    : Transport(Transport), NonCachedCDB(std::move(CompileCommandsDir)), CDB(NonCachedCDB),
       CCOpts(CCOpts), SupportedSymbolKinds(defaultSymbolKinds()),
       Server(CDB, FSProvider, /*DiagConsumer=*/*this, Opts) {}
 
-bool ClangdLSPServer::run(std::FILE *In, JSONStreamStyle InputStyle) {
+bool ClangdLSPServer::run() {
+  auto Error = Transport.loop([&](json::Value Message) {
+    // XXX
+  });
+  if (Error)
+    elog("Transport error: {0}", Error);
+  return ShutdownRequestReceived && Error;
   assert(!IsDone && "Run was called before");
 
   // Set up JSONRPCDispatcher.
   JSONRPCDispatcher Dispatcher([](const json::Value &Params) {
     replyError(ErrorCode::MethodNotFound, "method not found");
   });
   registerCallbackHandlers(Dispatcher, /*Callbacks=*/*this);
-
-  // Run the Language Server loop.
-  runLanguageServerLoop(In, Out, InputStyle, Dispatcher, IsDone);
-
-  // Make sure IsDone is set to true after this method exits to ensure assertion
-  // at the start of the method fires if it's ever executed again.
-  IsDone = true;
-
-  return ShutdownRequestReceived;
 }
 
 std::vector<Fix> ClangdLSPServer::getFixes(StringRef File,
                                            const clangd::Diagnostic &D) {
   std::lock_guard<std::mutex> Lock(FixItsMutex);
   auto DiagToFixItsIter = FixItsMap.find(File);
   if (DiagToFixItsIter == FixItsMap.end())
     return {};

clangd/Protocol.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROTOCOL_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PROTOCOL_H
 
 #include "URI.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/JSON.h"
 #include <bitset>
 #include <string>
 #include <vector>
 
 namespace clang {
 namespace clangd {
 
 enum class ErrorCode {
   // Defined by JSON RPC.
   ParseError = -32700,
   InvalidRequest = -32600,
   MethodNotFound = -32601,
   InvalidParams = -32602,
   InternalError = -32603,
 
   ServerNotInitialized = -32002,
   UnknownErrorCode = -32001,
 
   // Defined by the protocol.
   RequestCancelled = -32800,
 };
+struct LSPError : public llvm::ErrorInfo<LSPError> {
+  ErrorCode Code;
+  std::string Message;
+  void log(llvm::raw_ostream& OS) const override { OS << Message; }
+  std::error_code convertToErrorCode() const override {
+    return llvm::inconvertibleErrorCode();
+  }
+};
 
 struct URIForFile {
   URIForFile() = default;
   explicit URIForFile(std::string AbsPath);
 
   /// Retrieves absolute path to the file.
   llvm::StringRef file() const { return File; }
 

clangd/Transport.h

+//===--- Transport.h - sending and receiving LSP messages -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The language server protocol is usually implemented by writing messages as
+// JSON-RPC over the stdin/stdout of a subprocess. However other communications
+// mechanisms are possible, such as XPC on mac (see xpc/ directory).
+//
+// The Transport interface allows the mechanism to be replaced, and the JSONRPC
+// Transport is the standard implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TRANSPORT_H_
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TRANSPORT_H_
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace clang {
+namespace clangd {
+
+// A transport is responsible for maintaining the connection to a client
+// application, and reading/writing structured messages to it.
+//
+// Transports have limited thread safety requirements:
+//  - messages will not be sent concurrently
+//  - messages MAY be sent while loop() is reading, or its callback is active
+class Transport {
+public:
+  virtual ~Transport() = default;
+
+  // Called by Clangd to send messages to the client.
+  // (Because JSON and XPC are so similar, these are concrete and delegate to
+  // sendMessage. We could change this to support more diverse transports).
+  void notify(llvm::StringRef Method, llvm::json::Value Params);
+  void call(llvm::StringRef Method, llvm::json::Value Params,
+            llvm::json::Value ID);
+  void reply(llvm::json::Value ID, llvm::Expected<llvm::json::Value> Result);
+
+  // Implemented by Clangd to handle incoming messages. (See loop() below).
+  class MessageHandler {
+  public:
+    virtual ~MessageHandler() = 0;
+    virtual bool notify(llvm::StringRef Method, llvm::json::Value ) = 0;
+    virtual bool call(llvm::StringRef Method, llvm::json::Value Params,
+                      llvm::json::Value ID) = 0;
+    virtual bool reply(llvm::json::Value ID,
+                       llvm::Expected<llvm::json::Value> Result) = 0;
+  };
+  // Called by Clangd to receive messages from the client.
+  // The transport should in turn invoke the handler to process messages.
+  // If handler returns true, the transport should immedately return success.
+  // Otherwise, it returns an error when the transport becomes unusable.
+  // (Because JSON and XPC are so similar, they share handleMessage()).
+  virtual llvm::Error loop(MessageHandler &) = 0;
+
+protected:
+  // Common implementation for notify(), call(), and reply().
+  virtual void sendMessage(llvm::json::Value) = 0;
+  // Delegates to notify(), call(), and reply().
+  bool handleMessage(llvm::json::Value, MessageHandler&);
+};
+
+// Controls the way JSON-RPC messages are encoded (both input and output).
+enum JSONStreamStyle {
+  // Encoding per the LSP specification, with mandatory Content-Length header.
+  Standard,
+  // Messages are delimited by a '---' line. Comment lines start with #.
+  Delimited
+};
+
+// Returns a Transport that speaks JSON-RPC over a pair of streams.
+// The input stream must be opened in binary mode.
+std::unique_ptr<Transport>
+newJSONTransport(std::FILE *In, llvm::raw_ostream &Out,
+                 JSONStreamStyle = JSONStreamStyle::Standard);
+
+} // namespace clangd
+} // namespace clang
+
+#endif
+
+

clangd/Transport.cpp

+//===--- Transport.cpp - sending and receiving LSP messages -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The JSON-RPC transport is implemented here.
+// The alternative Mac-only XPC transport is in the xpc/ directory.
+//
+//===----------------------------------------------------------------------===//
+#include "Transport.h"
+#include "Logger.h"
+#include "Protocol.h"
+#include "llvm/Support/Errno.h"
+
+using namespace llvm;
+namespace clang {
+namespace clangd {
+
+void Transport::notify(llvm::StringRef Method, llvm::json::Value Params) {
+  sendMessage(json::Object{
+      {"jsonrpc", "2.0"},
+      {"method", Method},
+      {"params", std::move(Params)},
+  });
+}
+void Transport::call(llvm::StringRef Method, llvm::json::Value Params,
+                     llvm::json::Value ID) {
+  sendMessage(json::Object{
+      {"jsonrpc", "2.0"},
+      {"id", std::move(ID)},
+      {"method", Method},
+      {"params", std::move(Params)},
+  });
+}
+void Transport::reply(llvm::json::Value ID,
+                      llvm::Expected<llvm::json::Value> Result) {
+  auto Message = json::Object{
+      {"jsonrpc", "2.0"},
+      {"id", std::move(ID)},
+  };
+  if (Result)
+    Message["result"] = std::move(*Result);
+  else {
+    ErrorCode Code = ErrorCode::UnknownErrorCode;
+    std::string Msg =
+        toString(handleErrors(Result.takeError(), [&](const LSPError &Err) {
+          Code = Err.Code;
+          return make_error<LSPError>(Err); // Recreate error for its message.
+        }));
+    Message["error"] = json::Object{
+        {"message", Msg},
+        {"code", static_cast<int>(Code)},
+    };
+  }
+  sendMessage(std::move(Message));
+}
+
+namespace {
+
+// Tries to read a line up to and including \n.
+// If failing, feof() or ferror() will be set.
+static bool readLine(std::FILE *In, std::string &Out) {
+  static constexpr int BufSize = 1024;
+  size_t Size = 0;
+  Out.clear();
+  for (;;) {
+    Out.resize(Size + BufSize);
+    // Handle EINTR which is sent when a debugger attaches on some platforms.
+    if (!llvm::sys::RetryAfterSignal(nullptr, ::fgets, &Out[Size], BufSize, In))
+      return false;
+    clearerr(In);
+    // If the line contained null bytes, anything after it (including \n) will
+    // be ignored. Fortunately this is not a legal header or JSON.
+    size_t Read = std::strlen(&Out[Size]);
+    if (Read > 0 && Out[Size + Read - 1] == '\n') {
+      Out.resize(Size + Read);
+      return true;
+    }
+    Size += Read;
+  }
+}
+
+// Returns None when:
+//  - ferror() or feof() are set.
+//  - Content-Length is missing or empty (protocol error)
+static llvm::Optional<std::string> readStandardMessage(std::FILE *In) {
+  // A Language Server Protocol message starts with a set of HTTP headers,
+  // delimited  by \r\n, and terminated by an empty line (\r\n).
+  unsigned long long ContentLength = 0;
+  std::string Line;
+  while (true) {
+    if (feof(In) || ferror(In) || !readLine(In, Line))
+      return llvm::None;
+
+    llvm::StringRef LineRef(Line);
+
+    // We allow comments in headers. Technically this isn't part
+    // of the LSP specification, but makes writing tests easier.
+    if (LineRef.startswith("#"))
+      continue;
+
+    // Content-Length is a mandatory header, and the only one we handle.
+    if (LineRef.consume_front("Content-Length: ")) {
+      if (ContentLength != 0) {
+        elog("Warning: Duplicate Content-Length header received. "
+             "The previous value for this message ({0}) was ignored.",
+             ContentLength);
+      }
+      llvm::getAsUnsignedInteger(LineRef.trim(), 0, ContentLength);
+      continue;
+    } else if (!LineRef.trim().empty()) {
+      // It's another header, ignore it.
+      continue;
+    } else {
+      // An empty line indicates the end of headers.
+      // Go ahead and read the JSON.
+      break;
+    }
+  }
+
+  // The fuzzer likes crashing us by sending "Content-Length: 9999999999999999"
+  if (ContentLength > 1 << 30) { // 1024M
+    elog("Refusing to read message with long Content-Length: {0}. "
+         "Expect protocol errors",
+         ContentLength);
+    return llvm::None;
+  }
+  if (ContentLength == 0) {
+    log("Warning: Missing Content-Length header, or zero-length message.");
+    return llvm::None;
+  }
+
+  std::string JSON(ContentLength, '\0');
+  for (size_t Pos = 0, Read; Pos < ContentLength; Pos += Read) {
+    // Handle EINTR which is sent when a debugger attaches on some platforms.
+    Read = llvm::sys::RetryAfterSignal(0u, ::fread, &JSON[Pos], 1,
+                                       ContentLength - Pos, In);
+    if (Read == 0) {
+      elog("Input was aborted. Read only {0} bytes of expected {1}.", Pos,
+           ContentLength);
+      return llvm::None;
+    }
+    clearerr(In); // If we're done, the error was transient. If we're not done,
+                  // either it was transient or we'll see it again on retry.
+    Pos += Read;
+  }
+  return std::move(JSON);
+}
+
+// For lit tests we support a simplified syntax:
+// - messages are delimited by '---' on a line by itself
+// - lines starting with # are ignored.
+// This is a testing path, so favor simplicity over performance here.
+// When returning None, feof() or ferror() will be set.
+llvm::Optional<std::string> readDelimitedMessage(std::FILE *In) {
+  std::string JSON;
+  std::string Line;
+  while (readLine(In, Line)) {
+    auto LineRef = llvm::StringRef(Line).trim();
+    if (LineRef.startswith("#")) // comment
+      continue;
+
+    // found a delimiter
+    if (LineRef.rtrim() == "---")
+      break;
+
+    JSON += Line;
+  }
+
+  if (ferror(In)) {
+    elog("Input error while reading message!");
+    return llvm::None;
+  }
+  return std::move(JSON); // Including at EOF
+}
+
+class JSONTransport : public Transport {
+public:
+  JSONTransport(std::FILE *In, llvm::raw_ostream &Out, JSONStreamStyle Style)
+      : In(In), Out(Out), Style(Style) {}
+
+  FILE *In;
+  llvm::raw_ostream &Out;
+  JSONStreamStyle Style;
+
+  llvm::Error loop(MessageHandler &Handler) override {
+    auto &ReadMessage =
+        (Style == Delimited) ? readDelimitedMessage : readStandardMessage;
+    while (!feof(In)) {
+      if (ferror(In))
+        return errorCodeToError(std::error_code(errno, std::system_category()));
+      if (auto JSON = ReadMessage(In)) {
+        if (auto Doc = json::parse(*JSON)) {
+           if (handleMessage(std::move(*Doc), Handler)
+             return Error::success();
+        } else {
+          // Parse error. Log the raw message.
+          vlog("<<< {0}\n", *JSON);
+          elog("JSON parse error: {0}", llvm::toString(Doc.takeError()));
+        }
+      }
+    }
+    return errorCodeToError(std::make_error_code(std::errc::io_error));
+  }
+
+private:
+  void sendMessage(llvm::json::Value Message) override {
+    Out << llvm::formatv("{0:2}", Message);
+    Out.flush();
+  }
+};
+
+} // namespace
+
+std::unique_ptr<Transport>
+newJSONTransport(std::FILE *In, llvm::raw_ostream &Out, JSONStreamStyle Style) {
+  return llvm::make_unique<JSONTransport>(In, Out, Style);
+}
+
+} // namespace clangd
+} // namespace clang