Introduce clangd-server-monitor tool

[kadircet]

thanks for doing this! can we also have a lit test? we already have some tests spinning up an index-server, i suppose we can make an extra rpc to those and ensure we get a success response?

clang-tools-extra/clangd/index/remote/monitor/CMakeLists.txt

5

s/server/index-server/

being more specific would be nice, just in case we have more servers (i.e. review) in the future.

clang-tools-extra/clangd/index/remote/monitor/Monitor.cpp

55

nit: elog

60

rather than dumping protobuf, can we dump in json ? see https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.util.json_util

[kbobyrev]

Address review comments.

[kbobyrev]

Inline gRPC deadline.

[kadircet]

clang-tools-extra/clangd/test/remote-index/pipeline_helper.py

78

logs for index-server and monitor gets intertwined (as we put logs from index-server until initialization above). so can we output monitoring logs after we've dumped all the logs from index-server?

97

can we just dump the stdout to main process (as we do with other subprocesses) and check for the output in the lit tests instead? that way we can check for particular values without complicating the helper instead

[kbobyrev]

Address review comments

[kbobyrev]

Remove (now) unused Python JSON import.

[kadircet]

thanks, let's ship it!

clang-tools-extra/clangd/index/remote/monitor/Monitor.cpp

16

nit: this is probably not needed now as we dropped formatv

73

nit: i am not sure if this really compiles (was testing it for status-updater patch on gcp). as this is a stringpiece, so you might wanna call ToString/as_string on it.

clang-tools-extra/clangd/test/remote-index/pipeline_helper.py

34

nit: as discussed offline this is only to save some runtime on tests that don't care about it. I'd probably drop this and add later if we feel the need, but up to you.

[kbobyrev]

Resolve post-LGTM comments.

[kbobyrev]

Remove leftover --with-monitor from the lit test.

clang-tools-extra/clangd/index/remote/CMakeLists.txt

add_clang_library(clangdRemoteIndex
 
     DEPENDS
     RemoteIndexProto
     RemoteIndexServiceProto
     )
 
   add_subdirectory(marshalling)
   add_subdirectory(server)
+  add_subdirectory(monitor)
 else()
   # Provides a no-op implementation of clangdRemoteIndex.
   add_subdirectory(unimplemented)
 endif()

clang-tools-extra/clangd/index/remote/monitor/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+add_clang_executable(clangd-index-server-monitor
+  Monitor.cpp
+
+  DEPENDS
+  RemoteIndexServiceProto
+  )
+
+target_link_libraries(clangd-index-server-monitor
+  PRIVATE
+  clangBasic
+  clangdSupport
+
+  MonitoringServiceProto
+  RemoteIndexServiceProto
+  )

clang-tools-extra/clangd/index/remote/monitor/Monitor.cpp

+//===--- Monitor.cpp - Request server monitoring information through CLI --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MonitoringService.grpc.pb.h"
+#include "MonitoringService.pb.h"
+
+#include "support/Logger.h"
+#include "clang/Basic/Version.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Signals.h"
+
+#include <chrono>
+#include <google/protobuf/util/json_util.h>
+#include <grpc++/grpc++.h>
+
+namespace clang {
+namespace clangd {
+namespace remote {
+namespace {
+
+static constexpr char Overview[] = R"(
+This tool requests monitoring information (uptime, index freshness) from the
+server and prints it to stdout.
+)";
+
+llvm::cl::opt<std::string>
+    ServerAddress("server-address", llvm::cl::Positional,
+                  llvm::cl::desc("Address of the invoked server."),
+                  llvm::cl::Required);
+
+} // namespace
+} // namespace remote
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, char *argv[]) {
+  using namespace clang::clangd::remote;
+  llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  const auto Channel =
+      grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials());
+  const auto Stub = clang::clangd::remote::v1::Monitor::NewStub(Channel);
+  grpc::ClientContext Context;
+  Context.set_deadline(std::chrono::system_clock::now() +
+                       std::chrono::seconds(10));
+  Context.AddMetadata("version", clang::getClangToolFullVersion("clangd"));
+  const clang::clangd::remote::v1::MonitoringInfoRequest Request;
+  clang::clangd::remote::v1::MonitoringInfoReply Response;
+  const auto Status = Stub->MonitoringInfo(&Context, Request, &Response);
+  if (!Status.ok()) {
+    clang::clangd::elog("Can not request monitoring information ({0}): {1}\n",
+                        Status.error_code(), Status.error_message());
+    return -1;
+  }
+  std::string Output;
+  google::protobuf::util::JsonPrintOptions Options;
+  Options.add_whitespace = true;
+  Options.always_print_primitive_fields = true;
+  Options.preserve_proto_field_names = true;
+  const auto JsonStatus =
+      google::protobuf::util::MessageToJsonString(Response, &Output, Options);
+  if (!JsonStatus.ok()) {
+    clang::clangd::elog("Can not convert response ({0}) to JSON ({1}): {2}\n",
+                        Response.DebugString(), JsonStatus.error_code(),
+                        JsonStatus.error_message());
+    return -1;
+  }
+  llvm::outs() << Output;
+}

clang-tools-extra/clangd/test/CMakeLists.txt

set(CLANGD_TEST_DEPS
   )
 
 if(CLANGD_BUILD_XPC)
   list(APPEND CLANGD_TEST_DEPS clangd-xpc-test-client)
   list(APPEND CLANGD_TEST_DEPS ClangdXpcUnitTests)
 endif()
 
 if(CLANGD_ENABLE_REMOTE)
-  list(APPEND CLANGD_TEST_DEPS clangd-index-server)
+  list(APPEND CLANGD_TEST_DEPS clangd-index-server clangd-index-server-monitor)
 endif()
 
 foreach(dep FileCheck count not llvm-config)
   if(TARGET ${dep})
     list(APPEND CLANGD_TEST_DEPS ${dep})
   endif()
 endforeach()
 

clang-tools-extra/clangd/test/remote-index/pipeline.test

 # RUN: rm -rf %t
 # RUN: clangd-indexer %S/Inputs/Source.cpp > %t.idx
-# RUN: %python %S/pipeline_helper.py --input-file-name=%s --project-root=%S --index-file=%t.idx | FileCheck %s
+# RUN: %python %S/pipeline_helper.py --with-monitor --input-file-name=%s --project-root=%S --index-file=%t.idx | FileCheck %s
 # REQUIRES: clangd-remote-index
 
+#      CHECK:  "uptime_seconds": "{{.*}}",
+# CHECK-NEXT:  "index_age_seconds": "{{.*}}",
+# CHECK-NEXT:  "index_commit_hash": "{{.*}}",
+# CHECK-NEXT:  "index_link": "{{.*}}"
+
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
 #      CHECK:  "id": 0,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 ---
 {"jsonrpc":"2.0","id":1,"method":"workspace/symbol","params":{"query":"Character"}}
 #      CHECK:  "id": 1,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 # CHECK-NEXT:    "result": [

clang-tools-extra/clangd/test/remote-index/pipeline_helper.py

 
 import argparse
 import os
 import subprocess
 from socket import socket
 import sys
 import time
 import threading
+import json
 
 
-def kill_server_after_delay(server_process):
+def kill_process_after_delay(server_process):
   time.sleep(10)
   if server_process.poll() is None:
     server_process.kill()
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--input-file-name', required=True)
   parser.add_argument('--project-root', required=True)
   parser.add_argument('--index-file', required=True)
   parser.add_argument('--server-arg', action='append', default=[])
   parser.add_argument('--server-log', nargs='?', type=argparse.FileType('wb'), default=os.devnull)
+  parser.add_argument('--with-monitor', action='store_true')
 
   args = parser.parse_args()
 
   # Grab an available port.
   with socket() as s:
     s.bind(('localhost', 0))
     server_address = 'localhost:' + str(s.getsockname()[1])
 
   print('Initializing clangd-index-server...', file=sys.stderr)
   index_server_process = subprocess.Popen([
       'clangd-index-server', '--server-address=' + server_address,
       args.index_file, args.project_root
   ] + args.server_arg,
                                           stderr=subprocess.PIPE)
 
   # This will kill index_server_process if it hangs without printing init
   # message.
   shutdown_thread = threading.Thread(
-      target=kill_server_after_delay, args=(index_server_process,))
+      target=kill_process_after_delay, args=(index_server_process,))
   shutdown_thread.daemon = True
   shutdown_thread.start()
 
   # Wait for the server to warm-up.
   found_init_message = False
   while index_server_process.poll() is None:
     line = index_server_process.stderr.readline()
     args.server_log.write(line)
     args.server_log.flush()
     if b'Server listening' in line:
       print('Server initialization complete.', file=sys.stderr)
       found_init_message = True
       break
 
   if not found_init_message:
     print('Server initialization failed. Shutting down.', file=sys.stderr)
     sys.exit(1)
 
+  if args.with_monitor:
+    print('Running clangd-index-server-monitor...', file=sys.stderr)
+    index_server_monitor_process = subprocess.Popen([
+        'clangd-index-server-monitor', server_address,
+    ], stderr=subprocess.PIPE)
+
+    index_server_monitor_process.wait()
+
   in_file = open(args.input_file_name)
 
   print('Staring clangd...', file=sys.stderr)
   clangd_process = subprocess.Popen([
       'clangd', '--remote-index-address=' + server_address,
       '--project-root=' + args.project_root, '--lit-test', '--sync'
   ],
                                     stdin=in_file)
   clangd_process.wait()
   print(
       'Clangd executed successfully, shutting down child processes.',
       file=sys.stderr)
   index_server_process.kill()
   for line in index_server_process.stderr:
     args.server_log.write(line)
     args.server_log.flush()
 
+  if args.with_monitor:
+    for line in index_server_monitor_process.stderr:
+        args.server_log.write(line)
+        args.server_log.flush()
+
 
 if __name__ == '__main__':
   main()