diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -153,3 +153,12 @@ add_subdirectory(test) add_subdirectory(unittests) endif() + +# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable. +option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF) +set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.") + +if (CLANGD_ENABLE_REMOTE) + include(FindGRPC) + add_subdirectory(index/remote) +endif() diff --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt @@ -0,0 +1,7 @@ +generate_grpc_protos(RemoteIndexProtos "Index.proto") + +include_directories("${CMAKE_CURRENT_BINARY_DIR}") +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../) + +add_subdirectory(client) +add_subdirectory(server) diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/Index.proto @@ -0,0 +1,19 @@ +//===--- Index.proto - Remote index Protocol Buffers definition -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +syntax = "proto3"; + +package clang.clangd.remote; + +service Index { + rpc Lookup(LookupRequest) returns (stream LookupReply) {} +} + +message LookupRequest { string id = 1; } + +message LookupReply { string symbol_yaml = 1; } diff --git a/clang-tools-extra/clangd/index/remote/README.md b/clang-tools-extra/clangd/index/remote/README.md new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/README.md @@ -0,0 +1,59 @@ +# Clangd remote index + +Clangd uses a global index for project-wide code completion, navigation and +other features. For large projects, building this can take many hours and +keeping it loaded uses a lot of memory. + +To relieve that burden, we're building remote index — a global index +served on a different machine and shared between developers. This directory +contains code that is used as Proof of Concept for the upcoming remote index +feature. + +## Building + +This feature uses gRPC and Protobuf libraries, so you will need to install them. +There are two ways of doing that. + +However you install dependencies, to enable this feature and build remote index +tools you will need to set this CMake flag — `-DCLANGD_ENABLE_REMOTE=On`. + +### System-installed libraries + +On Debian-like systems gRPC and Protobuf can be installed from apt: + +```bash +apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc +``` + +### Building from sources + +Another way of installing gRPC and Protobuf is building from sources using +CMake. The easiest way of doing that would be to choose a directory where you +want to install so that the installation files are not copied to system root and +you can uninstall gRPC or use different versions of the library. + +```bash +# Get source code. +$ git clone -b v1.28.1 https://github.com/grpc/grpc +$ cd grpc +$ git submodule update --init +# Choose directory where you want gRPC installation to live. +$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed +# Build and install gRPC to ${GRPC_INSTALL_PATH} +$ mkdir build; cd build +$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. +$ make install +``` + +This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into +more detail on how to build gRPC from sources. + +By default, CMake will look for system-installed libraries when building remote +index tools so you will have to adjust LLVM's CMake invocation. The following +flag will inform build system that you chose this option — +`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`. + +## Running + +The remote index isn't usable with Clangd yet, but you can try the +proof-of-concept tools in `client/` and `server/` subdirectories. diff --git a/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt @@ -0,0 +1,19 @@ +set(LLVM_LINK_COMPONENTS + LineEditor + Support + ) +add_clang_executable(clangd-index-client + Client.cpp + ) +target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1) +clang_target_link_libraries(clangd-index-client + PRIVATE + clangDaemon + ) +target_link_libraries(clangd-index-client + PRIVATE + RemoteIndexProtos + + protobuf + grpc++ + ) diff --git a/clang-tools-extra/clangd/index/remote/client/Client.cpp b/clang-tools-extra/clangd/index/remote/client/Client.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/client/Client.cpp @@ -0,0 +1,91 @@ +//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple interactive tool which can be used to manually +// evaluate symbol search quality of Clangd index. +// +//===----------------------------------------------------------------------===// + +#include "SourceCode.h" +#include "index/Serialization.h" +#include "index/dex/Dex.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +llvm::cl::opt + ServerAddress("server-address", + llvm::cl::desc("Address of remote index server to use."), + llvm::cl::init("0.0.0.0:50051")); + +static const std::string Overview = R"( +This is an **experimental** interactive tool to process user-provided search +queries over given symbol collection obtained via clangd-indexer with the help +of remote index server. The client will connect to remote index server and pass +it lookup queries. +)"; + +class RemoteIndexClient { +public: + RemoteIndexClient(std::shared_ptr Channel) + : Stub(remote::Index::NewStub(Channel)) {} + + void lookup(llvm::StringRef ID) { + llvm::outs() << "Lookup of symbol with ID " << ID << '\n'; + remote::LookupRequest Proto; + Proto.set_id(ID.str()); + + grpc::ClientContext Context; + remote::LookupReply Reply; + std::unique_ptr> Reader( + Stub->Lookup(&Context, Proto)); + while (Reader->Read(&Reply)) { + llvm::outs() << Reply.symbol_yaml(); + } + grpc::Status Status = Reader->Finish(); + if (Status.ok()) { + llvm::outs() << "lookupRequest rpc succeeded.\n"; + } else { + llvm::outs() << "lookupRequest rpc failed.\n"; + } + } + +private: + std::unique_ptr Stub; +}; + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, const char *argv[]) { + using namespace clang::clangd; + + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); + llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands. + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + RemoteIndexClient IndexClient( + grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials())); + + llvm::LineEditor LE("remote-index-client"); + while (llvm::Optional Request = LE.readLine()) + IndexClient.lookup(std::move(*Request)); +} diff --git a/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt @@ -0,0 +1,20 @@ +set(LLVM_LINK_COMPONENTS + LineEditor + Support + ) +add_clang_executable(clangd-index-server + Server.cpp + ) +target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1) +clang_target_link_libraries(clangd-index-server + PRIVATE + clangDaemon + ) +target_link_libraries(clangd-index-server + PRIVATE + RemoteIndexProtos + + protobuf + grpc++ + clangDaemon + ) diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -0,0 +1,102 @@ +//===--- Server.cpp - gRPC-based Remote Index Server ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "index/Index.h" +#include "index/Serialization.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" +#include "grpcpp/health_check_service_interface.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +static const std::string Overview = R"( +This is an experimental remote index implementation. The server opens Dex and +awaits gRPC lookup requests from the client. +)"; + +llvm::cl::opt IndexPath(llvm::cl::desc(""), + llvm::cl::Positional, llvm::cl::Required); + +llvm::cl::opt ServerAddress("server-address", + llvm::cl::init("0.0.0.0:50051")); + +std::unique_ptr openIndex(llvm::StringRef Index) { + return loadIndex(Index, /*UseIndex=*/true); +} + +class RemoteIndexServer final : public remote::Index::Service { +public: + RemoteIndexServer(std::unique_ptr Index) + : Index(std::move(Index)) {} + +private: + grpc::Status Lookup(grpc::ServerContext *Context, + const remote::LookupRequest *Request, + grpc::ServerWriter *Reply) override { + llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n'; + LookupRequest Req; + auto SID = SymbolID::fromStr(Request->id()); + if (!SID) { + llvm::outs() << llvm::toString(SID.takeError()) << "\n"; + return grpc::Status::CANCELLED; + } + Req.IDs.insert(*SID); + Index->lookup(Req, [&](const Symbol &Sym) { + remote::LookupReply NextSymbol; + NextSymbol.set_symbol_yaml(toYAML(Sym)); + Reply->Write(NextSymbol); + }); + return grpc::Status::OK; + } + + std::unique_ptr Index; +}; + +void runServer(std::unique_ptr Index, + const std::string &ServerAddress) { + RemoteIndexServer Service(std::move(Index)); + + grpc::EnableDefaultHealthCheckService(true); + grpc::ServerBuilder Builder; + Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials()); + Builder.RegisterService(&Service); + std::unique_ptr Server(Builder.BuildAndStart()); + llvm::outs() << "Server listening on " << ServerAddress << '\n'; + + Server->Wait(); +} + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, char *argv[]) { + using namespace clang::clangd; + llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview); + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + std::unique_ptr Index = openIndex(IndexPath); + + if (!Index) { + llvm::outs() << "Failed to open the index.\n"; + return -1; + } + + runServer(std::move(Index), ServerAddress); +} diff --git a/llvm/cmake/modules/FindGRPC.cmake b/llvm/cmake/modules/FindGRPC.cmake new file mode 100644 --- /dev/null +++ b/llvm/cmake/modules/FindGRPC.cmake @@ -0,0 +1,50 @@ +# This setup requires gRPC to be built from sources using CMake and installed to +# ${GRPC_INSTALL_PATH} via -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH}. +if (GRPC_INSTALL_PATH) + set(protobuf_MODULE_COMPATIBLE TRUE) + find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using protobuf ${protobuf_VERSION}") + find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using gRPC ${gRPC_VERSION}") + + include_directories(${Protobuf_INCLUDE_DIRS}) + + # gRPC CMake CONFIG gives the libraries slightly odd names, make them match + # the conventional system-installed names. + set_target_properties(protobuf::libprotobuf PROPERTIES IMPORTED_GLOBAL TRUE) + add_library(protobuf ALIAS protobuf::libprotobuf) + set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_GLOBAL TRUE) + add_library(grpc++ ALIAS gRPC::grpc++) + + set(GRPC_CPP_PLUGIN $) + set(PROTOC ${Protobuf_PROTOC_EXECUTABLE}) +else() + find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin) + find_program(PROTOC protoc) +endif() + +# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}. +# Libraries that use these headers should adjust the include path. +# FIXME(kirillbobyrev): Allow optional generation of gRPC code and give callers +# control over it via additional parameters. +function(generate_grpc_protos LibraryName ProtoFile) + get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE) + get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH) + + set(GeneratedProtoSource "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc") + set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h") + set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc") + set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h") + add_custom_command( + OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}" + COMMAND ${PROTOC} + ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + --proto_path="${ProtoSourcePath}" + --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}" + "${ProtoSourceAbsolutePath}" + DEPENDS "${ProtoSourceAbsolutePath}") + + add_library(${LibraryName} ${GeneratedProtoSource} ${GeneratedGRPCSource}) + target_link_libraries(${LibraryName} grpc++ protobuf) +endfunction()