diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -140,7 +140,6 @@ endif() add_subdirectory(tool) add_subdirectory(indexer) -add_subdirectory(index/dex/dexp) if (LLVM_INCLUDE_BENCHMARKS) add_subdirectory(benchmarks) @@ -153,3 +152,11 @@ add_subdirectory(test) add_subdirectory(unittests) endif() + +# This setup requires gRPC to be built from sources using CMake and installed to +# ${GRPC_INSTALL_PATH} via -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH}. +if (CLANGD_ENABLE_REMOTE) + add_subdirectory(index/remote) +endif() + +add_subdirectory(index/dex/dexp) diff --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt @@ -0,0 +1,12 @@ +generate_grpc_protos("Index.proto" RemoteIndexProtos) + +include_directories("${CMAKE_CURRENT_BINARY_DIR}") +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../) + +set(LLVM_LINK_COMPONENTS + LineEditor + Support + ) + +add_subdirectory(client) +add_subdirectory(server) diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/Index.proto @@ -0,0 +1,19 @@ +//===--- Index.proto - Remote index Protocol Buffers definition -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +syntax = "proto3"; + +package clang.clangd.remote; + +service Index { + rpc requestLookup(LookupRequest) returns (stream LookupReply) {} +} + +message LookupRequest { string ID = 1; } + +message LookupReply { string Symbol_YAML = 1; } diff --git a/clang-tools-extra/clangd/index/remote/README.md b/clang-tools-extra/clangd/index/remote/README.md new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/README.md @@ -0,0 +1,114 @@ +# Clangd remote index + +Clangd global index used for code completion, code navigation and a number of +useful features takes a long time to build (3-4 hours on powerful machines for +Chrome-sized projects) and induces a significant memory overhead (up to 2 GB) +to serve. As a result, developers in large open source projects like LLVM often +don't get the best experience for editing and navigating source code. + +To relieve that burden, we're building remote index — a global index +served on a different machine. This directory contains code that is used as +Proof of Concept for the upcoming remote index feature. + +## Building + +This feature uses gRPC and Protobuf libraries, so you will need to install them. +There are two ways of doing that. + +Regardless of the dependencies installation way, to enable this feature and +build remote index tools you will need to set this CMake flag — +`-DCLANGD_ENABLE_REMOTE=On`. + +### System-installed libraries + +On Debian-like systems gRPC and Protobuf can be installed from apt: + +```bash +apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc +``` + +### Building from sources + +Another way of installing gRPC and Protobuf is building from sources using +CMake. The easiest way of doing that would be to choose a directory where you +want to install so that the installation files are not copied to system root and +you can uninstall gRPC or use different versions of the library. + +```bash +# Get source code. +$ git clone -b v1.28.1 https://github.com/grpc/grpc +$ cd grpc +$ git submodule update --init +# Choose directory where you want gRPC installation to live. +$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed +# Build and install gRPC to ${GRPC_INSTALL_PATH} +$ mkdir build; cd build +$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. +$ make install +``` + +This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into +more detail on how to build gRPC from sources. + +After installation, you need to + +By default, CMake will look for system-installed libraries when building remote +index tools so you will have to adjust LLVM's CMake invocation. The following +flag will inform build system that you chose this option — +`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`. + +## Running + +Right now it is only possible to connect to the index service hosted on the +same machine — `clangd-remote-server`. `clangd-remote-index` is a simple +`dexp`-like tool that passes lookup requests to the server. The easiest way to +try it out is running `clangd-remote-index` and `clangd-remote-server` in two +different processes by opening two terminals: + +Terminal one: + +```bash +# Get ID of some symbol from the index. +$ ./bin/dexp /tmp/llvm.dex -c="lookup -name clang::clangd" +--- +ID: 43C8113D5731B053 +Name: clangd +Scope: 'clang::' +... +$ ./bin/clangd-index-server /tmp/llvm.dex +``` + +Terminal two: + +```bash +$ ./bin/clangd-index-client +# Paste ID of the symbol from dexp lookup. +remote-index-client > 43C8113D5731B053 +Lookup of symbol with ID 43C8113D5731B053 +--- +ID: 43C8113D5731B053 +Name: clangd +Scope: 'clang::' +SymInfo: + Kind: Namespace + Lang: Cpp +CanonicalDeclaration: + FileURI: 'file:///usr/local/google/home/kbobyrev/dev/src/llvm-project/clang-tools-extra/clangd/FuzzyMatch.h' + Start: + Line: 24 + Column: 10 + End: + Line: 24 + Column: 16 +References: 25 +Origin: 12 +Flags: 9 +Signature: '' +TemplateSpecializationArgs: '' +CompletionSnippetSuffix: '' +Documentation: '' +ReturnType: '' +Type: '' +... +lookupRequest rpc succeeded. +``` diff --git a/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt @@ -0,0 +1,15 @@ +add_clang_executable(clangd-index-client + Client.cpp + ) +target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1) +clang_target_link_libraries(clangd-index-client + PRIVATE + clangBasic + ) +target_link_libraries(clangd-index-client + PRIVATE + RemoteIndexProtos + + protobuf + grpc++ + ) diff --git a/clang-tools-extra/clangd/index/remote/client/Client.cpp b/clang-tools-extra/clangd/index/remote/client/Client.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/client/Client.cpp @@ -0,0 +1,91 @@ +//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple interactive tool which can be used to manually +// evaluate symbol search quality of Clangd index. +// +//===----------------------------------------------------------------------===// + +#include "SourceCode.h" +#include "index/Serialization.h" +#include "index/dex/Dex.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +llvm::cl::opt + ServerAddress("server-address", + llvm::cl::desc("Address of remote index server to use."), + llvm::cl::init("0.0.0.0:50051")); + +static const std::string Overview = R"( +This is an **experimental** interactive tool to process user-provided search +queries over given symbol collection obtained via clangd-indexer with the help +of remote index server. The client will connect to remote index server and pass +it lookup queries. +)"; + +class RemoteIndexClient { +public: + RemoteIndexClient(std::shared_ptr Channel) + : Stub(remote::Index::NewStub(Channel)) {} + + void lookup(llvm::StringRef ID) { + llvm::outs() << "Lookup of symbol with ID " << ID << '\n'; + remote::LookupRequest Proto; + Proto.set_id(ID.str()); + + grpc::ClientContext Context; + remote::LookupReply Reply; + std::unique_ptr> Reader( + Stub->requestLookup(&Context, Proto)); + while (Reader->Read(&Reply)) { + llvm::outs() << Reply.symbol_yaml(); + } + grpc::Status Status = Reader->Finish(); + if (Status.ok()) { + llvm::outs() << "lookupRequest rpc succeeded.\n"; + } else { + llvm::outs() << "lookupRequest rpc failed.\n"; + } + } + +private: + std::unique_ptr Stub; +}; + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, const char *argv[]) { + using namespace clang::clangd; + + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); + llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands. + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + RemoteIndexClient IndexClient( + grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials())); + + llvm::LineEditor LE("remote-index-client"); + while (llvm::Optional Request = LE.readLine()) + IndexClient.lookup(std::move(*Request)); +} diff --git a/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt @@ -0,0 +1,16 @@ +add_clang_executable(clangd-index-server + Server.cpp + ) +target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1) +clang_target_link_libraries(clangd-index-server + PRIVATE + clangBasic + ) +target_link_libraries(clangd-index-server + PRIVATE + RemoteIndexProtos + + protobuf + grpc++ + clangDaemon + ) diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -0,0 +1,103 @@ +//===--- Server.cpp - gRPC-based Remote Index Server ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "index/Index.h" +#include "index/Serialization.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" + +#include "grpcpp/grpcpp.h" +#include "grpcpp/health_check_service_interface.h" + +#include "Index.grpc.pb.h" + +namespace clang { +namespace clangd { +namespace { + +static const std::string Overview = R"( +This is an experimental remote index implementation. The server opens Dex and +awaits gRPC lookup requests from the client. +)"; + +llvm::cl::opt IndexPath(llvm::cl::desc(""), + llvm::cl::Positional, llvm::cl::Required); + +llvm::cl::opt ServerAddress("server-address", + llvm::cl::init("0.0.0.0:50051")); + +std::unique_ptr openIndex(llvm::StringRef Index) { + return loadIndex(Index, /*UseIndex=*/true); +} + +class RemoteIndexServer final : public remote::Index::Service { +public: + RemoteIndexServer(std::unique_ptr Index) + : Index(std::move(Index)) {} + +private: + grpc::Status + requestLookup(grpc::ServerContext *Context, + const remote::LookupRequest *Request, + grpc::ServerWriter *Reply) override { + llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n'; + LookupRequest Req; + auto SID = SymbolID::fromStr(Request->id()); + if (!SID) { + llvm::outs() << llvm::toString(SID.takeError()) << "\n"; + return grpc::Status::CANCELLED; + } + Req.IDs.insert(*SID); + Index->lookup(Req, [&](const Symbol &Sym) { + remote::LookupReply NextSymbol; + NextSymbol.set_symbol_yaml(toYAML(Sym)); + Reply->Write(NextSymbol); + }); + return grpc::Status::OK; + } + + std::unique_ptr Index; +}; + +void runServer(std::unique_ptr Index, + const std::string &ServerAddress) { + RemoteIndexServer Service(std::move(Index)); + + grpc::EnableDefaultHealthCheckService(true); + grpc::ServerBuilder Builder; + Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials()); + Builder.RegisterService(&Service); + std::unique_ptr Server(Builder.BuildAndStart()); + llvm::outs() << "Server listening on " << ServerAddress << '\n'; + + Server->Wait(); +} + +} // namespace +} // namespace clangd +} // namespace clang + +int main(int argc, char *argv[]) { + using namespace clang::clangd; + llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview); + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + std::unique_ptr Index = openIndex(IndexPath); + + if (!Index) { + llvm::outs() << "Failed to open the index.\n"; + return -1; + } + + runServer(std::move(Index), ServerAddress); +} diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -375,6 +375,15 @@ set(LLVM_WITH_Z3 1) endif() +# FIXME(kirillbobyrev): Document this in the LLVM docs. +option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF) +# FIXME(kirillbobyrev): Check if it works with optimized tablegen CMake option. +set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library installation.") + +if (CLANGD_ENABLE_REMOTE) + include(FindGRPC) +endif() + if( LLVM_TARGETS_TO_BUILD STREQUAL "all" ) set( LLVM_TARGETS_TO_BUILD ${LLVM_ALL_TARGETS} ) endif() diff --git a/llvm/cmake/modules/FindGRPC.cmake b/llvm/cmake/modules/FindGRPC.cmake new file mode 100644 --- /dev/null +++ b/llvm/cmake/modules/FindGRPC.cmake @@ -0,0 +1,49 @@ +# FIXME(kirillbobyrev): Allow using system installed gRPC. +if (GRPC_INSTALL_PATH) + set(protobuf_MODULE_COMPATIBLE TRUE) + find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using protobuf ${protobuf_VERSION}") + find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH}) + message(STATUS "Using gRPC ${gRPC_VERSION}") + + include_directories(${Protobuf_INCLUDE_DIRS}) + + # gRPC CMake CONFIG gives the libraries slightly odd names, make them match + # the conventional system-installed names. + set_target_properties(protobuf::libprotobuf PROPERTIES IMPORTED_GLOBAL TRUE) + add_library(protobuf ALIAS protobuf::libprotobuf) + set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_GLOBAL TRUE) + add_library(grpc++ ALIAS gRPC::grpc++) + + set(GRPC_CPP_PLUGIN $) + set(PROTOC ${Protobuf_PROTOC_EXECUTABLE}) +else() + get_filename_component(GRPC_CPP_PLUGIN grpc_cpp_plugin ABSOLUTE) + get_filename_component(PROTOC protoc ABSOLUTE) +endif() + +# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}. +# Libraries that use these headers should adjust the include path. +# FIXME(kirillbobyrev): Allow optional generation of prptos only and give +# callers control over it via additional parameters. +function(generate_grpc_protos ProtoFile LibraryName) + get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE) + get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH) + + set(GeneratedProtoSource "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc") + set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h") + set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc") + set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h") + add_custom_command( + OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}" + COMMAND ${PROTOC} + ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + --proto_path="${ProtoSourcePath}" + --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}" + "${ProtoSourceAbsolutePath}" + DEPENDS "${ProtoSourceAbsolutePath}") + + add_library(${LibraryName} ${GeneratedProtoSource} ${GeneratedProtoHeader} ${GeneratedGRPCSource} ${GeneratedGRPCHeader}) + target_link_libraries(${LibraryName} grpc++ protobuf) +endfunction()