[clangd] Add option to use remote index as static index

[kbobyrev]

Remove unintended change.

[kbobyrev]

Just stash changes, this is not a patch that is ready for a review.

[kbobyrev]

Don't touch Marshalling.cpp

[kbobyrev]

Remove remaining changes from Marshalling.cpp

[kbobyrev]

Still WIP, stashing changes.

[kbobyrev]

Add flag description.

[kbobyrev]

Rebase on top of master.

[hokein]

clang-tools-extra/clangd/tool/ClangdMain.cpp

457

not sure whether we need a new category, probably can live in Features?

459

since we hide this in the CLANGD_ENABLE_REMOTE flag, maybe remove the Hidden?

463

maybe ProjectRoot?

704

the new code section here should be guarded under #ifdef CLANGD_ENABLE_REMOTE

709

if remote-index is on, should we turn off the background index? I think this is our plan, right?

711

nit: use elog.

713

instead of exiting clangd, maybe just continue (since this is not a fatal error), and adjust the error message stating that index-file option is ignored.

[kbobyrev]

Address review comments.

[kbobyrev]

Remove unused Remote category.

[hokein]

mostly good, just a concern about the linking dependence.

clang-tools-extra/clangd/CMakeLists.txt

138

the dependence here seems to be a layer violation. we add remote-index library when linking the clangDaemon, but clangdRemoteIndex *depends* on clangDaemon

I think we should do this when linking the clangd binary (in clangd/tool/CMakeLists.txt).

clang-tools-extra/clangd/tool/ClangdMain.cpp

463

nit: project-path -> project-root.

[kbobyrev]

Resolve more comments.

[hokein]

looks good, thanks.

[hans]

clang-tools-extra/clangd/Features.inc.in

2

I'm guessing this should be @CLANGD_ENABLE_REMOTE@
The gn build was upset: http://45.33.8.238/linux/23942/step_4.txt
Speculatively fixed in 47a0254229ca425aa4e169c2db14e92b8db86784. Please shout if this was wrong.

[hans]

clang-tools-extra/clangd/tool/ClangdMain.cpp

704

I think it should be an #if since the macro is always defined (0 or 1). Doing that in 40d11a878044711708fb6738e4b78a4c9ac3de7b

[thakis]

clang-tools-extra/clangd/Features.inc.in

2

Also, you probably either want #cmakedefine or you want to check with #if instead of #ifdef below. As-is, in the disabled case this will write #define CLANGD_ENABLE_REMOTE 0 in the disabled case, which causes the #ifdef to activate.

clang-tools-extra/clangd/CMakeLists.txt

add_clang_library(clangDaemon
   clangTidy
   ${LLVM_PTHREAD_LIB}
   ${ALL_CLANG_TIDY_CHECKS}
 
   DEPENDS
   omp_gen
   )
 
+# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+  include(FindGRPC)
+endif()
+add_subdirectory(index/remote)
+
 clang_target_link_libraries(clangDaemon
   PRIVATE
   clangAST
   clangASTMatchers
   clangBasic
   clangDriver
   clangFormat
   clangFrontend
   clangIndex
   clangLex
   clangSema
   clangSerialization
   clangTooling
   clangToolingCore
   clangToolingInclusions
   clangToolingRefactoring
   clangToolingSyntax
+  clangdRemoteIndex
   )
 
 add_subdirectory(refactor/tweaks)
 if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
   # FIXME: Make fuzzer not use linux-specific APIs, build it everywhere.
   add_subdirectory(fuzzer)
 endif()
 add_subdirectory(tool)
 add_subdirectory(indexer)
 
 if (LLVM_INCLUDE_BENCHMARKS)
   add_subdirectory(benchmarks)
 endif()
 if ( CLANGD_BUILD_XPC )
   add_subdirectory(xpc)
 endif ()
 
 if(CLANG_INCLUDE_TESTS)
 add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
 
-# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable.
-option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
-set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.")
-
-if (CLANGD_ENABLE_REMOTE)
-  include(FindGRPC)
-endif()
-add_subdirectory(index/remote)
 add_subdirectory(index/dex/dexp)

clang-tools-extra/clangd/Features.inc.in

 #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
+#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMTE@

clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp

clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
   for (const auto &Type : Request->preferred_types())
     Result.ProximityPaths.push_back(Type);
   return Result;
 }
 
 llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
                                             llvm::UniqueStringSaver *Strings,
                                             llvm::StringRef IndexRoot) {
-  if (!Message.has_info() || !Message.has_definition() ||
-      !Message.has_canonical_declaration()) {
+  if (!Message.has_info()) {
     elog("Cannot convert Symbol from Protobuf: {0}",
          Message.ShortDebugString());
     return llvm::None;
   }
   clangd::Symbol Result;
   auto ID = SymbolID::fromStr(Message.id());
   if (!ID) {
     elog("Cannot parse SymbolID {0} given Protobuf: {1}", ID.takeError(),
          Message.ShortDebugString());
     return llvm::None;
   }
   Result.ID = *ID;
   Result.SymInfo = fromProtobuf(Message.info());
   Result.Name = Message.name();
   Result.Scope = Message.scope();
   auto Definition = fromProtobuf(Message.definition(), Strings, IndexRoot);
-  if (!Definition)
-    return llvm::None;
+  if (Definition)
     Result.Definition = *Definition;
   auto Declaration =
       fromProtobuf(Message.canonical_declaration(), Strings, IndexRoot);
   if (!Declaration)
-    return llvm::None;
     Result.CanonicalDeclaration = *Declaration;
   Result.References = Message.references();
   Result.Origin = static_cast<clangd::SymbolOrigin>(Message.origin());
   Result.Signature = Message.signature();
   Result.TemplateSpecializationArgs = Message.template_specialization_args();
   Result.CompletionSnippetSuffix = Message.completion_snippet_suffix();
   Result.Documentation = Message.documentation();
   Result.ReturnType = Message.return_type();
   Result.Type = Message.type();

clang-tools-extra/clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangdLSPServer.h"
 #include "CodeComplete.h"
 #include "Features.inc"
 #include "PathMapping.h"
 #include "Protocol.h"
 #include "Transport.h"
 #include "index/Background.h"
 #include "index/Serialization.h"
+#include "index/remote/Client.h"
 #include "refactor/Rename.h"
 #include "support/Path.h"
 #include "support/Shutdown.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/Basic/Version.h"
 #include "clang/Format/Format.h"
 #include "llvm/ADT/Optional.h"

 using llvm::cl::values;
 
 // All flags must be placed in a category, or they will be shown neither in
 // --help, nor --help-hidden!
 OptionCategory CompileCommands("clangd compilation flags options");
 OptionCategory Features("clangd feature options");
 OptionCategory Misc("clangd miscellaneous options");
 OptionCategory Protocol("clangd protocol and logging options");
+OptionCategory Remote("clangd remote index options");
 const OptionCategory *ClangdCategories[] = {&Features, &Protocol,
                                             &CompileCommands, &Misc};
 
 enum CompileArgsFrom { LSPCompileArgs, FilesystemCompileArgs };
 opt<CompileArgsFrom> CompileArgsFrom{
     "compile_args_from",
     cat(CompileCommands),
     desc("The source of compile commands"),
desc(
         "User config is from clangd/config.yaml in the following directories:\n"
         "\tWindows: %USERPROFILE%\\AppData\\Local\n"
         "\tMac OS: ~/Library/Preferences/\n"
         "\tOthers: $XDG_CONFIG_HOME, usually ~/.config\n"
         "Configuration is documented at https://clangd.llvm.org/config.html"),
     init(false),
 };
 
+#ifdef CLANGD_ENABLE_REMOTE
+opt<std::string> RemoteIndexAddress{
+    "remote-index-address",
+    cat(Remote),
+    desc("Address of the remote index server"),
+    Hidden,
+};
+
+// FIXME(kirillbobyrev): This can be inferred by Clangd. Add note: requires
+// RemoteIndexAddress to be set.
+opt<std::string> ProjectPath{
+    "project-path",
+    cat(Remote),
+    // FIXME(kirillbobyrev): Add description.
+    desc(""),
+    Hidden,
+};
+#endif
+
 /// Supports a test URI scheme with relaxed constraints for lit tests.
 /// The path in a test URI will be combined with a platform-specific fake
 /// directory to form an absolute path. For example, test:///a.cpp is resolved
 /// C:\clangd-test\a.cpp on Windows and /clangd-test/a.cpp on Unix.
 class TestScheme : public URIScheme {
 public:
   llvm::Expected<std::string>
   getAbsolutePath(llvm::StringRef /*Authority*/, llvm::StringRef Body,
if (EnableIndex && !IndexFile.empty()) {
     StaticIdx.reset(Placeholder = new SwapIndex(std::make_unique<MemIndex>()));
     AsyncIndexLoad = runAsync<void>([Placeholder] {
       if (auto Idx = loadIndex(IndexFile, /*UseDex=*/true))
         Placeholder->reset(std::move(Idx));
     });
     if (Sync)
       AsyncIndexLoad.wait();
   }
+  if (RemoteIndexAddress.empty() != ProjectPath.empty()) {
+    llvm::errs() << "remote-index-address and project-path have to be "
+                    "specified at the same time.";
+    return 1;
+  }
+  if (!RemoteIndexAddress.empty()) {
+    if (!IndexFile.empty()) {
+      llvm::errs() << "When remote index is enabled, IndexFile should not be "
+                      "specified. Only one can be used at time.";
+      return 1;
+    }
+    log("Connecting to remote index at {0}", RemoteIndexAddress);
+    StaticIdx = remote::getClient(RemoteIndexAddress, ProjectPath);
+  }
   Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.BuildRecoveryAST = RecoveryAST;
   Opts.PreserveRecoveryASTType = RecoveryASTType;
   Opts.FoldingRanges = FoldingRanges;
 
   clangd::CodeCompleteOptions CCOpts;
   CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;