[clangd] Move remote-index dependency from clangDaemon to ClangdMain

[kadircet]

Summary

This is a hack to prevent cyclic dependency between clangDaemon and
remote-index-marshalling.

[sammccall]

clang-tools-extra/clangd/index/ProjectAware.h

25–26

while here, IndexGenerator -> IndexFactory, and "can be use to customize" --> "specifies" now that it's mandatory

clang-tools-extra/clangd/tool/ClangdMain.cpp

558

This sort of a comment seems to be rebutting the previous version of the code, which is important today but probably not in the long run.

It may not be needed at all - this factoring actually looks more natural than I expected!
If it is, I think it belongs on createProjectAwareIndex rather than here, as an aside.
(IndexFactory must be injected because this code cannot depend on the remote index client)

561

maybe just loadExternalIndex, which more clearly describes what it *does* as opposed to where it's passed.

[kadircet]

• Address comments

[kadircet]

• Merge with https://reviews.llvm.org/D90751

clang-tools-extra/clangd/CMakeLists.txt

 
 target_link_libraries(clangDaemon
   PRIVATE
   ${LLVM_PTHREAD_LIB}
 
   clangTidy
   ${ALL_CLANG_TIDY_CHECKS}
 
-  clangdRemoteIndex
   clangdSupport
   )
 
 add_subdirectory(refactor/tweaks)
 if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
   # FIXME: Make fuzzer not use linux-specific APIs, build it everywhere.
   add_subdirectory(fuzzer)
 endif()

clang-tools-extra/clangd/index/ProjectAware.h

 namespace clang {
 namespace clangd {
 
 /// A functor to create an index for an external index specification. Functor
 /// should perform any high latency operation in a separate thread through
 /// AsyncTaskRunner.
 using IndexFactory = std::function<std::unique_ptr<SymbolIndex>(
     const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
 
-/// Returns an index that answers queries using external indices. IndexGenerator
-/// can be used to customize how to generate an index from an external source.
-/// The default implementation loads the index asynchronously on the
-/// AsyncTaskRunner. The index will appear empty until loaded.
-std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory = nullptr);
+/// Returns an index that answers queries using external indices. IndexFactory
+/// specifies how to generate an index from an external source.
+/// IndexFactory must be injected because this code cannot depend on the remote
+/// index client.
+std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory);
 } // namespace clangd
 } // namespace clang
 
 #endif

clang-tools-extra/clangd/index/ProjectAware.cpp

 #include "Config.h"
 #include "index/Index.h"
 #include "index/MemIndex.h"
 #include "index/Merge.h"
 #include "index/Ref.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
-#include "index/remote/Client.h"
 #include "support/Logger.h"
 #include "support/Threading.h"
 #include "support/Trace.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
if (!C.Index.External)
     return nullptr;
   const auto &External = *C.Index.External;
   std::lock_guard<std::mutex> Lock(Mu);
   auto Entry = IndexForSpec.try_emplace(External, nullptr);
   if (Entry.second)
     Entry.first->getSecond() = Gen(External, Tasks);
   return Entry.first->second.get();
 }
-
-std::unique_ptr<SymbolIndex>
-defaultFactory(const Config::ExternalIndexSpec &External,
-               AsyncTaskRunner &Tasks) {
-  switch (External.Kind) {
-  case Config::ExternalIndexSpec::Server:
-    log("Associating {0} with remote index at {1}.", External.MountPoint,
-        External.Location);
-    return remote::getClient(External.Location, External.MountPoint);
-  case Config::ExternalIndexSpec::File:
-    log("Associating {0} with monolithic index at {1}.", External.MountPoint,
-        External.Location);
-    auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
-    Tasks.runAsync("Load-index:" + External.Location,
-                   [File = External.Location, PlaceHolder = NewIndex.get()] {
-                     if (auto Idx = loadIndex(File, /*UseDex=*/true))
-                       PlaceHolder->reset(std::move(Idx));
-                   });
-    return std::move(NewIndex);
-  }
-  llvm_unreachable("Invalid ExternalIndexKind.");
-}
 } // namespace
 
 std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory Gen) {
-  return std::make_unique<ProjectAwareIndex>(Gen ? std::move(Gen)
-                                                 : defaultFactory);
+  assert(Gen);
+  return std::make_unique<ProjectAwareIndex>(std::move(Gen));
 }
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangdLSPServer.h"
 #include "CodeComplete.h"
 #include "Features.inc"
 #include "PathMapping.h"
 #include "Protocol.h"
 #include "Transport.h"
 #include "index/Background.h"
+#include "index/Index.h"
+#include "index/Merge.h"
+#include "index/ProjectAware.h"
 #include "index/Serialization.h"
 #include "index/remote/Client.h"
 #include "refactor/Rename.h"
 #include "support/Path.h"
 #include "support/Shutdown.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/Basic/Version.h"

 #include "llvm/Support/raw_ostream.h"
 #include <chrono>
 #include <cstdlib>
 #include <iostream>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <thread>
+#include <vector>
 
 #ifndef _WIN32
 #include <unistd.h>
 #endif
 
 namespace clang {
 namespace clangd {
 

 };
 
 #ifdef _WIN32
 const char TestScheme::TestDir[] = "C:\\clangd-test";
 #else
 const char TestScheme::TestDir[] = "/clangd-test";
 #endif
 
+std::unique_ptr<SymbolIndex>
+loadExternalIndex(const Config::ExternalIndexSpec &External,
+                  AsyncTaskRunner &Tasks) {
+  switch (External.Kind) {
+  case Config::ExternalIndexSpec::Server:
+    log("Associating {0} with remote index at {1}.", External.MountPoint,
+        External.Location);
+    return remote::getClient(External.Location, External.MountPoint);
+  case Config::ExternalIndexSpec::File:
+    log("Associating {0} with monolithic index at {1}.", External.MountPoint,
+        External.Location);
+    auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
+    Tasks.runAsync("Load-index:" + External.Location,
+                   [File = External.Location, PlaceHolder = NewIndex.get()] {
+                     if (auto Idx = loadIndex(File, /*UseDex=*/true))
+                       PlaceHolder->reset(std::move(Idx));
+                   });
+    return std::move(NewIndex);
+  }
+  llvm_unreachable("Invalid ExternalIndexKind.");
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
 
 enum class ErrorResultCode : int {
   NoShutdownRequest = 1,
   CantRunAsXPCService = 2,
   CheckFailed = 3
)";
   case PCHStorageFlag::Disk:
     Opts.StorePreamblesInMemory = false;
     break;
   }
   if (!ResourceDir.empty())
     Opts.ResourceDir = ResourceDir;
   Opts.BuildDynamicSymbolIndex = EnableIndex;
   Opts.CollectMainFileRefs = CollectMainFileRefs;
+  std::vector<std::unique_ptr<SymbolIndex>> IdxStack;
   std::unique_ptr<SymbolIndex> StaticIdx;
   std::future<void> AsyncIndexLoad; // Block exit while loading the index.
   if (EnableIndex && !IndexFile.empty()) {
     // Load the index asynchronously. Meanwhile SwapIndex returns no results.
     SwapIndex *Placeholder;
     StaticIdx.reset(Placeholder = new SwapIndex(std::make_unique<MemIndex>()));
     AsyncIndexLoad = runAsync<void>([Placeholder] {
       if (auto Idx = loadIndex(IndexFile, /*UseDex=*/true))
if (IndexFile.empty()) {
       EnableBackgroundIndex = false;
     } else {
       elog("When enabling remote index, IndexFile should not be specified. "
            "Only one can be used at time. Remote index will ignored.");
     }
   }
 #endif
   Opts.BackgroundIndex = EnableBackgroundIndex;
-  Opts.StaticIndex = StaticIdx.get();
+  auto PAI = createProjectAwareIndex(loadExternalIndex);
+  if (StaticIdx) {
+    IdxStack.emplace_back(std::move(StaticIdx));
+    IdxStack.emplace_back(
+        std::make_unique<MergedIndex>(PAI.get(), IdxStack.back().get()));
+    Opts.StaticIndex = IdxStack.back().get();
+  } else {
+    Opts.StaticIndex = PAI.get();
+  }
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.BuildRecoveryAST = RecoveryAST;
   Opts.PreserveRecoveryASTType = RecoveryASTType;
   Opts.FoldingRanges = FoldingRanges;
 
   Opts.CodeComplete.IncludeIneligibleResults = IncludeIneligibleResults;
   Opts.CodeComplete.Limit = LimitResults;
   if (CompletionStyle.getNumOccurrences())