[clangd] Triggering indexing on each translation unit with an error

[kadircet]

Summary

Background indexing triggers indexing only on the first TU that
includes a stale dependency.

This patch extends this behavior to also trigger indexing on TUs with errors.

[kadircet]

this requires keeping an inverted dependency graph, which can be quite expensive memory-wise.
In a project like chromium there are ~40k translation units, and it is safe to assume there are ~1k core headers just coming from std library,
that are most likely included by all of the translation units directly or transatively. This results in ~320MB of memory usage even if we just
store a pointer.

It is quite likely that there would be a lot more "base" headers like that, which could easily drive up the memory usage.

We might need some tradeoffs to limit that usage.

clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp

bool BackgroundIndexLoader::load(PathRef MainFile,
   }
   while (!ToVisit.empty()) {
     PathRef ShardIdentifier = ToVisit.front();
     ToVisit.pop();
 
     auto LoadResult = loadShard(ShardIdentifier, Storage);
     const CachedShard &CS = LoadResult.first;
     // Report the file as stale if it is the first time we see it.
-    // FIXME: We should still update staleness for main file, if it had errors.
-    if (!LoadResult.second && CS.IsStale)
+    if ((!LoadResult.second || MainShard.SI.HadErrors) && CS.IsStale)
       IsStale = true;
     for (PathRef Edge : CS.Edges) {
       if (InQueue.insert(Edge).second)
         ToVisit.push(Edge);
     }
   }
 
   return !IsStale;

clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp

 
 TEST_F(BackgroundIndexTest, ShardStorageTest) {
   MockFSProvider FS;
   FS.Files[testPath("root/A.h")] = R"cpp(
       void common();
       void f_b();
       class A_CC {};
       )cpp";
-  std::string A_CC = "";
   FS.Files[testPath("root/A.cc")] = R"cpp(
       #include "A.h"
       void g() { (void)common; }
       class B_CC : public A_CC {};
       )cpp";
 
   llvm::StringMap<std::string> Storage;
   size_t CacheHits = 0;
TEST(BackgroundQueueTest, Priority) {
   Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
 
   // After finishing, check the tasks that ran.
   ThreadPool.wait();
   EXPECT_GE(HiRan, 10u);
   EXPECT_EQ(LoRan, 0u);
 }
 
+TEST(BackgroundIndexLoaderTest, IndexAllTUsWithErrors) {
+  MockFSProvider FS;
+  FS.Files[testPath("root/A.h")] = "broken_header";
+  FS.Files[testPath("root/A.cc")] = R"cpp(#include "A.h")cpp";
+  FS.Files[testPath("root/B.cc")] = R"cpp(#include "A.h")cpp";
+
+  llvm::StringMap<std::string> Storage;
+  size_t CacheHits = 0;
+  MemoryShardStorage MSS(Storage, CacheHits);
+
+  auto CmdForFile = [](PathRef RelativePath) {
+    tooling::CompileCommand Cmd;
+    Cmd.Filename = testPath(RelativePath);
+    Cmd.Directory = testPath("root");
+    Cmd.CommandLine = {"clang++", testPath(RelativePath)};
+    return Cmd;
+  };
+  std::vector<Path> TUs = {"root/A.cc", "root/B.cc"};
+
+  {
+    OverlayCDB CDB(/*Base=*/nullptr);
+    for (PathRef RelPath : TUs)
+      CDB.setCompileCommand(RelPath, CmdForFile(RelPath));
+    BackgroundIndex Idx(Context::empty(), FS, CDB,
+                        [&](llvm::StringRef) { return &MSS; });
+    Idx.enqueue(TUs);
+    ASSERT_TRUE(Idx.blockUntilIdleForTest());
+  }
+  EXPECT_EQ(CacheHits, 0U);
+  EXPECT_THAT(Storage.keys(),
+              UnorderedElementsAre(testPath("root/A.cc"), testPath("root/B.cc"),
+                                   testPath("root/A.h")));
+  for (PathRef RelPath : TUs) {
+    EXPECT_THAT(MSS.loadShard(testPath(RelPath))
+                    ->Sources->lookup(("unittest:///" + RelPath).str()),
+                HadErrors());
+  }
+
+  FS.Files[testPath("root/A.h")] = "int broken_header;";
+  {
+    OverlayCDB CDB(/*Base=*/nullptr);
+    for (PathRef RelPath : TUs)
+      CDB.setCompileCommand(RelPath, CmdForFile(RelPath));
+
+    BackgroundIndex Idx(Context::empty(), FS, CDB,
+                        [&](llvm::StringRef) { return &MSS; });
+    Idx.enqueue(TUs);
+    ASSERT_TRUE(Idx.blockUntilIdleForTest());
+  }
+  for (PathRef RelPath : TUs) {
+    EXPECT_THAT(MSS.loadShard(testPath(RelPath))
+                    ->Sources->lookup(("unittest:///" + RelPath).str()),
+                Not(HadErrors()));
+  }
+}
+
 } // namespace clangd
 } // namespace clang