[clangd][tests] Bump timeouts in TUSchedulerTests to 60 secs

[kadircet]

Summary

There are some slow/congested bots, that can't go idle in 10 secs, see https://github.com/llvm/llvm-project/issues/64964

[hiraditya]

not sure if this is the right way to fix these tests. The problem is if a device is constraiend, this will further slow down the device and create more backlogs. Can we allow a way to skip these tests based on a flag/environment variable?

[sammccall]

In D159338#4635010, @hiraditya wrote:

not sure if this is the right way to fix these tests. The problem is if a device is constraiend, this will further slow down the device and create more backlogs. Can we allow a way to skip these tests based on a flag/environment variable?

The affected tests aren't fundamentally slower or less important than others (I expect many tests are very slow at the same time on such systems).
So skipping them doesn't seem appropriate.

The reason these deadlines exist in these tests is they coordinate multiple threads, and we want a deadlock bug (which you'd also see on healthy buildbots) to show up as a test failure rather than as the test running forever, which is harder to debug.
It's nice if it does so quickly, but if the buildbots are too slow, a longer deadline works fine for this purpose.

It will "further slow down the device" in the sense that running more tests slows down the device compared to running fewer tests, but again that's true for all tests, and we don't have reason to believe these ones are unreasonably expensive. (If so, we should understand why)

I believe @kadircet is looking into the possibility of of reducing/disabling test sharding of clangd tests on bots that don't have enough resources to gain throughput from it.

clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp

std::atomic<int> CallbackCount(0);
     updateWithDiags(S, Path, ";request no diags", WantDiagnostics::No,
                     [&](std::vector<Diag>) {
                       ADD_FAILURE() << "no diags should not be called back";
                     });
     updateWithDiags(S, Path, ";auto (produces)", WantDiagnostics::Auto,
                     [&](std::vector<Diag>) { ++CallbackCount; });
     Ready.notify();
 
-    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   }
   EXPECT_EQ(2, CallbackCount);
 }
 
 TEST_F(TUSchedulerTests, Debounce) {
   auto Opts = optsForTest();
   Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(500));
   TUScheduler S(CDB, Opts, captureDiags());
   auto Path = testPath("foo.cpp");
   // Issue a write that's going to be debounced away.
   updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,
                   [&](std::vector<Diag>) {
                     ADD_FAILURE()
                         << "auto should have been debounced and canceled";
                   });
   // Sleep a bit to verify that it's really debounce that's holding diagnostics.
   std::this_thread::sleep_for(std::chrono::milliseconds(50));
 
   // Issue another write, this time we'll wait for its diagnostics.
   Notification N;
   updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,
                   [&](std::vector<Diag>) { N.notify(); });
-  EXPECT_TRUE(N.wait(timeoutSeconds(5)));
+  EXPECT_TRUE(N.wait(timeoutSeconds(60)));
 
   // Once we start shutting down the TUScheduler, this one becomes a dead write.
   updateWithDiags(S, Path, "auto (discarded)", WantDiagnostics::Auto,
                   [&](std::vector<Diag>) {
                     ADD_FAILURE()
                         << "auto should have been discarded (dead write)";
                   });
 }
std::vector<StringRef> DiagsSeen, ReadsSeen, ReadsCanceled;
     Read("R1")();
     Update("U2")();
     Read("R2A")();
     Read("R2B");
     Update("U3");
     Read("R3")();
     Proceed.notify();
 
-    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   }
   EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))
       << "U1 and all dependent reads were cancelled. "
          "U2 has a dependent read R2A. "
          "U3 was not cancelled.";
   EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))
       << "All reads other than R2B were cancelled";
   EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))
       << "All reads other than R2B were cancelled";
 }
 
 TEST_F(TUSchedulerTests, InvalidationNoCrash) {
   auto Path = testPath("foo.cpp");
   TUScheduler S(CDB, optsForTest(), captureDiags());
 
   Notification StartedRunning;
   Notification ScheduledChange;
   // We expect invalidation logic to not crash by trying to invalidate a running
   // request.
   S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   S.runWithAST(
       "invalidatable-but-running", Path,
       [&](llvm::Expected<InputsAndAST> AST) {
         StartedRunning.notify();
         ScheduledChange.wait();
         ASSERT_TRUE(bool(AST));
       },
       TUScheduler::InvalidateOnUpdate);
   StartedRunning.wait();
   S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
   ScheduledChange.notify();
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 }
 
 TEST_F(TUSchedulerTests, Invalidation) {
   auto Path = testPath("foo.cpp");
   TUScheduler S(CDB, optsForTest(), captureDiags());
   std::atomic<int> Builds(0), Actions(0);
 
   Notification Start;
TEST_F(TUSchedulerTests, Invalidation) {
   S.runWithAST(
       "invalidatable", Path,
       [&](llvm::Expected<InputsAndAST> AST) {
         ++Actions;
         EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";
       },
       TUScheduler::InvalidateOnUpdate);
   Start.notify();
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 
   EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";
   EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";
 }
 
 // We don't invalidate requests for updates that don't change the file content.
 // These are mostly "refresh this file" events synthesized inside clangd itself.
 // (Usually the AST rebuild is elided after verifying that all inputs are
S.runWithAST(
             << "Should not invalidate based on an update with same content: "
             << llvm::toString(AST.takeError());
       },
       TUScheduler::InvalidateOnUpdate);
   updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
     ADD_FAILURE() << "Shouldn't build, identical to previous";
   });
   Start.notify();
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 
   EXPECT_EQ(1, Actions.load()) << "All actions should run";
 }
 
 TEST_F(TUSchedulerTests, ManyUpdates) {
   const int FilesCount = 3;
   const int UpdatesPerFile = 10;
 
for (int FileI = 0; FileI < FilesCount; ++FileI) {
 
                 std::lock_guard<std::mutex> Lock(Mut);
                 ++TotalPreambleReads;
                 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
               });
         }
       }
     }
-    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   } // TUScheduler destructor waits for all operations to finish.
 
   std::lock_guard<std::mutex> Lock(Mut);
   // Updates might get coalesced in preamble thread and result in dropping
   // diagnostics for intermediate snapshots.
   EXPECT_GE(TotalUpdates, FilesCount);
   EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
   // We should receive diags for last update.
TEST_F(TUSchedulerTests, EvictedAST) {
   auto Baz = testPath("baz.cpp");
 
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
   // Build one file in advance. We will not access it later, so it will be the
   // one that the cache will evict.
   updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,
                      [&BuiltASTCounter]() { ++BuiltASTCounter; });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_EQ(BuiltASTCounter.load(), 1);
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
 
   // Build two more files. Since we can retain only 2 ASTs, these should be
   // the ones we see in the cache later.
   updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,
                      [&BuiltASTCounter]() { ++BuiltASTCounter; });
   updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,
                      [&BuiltASTCounter]() { ++BuiltASTCounter; });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_EQ(BuiltASTCounter.load(), 3);
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));
 
   // Check only the last two ASTs are retained.
   ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));
 
   // Access the old file again.
   updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,
                      [&BuiltASTCounter]() { ++BuiltASTCounter; });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_EQ(BuiltASTCounter.load(), 4);
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
 
   // Check the AST for foo.cpp is retained now and one of the others got
   // evicted.
   EXPECT_THAT(S.getFilesWithCachedAST(),
               UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));
TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {
 
   auto Foo = testPath("foo.cpp");
   auto FooInputs = getInputs(Foo, "int x=1;");
   auto Bar = testPath("bar.cpp");
   auto BarInputs = getInputs(Bar, "int x=2;");
 
   // After opening Foo then Bar, AST cache contains Bar.
   S.update(Foo, FooInputs, WantDiagnostics::Auto);
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   S.update(Bar, BarInputs, WantDiagnostics::Auto);
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
 
   // Any number of no-op updates to Foo don't dislodge Bar from the cache.
   S.update(Foo, FooInputs, WantDiagnostics::Auto);
   S.update(Foo, FooInputs, WantDiagnostics::Auto);
   S.update(Foo, FooInputs, WantDiagnostics::Auto);
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
   // In fact each file has been built only once.
   ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);
   ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);
 }
 
 TEST_F(TUSchedulerTests, EmptyPreamble) {
   TUScheduler S(CDB, optsForTest());
S.runWithPreamble(
       "getNonEmptyPreamble", Foo, TUScheduler::Stale,
       [&](Expected<InputsAndPreamble> Preamble) {
         // We expect to get a non-empty preamble.
         EXPECT_GT(
             cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
             0u);
       });
   // Wait while the preamble is being built.
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 
   // Update the file which results in an empty preamble.
   S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);
   // Wait while the preamble is being built.
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   S.runWithPreamble(
       "getEmptyPreamble", Foo, TUScheduler::Stale,
       [&](Expected<InputsAndPreamble> Preamble) {
         // We expect to get an empty preamble.
         EXPECT_EQ(
             cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
             0u);
       });
TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {
   int func() {
     return tar::foo());
   }
   } // namespace ns
   )cpp";
   // Update the file which results in an empty preamble.
   S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);
   // Wait while the preamble is being built.
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   Notification TaskRun;
   S.runWithPreamble(
       "ASTSignals", Foo, TUScheduler::Stale,
       [&](Expected<InputsAndPreamble> IP) {
         ASSERT_FALSE(!IP);
         std::vector<std::pair<StringRef, int>> NS;
         for (const auto &P : IP->Signals->RelatedNamespaces)
           NS.emplace_back(P.getKey(), P.getValue());
TEST_F(TUSchedulerTests, RunWaitsForPreamble) {
   for (int I = 0; I < ReadsToSchedule; ++I) {
     S.runWithPreamble(
         "test", Foo, TUScheduler::Stale,
         [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {
           std::lock_guard<std::mutex> Lock(PreamblesMut);
           Preambles[I] = cantFail(std::move(IP)).Preamble;
         });
   }
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   // Check all actions got the same non-null preamble.
   std::lock_guard<std::mutex> Lock(PreamblesMut);
   ASSERT_NE(Preambles[0], nullptr);
   ASSERT_THAT(Preambles, Each(Preambles[0]));
 }
 
 TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {
   TUScheduler S(CDB, optsForTest(), captureDiags());
std::string SourceContents = R"cpp(
     )cpp";
 
   // Return value indicates if the updated callback was received.
   auto DoUpdate = [&](std::string Contents) -> bool {
     std::atomic<bool> Updated(false);
     Updated = false;
     updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,
                     [&Updated](std::vector<Diag>) { Updated = true; });
-    bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(10));
+    bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(60));
     if (!UpdateFinished)
       ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";
     return Updated;
   };
 
   // Test that subsequent updates with the same inputs do not cause rebuilds.
   ASSERT_TRUE(DoUpdate(SourceContents));
   ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
updateWithDiags(
       S, Source, Inputs, WantDiagnostics::Yes,
       [&DiagCount](std::vector<Diag> Diags) {
         ++DiagCount;
         EXPECT_THAT(Diags,
                     ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),
                                 Field(&Diag::Message,
                                       "use of undeclared identifier 'b'")));
       });
-  S.blockUntilIdle(timeoutSeconds(10));
+  S.blockUntilIdle(timeoutSeconds(60));
 
   FS.Files[HeaderB] = "int b;";
   FS.Timestamps[HeaderB] = time_t(1);
 
   // The addition of the missing header file triggers a rebuild, no errors.
   updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
                   [&DiagCount](std::vector<Diag> Diags) {
                     ++DiagCount;
                     EXPECT_THAT(Diags, IsEmpty());
                   });
 
   // Ensure previous assertions are done before we touch the FS again.
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   // Add the high-priority header file, which should reintroduce the error.
   FS.Files[HeaderA] = "int a;";
   FS.Timestamps[HeaderA] = time_t(1);
 
   // This isn't detected: we don't stat a/foo.h to validate the preamble.
   updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
                   [&DiagCount](std::vector<Diag> Diags) {
                     ++DiagCount;
                     ADD_FAILURE()
                         << "Didn't expect new diagnostics when adding a/foo.h";
                   });
 
   // Forcing the reload should cause a rebuild.
   Inputs.ForceRebuild = true;
   updateWithDiags(
       S, Source, Inputs, WantDiagnostics::Yes,
       [&DiagCount](std::vector<Diag> Diags) {
         ++DiagCount;
         ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));
       });
 
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_EQ(DiagCount, 3U);
 }
 
 TEST_F(TUSchedulerTests, NoChangeDiags) {
   trace::TestTracer Tracer;
   TUScheduler S(CDB, optsForTest(), captureDiags());
 
   auto FooCpp = testPath("foo.cpp");
   const auto *Contents = "int a; int b;";
 
   EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
   updateWithDiags(
       S, FooCpp, Contents, WantDiagnostics::No,
       [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
   S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {
     // Make sure the AST was actually built.
     cantFail(std::move(IA));
   });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
   EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));
 
   // Even though the inputs didn't change and AST can be reused, we need to
   // report the diagnostics, as they were not reported previously.
   std::atomic<bool> SeenDiags(false);
   updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,
                   [&](std::vector<Diag>) { SeenDiags = true; });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   ASSERT_TRUE(SeenDiags);
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));
   EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
 
   // Subsequent request does not get any diagnostics callback because the same
   // diags have previously been reported and the inputs didn't change.
   updateWithDiags(
       S, FooCpp, Contents, WantDiagnostics::Auto,
       [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 }
 
 TEST_F(TUSchedulerTests, Run) {
   for (bool Sync : {false, true}) {
     auto Opts = optsForTest();
     if (Sync)
       Opts.AsyncThreadsCount = 0;
     TUScheduler S(CDB, Opts);
     std::atomic<int> Counter(0);
     S.run("add 1", /*Path=*/"", [&] { ++Counter; });
     S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
-    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
     EXPECT_EQ(Counter.load(), 3);
 
     Notification TaskRun;
     Key<int> TestKey;
     WithContextValue CtxWithKey(TestKey, 10);
     const char *Path = "somepath";
     S.run("props context", Path, [&] {
       EXPECT_EQ(Context::current().getExisting(TestKey), 10);
TEST_F(TUSchedulerTests, AsyncPreambleThread) {
   Notification Ready;
   TUScheduler S(CDB, optsForTest(),
                 std::make_unique<BlockPreambleThread>(InputsV1, Ready));
 
   Path File = testPath("foo.cpp");
   auto PI = getInputs(File, "");
   PI.Version = InputsV0.str();
   S.update(File, PI, WantDiagnostics::Auto);
-  S.blockUntilIdle(timeoutSeconds(10));
+  S.blockUntilIdle(timeoutSeconds(60));
 
   // Block preamble builds.
   PI.Version = InputsV1.str();
   // Issue second update which will block preamble thread.
   S.update(File, PI, WantDiagnostics::Auto);
 
   Notification RunASTAction;
   // Issue an AST read, which shouldn't be blocked and see latest version of the
TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {
   };
 
   int PreamblePublishCount = 0;
   TUScheduler S(CDB, optsForTest(),
                 std::make_unique<PreamblePublishCounter>(PreamblePublishCount));
 
   Path File = testPath("foo.cpp");
   S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
-  S.blockUntilIdle(timeoutSeconds(10));
+  S.blockUntilIdle(timeoutSeconds(60));
   EXPECT_EQ(PreamblePublishCount, 1);
   // Same contents, no publish.
   S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
-  S.blockUntilIdle(timeoutSeconds(10));
+  S.blockUntilIdle(timeoutSeconds(60));
   EXPECT_EQ(PreamblePublishCount, 1);
   // New contents, should publish.
   S.update(File, getInputs(File, "#define FOO"), WantDiagnostics::Auto);
-  S.blockUntilIdle(timeoutSeconds(10));
+  S.blockUntilIdle(timeoutSeconds(60));
   EXPECT_EQ(PreamblePublishCount, 2);
 }
 
 TEST_F(TUSchedulerTests, PublishWithStalePreamble) {
   // Callbacks that blocks the preamble thread after the first preamble is
   // built and stores preamble/main-file versions for diagnostics released.
   class BlockPreambleThread : public ParsingCallbacks {
   public:
     using DiagsCB = std::function<void(ParsedAST &)>;
     BlockPreambleThread(Notification &UnblockPreamble, DiagsCB CB)
         : UnblockPreamble(UnblockPreamble), CB(std::move(CB)) {}
 
     void onPreambleAST(
         PathRef Path, llvm::StringRef Version, CapturedASTCtx,
         std::shared_ptr<const include_cleaner::PragmaIncludes>) override {
       if (BuildBefore)
-        ASSERT_TRUE(UnblockPreamble.wait(timeoutSeconds(5)))
+        ASSERT_TRUE(UnblockPreamble.wait(timeoutSeconds(60)))
             << "Expected notification";
       BuildBefore = true;
     }
 
     void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
       CB(AST);
     }
 
TEST_F(TUSchedulerTests, PublishWithStalePreamble) {
   PI.Contents = "#define BAR\n" + PI.Version;
   ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "2"));
 
   PI.Version = "3";
   PI.Contents = "#define FOO\n" + PI.Version;
   ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "3"));
 
   UnblockPreamble.notify();
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(5)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 
   // Make sure that we have eventual consistency.
   EXPECT_THAT(Collector.diagVersions().back(), Pair(PI.Version, PI.Version));
 
   // Check that WantDiagnostics::No doesn't emit any diags.
   PI.Version = "4";
   PI.Contents = "#define FOO\n" + PI.Version;
   S.update(File, PI, WantDiagnostics::No);
-  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(5)));
+  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(Collector.diagVersions().back(), Pair("3", "3"));
 }
 
 // If a header file is missing from the CDB (or inferred using heuristics), and
 // it's included by another open file, then we parse it using that files flags.
 TEST_F(TUSchedulerTests, IncluderCache) {
   static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),
                      Main3 = testPath("main3.cpp"),
getCompileCommand(PathRef File) const override {
       return Basic;
     }
 
     std::atomic<bool> FailAll{false};
   } CDB;
   TUScheduler S(CDB, optsForTest());
   auto GetFlags = [&](PathRef Header) {
     S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);
-    EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
     Notification CmdDone;
     tooling::CompileCommand Cmd;
     S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,
                       [&](llvm::Expected<InputsAndPreamble> Inputs) {
                         ASSERT_FALSE(!Inputs) << Inputs.takeError();
                         Cmd = std::move(Inputs->Command);
                         CmdDone.notify();
                       });
     CmdDone.wait();
-    EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
     return Cmd.CommandLine;
   };
 
   for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})
     FS.Files[Path] = ";";
 
   // Initially these files have normal commands from the CDB.
   EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";
   EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";
 
   // Now make Main include the others, and some should pick up its flags.
   const char *AllIncludes = R"cpp(
     #include "no_cmd.h"
     #include "ok.h"
     #include "unreliable.h"
   )cpp";
   S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))
       << "Included from main file, has no own command";
   EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
       << "Included from main file, own command is heuristic";
   EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))
       << "Included from main file, but own command is used";
   EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))
       << "Not included from main file";
 
   // Open another file - it won't overwrite the associations with Main.
   std::string SomeIncludes = R"cpp(
     #include "no_cmd.h"
     #include "not_included.h"
   )cpp";
   S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(GetFlags(NoCmd),
               AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
       << "mainfile association is stable";
   EXPECT_THAT(GetFlags(NotIncluded),
               AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))
       << "new headers are associated with new mainfile";
 
   // Remove includes from main - this marks the associations as invalid but
   // doesn't actually remove them until another preamble claims them.
   S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(GetFlags(NoCmd),
               AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
       << "mainfile association not updated yet!";
 
   // Open yet another file - this time it claims the associations.
   S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
       << "association invalidated and then claimed by main3";
   EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
       << "association invalidated but not reclaimed";
   EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))
       << "association still valid";
 
   // Delete the file from CDB, it should invalidate the associations.
   CDB.FailAll = true;
   EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN3")))
       << "association should've been invalidated.";
   // Also run update for Main3 to invalidate the preeamble to make sure next
   // update populates include cache associations.
   S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   // Re-add the file and make sure nothing crashes.
   CDB.FailAll = false;
   S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
-  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
   EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
       << "association invalidated and then claimed by main3";
 }
 
 TEST_F(TUSchedulerTests, PreservesLastActiveFile) {
   for (bool Sync : {false, true}) {
     auto Opts = optsForTest();
     if (Sync)
       Opts.AsyncThreadsCount = 0;
     TUScheduler S(CDB, Opts);
 
     auto CheckNoFileActionsSeesLastActiveFile =
         [&](llvm::StringRef LastActiveFile) {
-          ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+          ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
           std::atomic<int> Counter(0);
           // We only check for run and runQuick as runWithAST and
           // runWithPreamble is always bound to a file.
           S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {
             ++Counter;
             EXPECT_EQ(LastActiveFile, boundPath());
           });
           S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {
             ++Counter;
             EXPECT_EQ(LastActiveFile, boundPath());
           });
-          ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+          ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
           EXPECT_EQ(2, Counter.load());
         };
 
     // Check that we see no file initially
     CheckNoFileActionsSeesLastActiveFile("");
 
     // Now check that every action scheduled with a particular file changes the
     // LastActiveFile.
std::vector<std::string> Filenames;
     std::vector<std::string> BuiltFilenames;
     TUScheduler S(CDB, Opts,
                   std::make_unique<CaptureBuiltFilenames>(BuiltFilenames));
     for (unsigned I = 0; I < NumRequests; ++I) {
       auto Path = testPath(std::to_string(I) + ".cc");
       Filenames.push_back(Path);
       S.update(Path, getInputs(Path, ""), WantDiagnostics::Yes);
     }
-    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
+    ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));
 
     // The throttler saw all files, and we built them.
     EXPECT_THAT(Throttler.Acquires,
                 testing::UnorderedElementsAreArray(Filenames));
     EXPECT_THAT(BuiltFilenames,
                 testing::UnorderedElementsAreArray(Filenames));
     // We built the files in reverse order that the throttler saw them.
     EXPECT_THAT(BuiltFilenames,