[clangd] Include compile command inference in fileStatus API

[hokein]

clangd/GlobalCompilationDatabase.cpp

70

default flags seems not quite clear to me, maybe use fallback compile command for unknown file?

[kadircet]

What about surfacing compile command itself as well? IMO, it is an important information that we always extract from logs by digging currently.

clangd/GlobalCompilationDatabase.cpp

 tooling::CompileCommand
 GlobalCompilationDatabase::getFallbackCommand(PathRef File) const {
   std::vector<std::string> Argv = {getFallbackClangPath()};
   // Clang treats .h files as C by default, resulting in unhelpful diagnostics.
   // Parsing as Objective C++ is friendly to more cases.
   if (llvm::sys::path::extension(File) == ".h")
     Argv.push_back("-xobjective-c++-header");
   Argv.push_back(File);
-  return tooling::CompileCommand(llvm::sys::path::parent_path(File),
+  tooling::CompileCommand Result(llvm::sys::path::parent_path(File),
                                  llvm::sys::path::filename(File),
                                  std::move(Argv),
                                  /*Output=*/"");
+  Result.Heuristic = "default flags for unknown file";
+  return Result;
 }
 
 DirectoryBasedGlobalCompilationDatabase::
     DirectoryBasedGlobalCompilationDatabase(
         llvm::Optional<Path> CompileCommandsDir)
     : CompileCommandsDir(std::move(CompileCommandsDir)) {}
 
 DirectoryBasedGlobalCompilationDatabase::

clangd/Protocol.h

 /// Clangd extension: indicates the current state of the file in clangd,
 /// sent from server via the `textDocument/clangd.fileStatus` notification.
 struct FileStatus {
   /// The text document's URI.
   URIForFile uri;
   /// The human-readable string presents the current state of the file, can be
   /// shown in the UI (e.g. status bar).
   std::string state;
-  // FIXME: add detail messages.
+  /// Indicates that the compile command is a guess, and describes why.
+  /// e.g. "inferred from foo/bar.cpp" or "default flags for unknown file".
+  llvm::Optional<std::string> compileCommandHeuristic;
 };
 llvm::json::Value toJSON(const FileStatus &FStatus);
 
 } // namespace clangd
 } // namespace clang
 
 namespace llvm {
 template <> struct format_provider<clang::clangd::Position> {

clangd/Protocol.cpp

 llvm::json::Value toJSON(const DocumentHighlight &DH) {
   return llvm::json::Object{
       {"range", toJSON(DH.range)},
       {"kind", static_cast<int>(DH.kind)},
   };
 }
 
 llvm::json::Value toJSON(const FileStatus &FStatus) {
-  return llvm::json::Object{
+  llvm::json::Object Result{
       {"uri", FStatus.uri},
       {"state", FStatus.state},
   };
+  if (FStatus.compileCommandHeuristic)
+    Result["compileCommandHeuristic"] = FStatus.compileCommandHeuristic;
+  return Result;
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
                               const DocumentHighlight &V) {
   O << V.range;
   if (V.kind == DocumentHighlightKind::Read)
     O << "(r)";
   if (V.kind == DocumentHighlightKind::Write)

clangd/TUScheduler.h

enum State {
     Queued,           // The TU is pending in the thread task queue to be built.
     RunningAction,    // Starting running actions on the TU.
     BuildingPreamble, // The preamble of the TU is being built.
     BuildingFile,     // The TU is being built. It is only emitted when building
                       // the AST for diagnostics in write action (update).
     Idle, // Indicates the worker thread is idle, and ready to run any upcoming
           // actions.
   };
-  TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
-  State S;
+  State S = State::Idle;
   /// The name of the action currently running, e.g. Update, GoToDef, Hover.
   /// Empty if we are in the idle state.
   std::string Name;
 };
 
 // Internal status of the TU in TUScheduler.
 struct TUStatus {
   struct BuildDetails {
     /// Indicates whether clang failed to build the TU.
     bool BuildFailed = false;
     /// Indicates whether we reused the prebuilt AST.
     bool ReuseAST = false;
   };
   /// Serialize this to an LSP file status item.
   FileStatus render(PathRef File) const;
 
   TUAction Action;
   BuildDetails Details;
+  std::string CompileCommandHeuristic;
 };
 
 class ParsingCallbacks {
 public:
   virtual ~ParsingCallbacks() = default;
 
   /// Called on the AST that was built for emitting the preamble. The built AST
   /// contains only AST nodes from the #include directives at the start of the

clangd/TUScheduler.cpp

private:
   // processed.
   void run();
   /// Signal that run() should finish processing pending requests and exit.
   void stop();
   /// Adds a new task to the end of the request queue.
   void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
                  llvm::Optional<WantDiagnostics> UpdateType);
   /// Updates the TUStatus and emits it. Only called in the worker thread.
-  void emitTUStatus(TUAction FAction,
+  void emitTUStatus(TUAction::State State, llvm::StringRef Name,
                     const TUStatus::BuildDetails *Detail = nullptr);
 
   /// Determines the next action to perform.
   /// All actions that should never run are discarded.
   /// Returns a deadline for the next action. If it's expired, run now.
   /// scheduleLocked() is called again at the deadline, or if requests arrive.
   Deadline scheduleLocked();
   /// Should the first task in the queue be skipped instead of run?

 }
 
 ASTWorker::ASTWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache,
                      Semaphore &Barrier, bool RunSync,
                      steady_clock::duration UpdateDebounce,
                      bool StorePreamblesInMemory, ParsingCallbacks &Callbacks)
     : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),
       FileName(FileName), StorePreambleInMemory(StorePreamblesInMemory),
-      Callbacks(Callbacks), Status{TUAction(TUAction::Idle, ""),
-                                   TUStatus::BuildDetails()},
-      Barrier(Barrier), Done(false) {}
+      Callbacks(Callbacks), Barrier(Barrier), Done(false) {}
 
 ASTWorker::~ASTWorker() {
   // Make sure we remove the cached AST, if any.
   IdleASTs.take(this);
 #ifndef NDEBUG
   std::lock_guard<std::mutex> Lock(Mutex);
   assert(Done && "handle was not destroyed");
   assert(Requests.empty() && "unprocessed requests when destroying ASTWorker");
 #endif
 }
 
 void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags) {
   llvm::StringRef TaskName = "Update";
   auto Task = [=]() mutable {
     // Will be used to check if we can avoid rebuilding the AST.
     bool InputsAreTheSame =
         std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
         std::tie(Inputs.CompileCommand, Inputs.Contents);
 
     tooling::CompileCommand OldCommand = std::move(FileInputs.CompileCommand);
     bool PrevDiagsWereReported = DiagsWereReported;
     FileInputs = Inputs;
     DiagsWereReported = false;
-    emitTUStatus({TUAction::BuildingPreamble, TaskName});
+    emitTUStatus(TUAction::BuildingPreamble, TaskName);
     log("Updating file {0} with command [{1}] {2}", FileName,
         Inputs.CompileCommand.Directory,
         llvm::join(Inputs.CompileCommand.CommandLine, " "));
     // Rebuild the preamble and the AST.
     std::unique_ptr<CompilerInvocation> Invocation =
         buildCompilerInvocation(Inputs);
     if (!Invocation) {
       elog("Could not build CompilerInvocation for file {0}", FileName);
       // Remove the old AST if it's still in cache.
       IdleASTs.take(this);
       TUStatus::BuildDetails Details;
       Details.BuildFailed = true;
-      emitTUStatus({TUAction::BuildingPreamble, TaskName}, &Details);
+      emitTUStatus(TUAction::BuildingPreamble, TaskName, &Details);
       // Make sure anyone waiting for the preamble gets notified it could not
       // be built.
       PreambleWasBuilt.notify();
       return;
     }
 
     std::shared_ptr<const PreambleData> OldPreamble =
         getPossiblyStalePreamble();
bool CanReuseAST = InputsAreTheSame && (OldPreamble == NewPreamble);
       std::lock_guard<std::mutex> Lock(Mutex);
       LastBuiltPreamble = NewPreamble;
     }
     // Before doing the expensive AST reparse, we want to release our reference
     // to the old preamble, so it can be freed if there are no other references
     // to it.
     OldPreamble.reset();
     PreambleWasBuilt.notify();
-    emitTUStatus({TUAction::BuildingFile, TaskName});
+    emitTUStatus(TUAction::BuildingFile, TaskName);
     if (!CanReuseAST) {
       IdleASTs.take(this); // Remove the old AST if it's still in cache.
     } else {
       // Since we don't need to rebuild the AST, we might've already reported
       // the diagnostics for it.
       if (PrevDiagsWereReported) {
         DiagsWereReported = true;
         // Take a shortcut and don't report the diagnostics, since they should
         // not changed. All the clients should handle the lack of OnUpdated()
         // call anyway to handle empty result from buildAST.
         // FIXME(ibiryukov): the AST could actually change if non-preamble
         // includes changed, but we choose to ignore it.
         // FIXME(ibiryukov): should we refresh the cache in IdleASTs for the
         // current file at this point?
         log("Skipping rebuild of the AST for {0}, inputs are the same.",
             FileName);
         TUStatus::BuildDetails Details;
         Details.ReuseAST = true;
-        emitTUStatus({TUAction::BuildingFile, TaskName}, &Details);
+        emitTUStatus(TUAction::BuildingFile, TaskName, &Details);
         return;
       }
     }
 
     // We only need to build the AST if diagnostics were requested.
     if (WantDiags == WantDiagnostics::No)
       return;
 
auto Task = [=]() mutable {
     llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);
     if (!AST) {
       llvm::Optional<ParsedAST> NewAST =
           buildAST(FileName, std::move(Invocation), Inputs, NewPreamble);
       AST = NewAST ? llvm::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
       if (!(*AST)) { // buildAST fails.
         TUStatus::BuildDetails Details;
         Details.BuildFailed = true;
-        emitTUStatus({TUAction::BuildingFile, TaskName}, &Details);
+        emitTUStatus(TUAction::BuildingFile, TaskName, &Details);
       }
     } else {
       // We are reusing the AST.
       TUStatus::BuildDetails Details;
       Details.ReuseAST = true;
-      emitTUStatus({TUAction::BuildingFile, TaskName}, &Details);
+      emitTUStatus(TUAction::BuildingFile, TaskName, &Details);
     }
     // We want to report the diagnostics even if this update was cancelled.
     // It seems more useful than making the clients wait indefinitely if they
     // spam us with updates.
     // Note *AST can still be null if buildAST fails.
     if (*AST) {
       {
         std::lock_guard<std::mutex> Lock(DiagsMu);
if (RunSync) {
     assert(!Done && "running a task after stop()");
     Requests.push_back(
         {std::move(Task), Name, steady_clock::now(),
          Context::current().derive(kFileBeingProcessed, FileName), UpdateType});
   }
   RequestsCV.notify_all();
 }
 
-void ASTWorker::emitTUStatus(TUAction Action,
+void ASTWorker::emitTUStatus(TUAction::State State, llvm::StringRef Name,
                              const TUStatus::BuildDetails *Details) {
-  Status.Action = std::move(Action);
+  Status.Action.S = State;
+  Status.Action.Name = Name;
   if (Details)
     Status.Details = *Details;
+  Status.CompileCommandHeuristic = FileInputs.CompileCommand.Heuristic;
   std::lock_guard<std::mutex> Lock(DiagsMu);
   // Do not emit TU statuses when the ASTWorker is shutting down.
   if (ReportDiagnostics) {
     Callbacks.onFileUpdated(FileName, Status);
   }
 }
 
 void ASTWorker::run() {
Request Req;
         // Tracing: we have a next request, attribute this sleep to it.
         llvm::Optional<WithContext> Ctx;
         llvm::Optional<trace::Span> Tracer;
         if (!Requests.empty()) {
           Ctx.emplace(Requests.front().Ctx.clone());
           Tracer.emplace("Debounce");
           SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
           if (!(Wait == Deadline::infinity())) {
-            emitTUStatus({TUAction::Queued, Req.Name});
+            emitTUStatus(TUAction::Queued, Req.Name);
             SPAN_ATTACH(*Tracer, "sleep_ms",
                         std::chrono::duration_cast<std::chrono::milliseconds>(
                             Wait.time() - steady_clock::now())
                             .count());
           }
         }
 
         wait(Lock, RequestsCV, Wait);
       }
       Req = std::move(Requests.front());
       // Leave it on the queue for now, so waiters don't see an empty queue.
     } // unlock Mutex
 
     {
       std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
       if (!Lock.owns_lock()) {
-        emitTUStatus({TUAction::Queued, Req.Name});
+        emitTUStatus(TUAction::Queued, Req.Name);
         Lock.lock();
       }
       WithContext Guard(std::move(Req.Ctx));
       trace::Span Tracer(Req.Name);
-      emitTUStatus({TUAction::RunningAction, Req.Name});
+      emitTUStatus(TUAction::RunningAction, Req.Name);
       Req.Action();
     }
 
     bool IsEmpty = false;
     {
       std::lock_guard<std::mutex> Lock(Mutex);
       Requests.pop_front();
       IsEmpty = Requests.empty();
     }
     if (IsEmpty)
-      emitTUStatus({TUAction::Idle, /*Name*/ ""});
+      emitTUStatus(TUAction::Idle, /*Name*/ "");
     RequestsCV.notify_all();
   }
 }
 
 Deadline ASTWorker::scheduleLocked() {
   if (Requests.empty())
     return Deadline::infinity(); // Wait for new requests.
   // Handle cancelled requests first so the rest of the scheduler doesn't.
if (HardwareConcurrency == 0)
     return 1;
   return HardwareConcurrency;
 }
 
 FileStatus TUStatus::render(PathRef File) const {
   FileStatus FStatus;
   FStatus.uri = URIForFile::canonicalize(File, /*TUPath=*/File);
   FStatus.state = renderTUAction(Action);
+  FStatus.compileCommandHeuristic = CompileCommandHeuristic;
   return FStatus;
 }
 
 struct TUScheduler::FileData {
   /// Latest inputs, passed to TUScheduler::update().
   std::string Contents;
   tooling::CompileCommand Command;
   ASTWorkerHandle Worker;

test/clangd/filestatus.test

 # RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"initializationOptions":{"clangdFileStatus": true},"trace":"off"}}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///main.cpp","languageId":"cpp","version":1,"text":"int x; int y = x;"}}}
 #      CHECK:  "method": "textDocument/clangd.fileStatus",
 # CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "compileCommandHeuristic": "default flags for unknown file"
 # CHECK-NEXT:    "state": "parsing includes",
 # CHECK-NEXT:    "uri": "{{.*}}/main.cpp"
 # CHECK-NEXT:  }
 ---
 {"jsonrpc":"2.0","id":3,"method":"shutdown"}
 ---
 {"jsonrpc":"2.0","method":"exit"}