[clangd] Populate the parse options to CodeCompletion/SignatureHelp.

[hokein]

Summary

A followup of D79938.

[sammccall]

clang-tools-extra/clangd/CodeComplete.cpp

1108

this change looks suspicious, really needed?

clang-tools-extra/clangd/Compiler.h

57

this is the default already - only need to do this for scalar types that would otherwise have indeterminate default

[hokein]

Revert an accident change.

[hokein]

clang-tools-extra/clangd/CodeComplete.cpp

1108

oops, I think this is an accident change, reverted.

clang-tools-extra/clangd/Compiler.h

57

yeah, exactly, however clang was not happy with the ParseInputs {Command, FS, contents} usage, it gave -Wmissing-field-initializers warning.

clang-tools-extra/clangd/ClangdServer.cpp

if (!IP->Preamble) {
         {
           std::lock_guard<std::mutex> Lock(
               CachedCompletionFuzzyFindRequestMutex);
           SpecFuzzyFind->CachedReq =
               CachedCompletionFuzzyFindRequestByFile[File];
         }
       }
     }
+    ParseInputs ParseInput{IP->Command, FS, IP->Contents.str()};
+    ParseInput.Index = Index;
+    ParseInput.Opts.BuildRecoveryAST = BuildRecoveryAST;
+    ParseInput.Opts.PreserveRecoveryASTType = PreserveRecoveryASTType;
+
     // FIXME(ibiryukov): even if Preamble is non-null, we may want to check
     // both the old and the new version in case only one of them matches.
     CodeCompleteResult Result = clangd::codeComplete(
-        File, IP->Command, IP->Preamble, IP->Contents, Pos, FS,
-        CodeCompleteOpts, SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr);
+        File, Pos, IP->Preamble, ParseInput, CodeCompleteOpts,
+        SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr);
     {
       clang::clangd::trace::Span Tracer("Completion results callback");
       CB(std::move(Result));
     }
     if (SpecFuzzyFind && SpecFuzzyFind->NewReq.hasValue()) {
       std::lock_guard<std::mutex> Lock(CachedCompletionFuzzyFindRequestMutex);
       CachedCompletionFuzzyFindRequestByFile[File] =
           SpecFuzzyFind->NewReq.getValue();
auto Action = [Pos, FS = FSProvider.getFileSystem(), File = File.str(),
     if (!IP)
       return CB(IP.takeError());
 
     const auto *PreambleData = IP->Preamble;
     if (!PreambleData)
       return CB(llvm::createStringError(llvm::inconvertibleErrorCode(),
                                         "Failed to parse includes"));
 
-    CB(clangd::signatureHelp(File, IP->Command, *PreambleData, IP->Contents,
-                             Pos, FS, Index));
+    ParseInputs ParseInput{IP->Command, FS, IP->Contents.str()};
+    ParseInput.Index = Index;
+    ParseInput.Opts.BuildRecoveryAST = BuildRecoveryAST;
+    ParseInput.Opts.PreserveRecoveryASTType = PreserveRecoveryASTType;
+    CB(clangd::signatureHelp(File, Pos, *PreambleData, ParseInput));
   };
 
   // Unlike code completion, we wait for a preamble here.
   WorkScheduler.runWithPreamble("SignatureHelp", File, TUScheduler::Stale,
                                 std::move(Action));
 }
 
 llvm::Expected<tooling::Replacements>

clang-tools-extra/clangd/CodeComplete.h

 // After "std::string S; S." we might suggest members of std::string.
 // Signature help describes the parameters of a function as you type them.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 
+#include "Compiler.h"
 #include "Headers.h"
 #include "Protocol.h"
 #include "Quality.h"
 #include "index/Index.h"
 #include "index/Symbol.h"
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
 #include "support/Markup.h"

 /// If \p Preamble is nullptr, this runs code completion without compiling the
 /// code.
 ///
 /// If \p SpecFuzzyFind is set, a speculative and asynchronous fuzzy find index
 /// request (based on cached request) will be run before parsing sema. In case
 /// the speculative result is used by code completion (e.g. speculation failed),
 /// the speculative result is not consumed, and `SpecFuzzyFind` is only
 /// destroyed when the async request finishes.
-CodeCompleteResult codeComplete(PathRef FileName,
-                                const tooling::CompileCommand &Command,
+CodeCompleteResult codeComplete(PathRef FileName, Position Pos,
                                 const PreambleData *Preamble,
-                                StringRef Contents, Position Pos,
-                                IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+                                const ParseInputs &ParseInput,
                                 CodeCompleteOptions Opts,
                                 SpeculativeFuzzyFind *SpecFuzzyFind = nullptr);
 
 /// Get signature help at a specified \p Pos in \p FileName.
-SignatureHelp signatureHelp(PathRef FileName,
-                            const tooling::CompileCommand &Command,
-                            const PreambleData &Preamble, StringRef Contents,
-                            Position Pos,
-                            IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
-                            const SymbolIndex *Index);
+SignatureHelp signatureHelp(PathRef FileName, Position Pos,
+                            const PreambleData &Preamble,
+                            const ParseInputs &ParseInput);
 
 // For index-based completion, we only consider:
 //   * symbols in namespaces or translation unit scopes (e.g. no class
 //     members, no locals)
 //   * enum constants in unscoped enum decl (e.g. "red" in "enum {red};")
 //   * primary templates (no specializations)
 // For the other cases, we let Clang do the completion because it does not
 // need any non-local information and it will be much better at following

clang-tools-extra/clangd/CodeComplete.cpp

private:
   SignatureHelp &SigHelp;
   std::shared_ptr<clang::GlobalCodeCompletionAllocator> Allocator;
   CodeCompletionTUInfo CCTUInfo;
   const SymbolIndex *Index;
 }; // SignatureHelpCollector
 
 struct SemaCompleteInput {
   PathRef FileName;
-  const tooling::CompileCommand &Command;
+  size_t Offset;
   const PreambleData &Preamble;
   const llvm::Optional<PreamblePatch> Patch;
-  llvm::StringRef Contents;
-  size_t Offset;
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS;
+  const ParseInputs &ParseInput;
 };
 
 void loadMainFilePreambleMacros(const Preprocessor &PP,
                                 const PreambleData &Preamble) {
   // The ExternalPreprocessorSource has our macros, if we know where to look.
   // We can read all the macros using PreambleMacros->ReadDefinedMacros(),
   // but this includes transitively included files, so may deserialize a lot.
   ExternalPreprocessorSource *PreambleMacros = PP.getExternalSource();

 
 // Invokes Sema code completion on a file.
 // If \p Includes is set, it will be updated based on the compiler invocation.
 bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
                       const clang::CodeCompleteOptions &Options,
                       const SemaCompleteInput &Input,
                       IncludeStructure *Includes = nullptr) {
   trace::Span Tracer("Sema completion");
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = Input.VFS;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = Input.ParseInput.FS;
   if (Input.Preamble.StatCache)
     VFS = Input.Preamble.StatCache->getConsumingFS(std::move(VFS));
-  ParseInputs ParseInput;
-  ParseInput.CompileCommand = Input.Command;
-  ParseInput.FS = VFS;
-  ParseInput.Contents = std::string(Input.Contents);
-  // FIXME: setup the recoveryAST and recoveryASTType in ParseInput properly.
 
   IgnoreDiagnostics IgnoreDiags;
-  auto CI = buildCompilerInvocation(ParseInput, IgnoreDiags);
+  auto CI = buildCompilerInvocation(Input.ParseInput, IgnoreDiags);
   if (!CI) {
     elog("Couldn't create CompilerInvocation");
     return false;
   }
   auto &FrontendOpts = CI->getFrontendOpts();
   FrontendOpts.SkipFunctionBodies = true;
   // Disable typo correction in Sema.
   CI->getLangOpts()->SpellChecking = false;
   // Code completion won't trigger in delayed template bodies.
   // This is on-by-default in windows to allow parsing SDK headers; we're only
   // disabling it for the main-file (not preamble).
   CI->getLangOpts()->DelayedTemplateParsing = false;
   // Setup code completion.
   FrontendOpts.CodeCompleteOpts = Options;
   FrontendOpts.CodeCompletionAt.FileName = std::string(Input.FileName);
   std::tie(FrontendOpts.CodeCompletionAt.Line,
            FrontendOpts.CodeCompletionAt.Column) =
-      offsetToClangLineColumn(Input.Contents, Input.Offset);
+      offsetToClangLineColumn(Input.ParseInput.Contents, Input.Offset);
 
   std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
-      llvm::MemoryBuffer::getMemBufferCopy(Input.Contents, Input.FileName);
+      llvm::MemoryBuffer::getMemBufferCopy(Input.ParseInput.Contents,
+                                           Input.FileName);
   // The diagnostic options must be set before creating a CompilerInstance.
   CI->getDiagnosticOpts().IgnoreWarnings = true;
   // We reuse the preamble whether it's valid or not. This is a
   // correctness/performance tradeoff: building without a preamble is slow, and
   // completion is latency-sensitive.
   // However, if we're completing *inside* the preamble section of the draft,
   // overriding the preamble will break sema completion. Fortunately we can just
   // skip all includes in this case; these completions are really simple.
   PreambleBounds PreambleRegion =
       ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
   bool CompletingInPreamble = PreambleRegion.Size > Input.Offset;
   if (Input.Patch)
     Input.Patch->apply(*CI);
   // NOTE: we must call BeginSourceFile after prepareCompilerInstance. Otherwise
   // the remapped buffers do not get freed.
   auto Clang = prepareCompilerInstance(
       std::move(CI), !CompletingInPreamble ? &Input.Preamble.Preamble : nullptr,
       std::move(ContentsBuffer), std::move(VFS), IgnoreDiags);
   Clang->getPreprocessorOpts().SingleFileParseMode = CompletingInPreamble;
   Clang->setCodeCompletionConsumer(Consumer.release());
 
   SyntaxOnlyAction Action;
   if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) {
     log("BeginSourceFile() failed when running codeComplete for {0}",
         Input.FileName);
     return false;
public:
   CodeCompleteFlow(PathRef FileName, const IncludeStructure &Includes,
                    SpeculativeFuzzyFind *SpecFuzzyFind,
                    const CodeCompleteOptions &Opts)
       : FileName(FileName), Includes(Includes), SpecFuzzyFind(SpecFuzzyFind),
         Opts(Opts) {}
 
   CodeCompleteResult run(const SemaCompleteInput &SemaCCInput) && {
     trace::Span Tracer("CodeCompleteFlow");
-    HeuristicPrefix =
-        guessCompletionPrefix(SemaCCInput.Contents, SemaCCInput.Offset);
-    populateContextWords(SemaCCInput.Contents);
+    HeuristicPrefix = guessCompletionPrefix(SemaCCInput.ParseInput.Contents,
+                                            SemaCCInput.Offset);
+    populateContextWords(SemaCCInput.ParseInput.Contents);
     if (Opts.Index && SpecFuzzyFind && SpecFuzzyFind->CachedReq.hasValue()) {
       assert(!SpecFuzzyFind->Result.valid());
       SpecReq = speculativeFuzzyFindRequestForCompletion(
           *SpecFuzzyFind->CachedReq, HeuristicPrefix);
       SpecFuzzyFind->Result = startAsyncFuzzyFind(*Opts.Index, *SpecReq);
     }
 
     // We run Sema code completion first. It builds an AST and calculates:
     //   - completion results based on the AST.
     //   - partial identifier and context. We need these for the index query.
     CodeCompleteResult Output;
     auto RecorderOwner = std::make_unique<CompletionRecorder>(Opts, [&]() {
       assert(Recorder && "Recorder is not set");
       CCContextKind = Recorder->CCContext.getKind();
       IsUsingDeclaration = Recorder->CCContext.isUsingDeclaration();
-      auto Style = getFormatStyleForFile(
-          SemaCCInput.FileName, SemaCCInput.Contents, SemaCCInput.VFS.get());
+      auto Style = getFormatStyleForFile(SemaCCInput.FileName,
+                                         SemaCCInput.ParseInput.Contents,
+                                         SemaCCInput.ParseInput.FS.get());
       // If preprocessor was run, inclusions from preprocessor callback should
       // already be added to Includes.
       Inserter.emplace(
-          SemaCCInput.FileName, SemaCCInput.Contents, Style,
-          SemaCCInput.Command.Directory,
+          SemaCCInput.FileName, SemaCCInput.ParseInput.Contents, Style,
+          SemaCCInput.ParseInput.CompileCommand.Directory,
           &Recorder->CCSema->getPreprocessor().getHeaderSearchInfo());
       for (const auto &Inc : Includes.MainFileIncludes)
         Inserter->addExisting(Inc);
 
       // Most of the cost of file proximity is in initializing the FileDistance
       // structures based on the observed includes, once per query. Conceptually
       // that happens here (though the per-URI-scheme initialization is lazy).
       // The per-result proximity scoring is (amortized) very cheap.
while (Rest.consume_back("::") && !Rest.endswith(":")) // reject ::::
     while (!Rest.empty() && isIdentifierBody(Rest.back()))
       Rest = Rest.drop_back();
   Result.Qualifier =
       Content.slice(Rest.size(), Result.Name.begin() - Content.begin());
 
   return Result;
 }
 
-CodeCompleteResult
-codeComplete(PathRef FileName, const tooling::CompileCommand &Command,
-             const PreambleData *Preamble, llvm::StringRef Contents,
-             Position Pos, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
-             CodeCompleteOptions Opts, SpeculativeFuzzyFind *SpecFuzzyFind) {
-  auto Offset = positionToOffset(Contents, Pos);
+CodeCompleteResult codeComplete(PathRef FileName, Position Pos,
+                                const PreambleData *Preamble,
+                                const ParseInputs &ParseInput,
+                                CodeCompleteOptions Opts,
+                                SpeculativeFuzzyFind *SpecFuzzyFind) {
+  auto Offset = positionToOffset(ParseInput.Contents, Pos);
   if (!Offset) {
     elog("Code completion position was invalid {0}", Offset.takeError());
     return CodeCompleteResult();
   }
   auto Flow = CodeCompleteFlow(
       FileName, Preamble ? Preamble->Includes : IncludeStructure(),
       SpecFuzzyFind, Opts);
   return (!Preamble || Opts.RunParser == CodeCompleteOptions::NeverParse)
-             ? std::move(Flow).runWithoutSema(Contents, *Offset, VFS)
-             : std::move(Flow).run({FileName, Command, *Preamble,
+             ? std::move(Flow).runWithoutSema(ParseInput.Contents, *Offset,
+                                              ParseInput.FS)
+             : std::move(Flow).run({FileName, *Offset, *Preamble,
                                     // We want to serve code completions with
                                     // low latency, so don't bother patching.
-                                    /*PreamblePatch=*/llvm::None, Contents,
-                                    *Offset, VFS});
+                                    /*PreamblePatch=*/llvm::None, ParseInput});
 }
 
-SignatureHelp signatureHelp(PathRef FileName,
-                            const tooling::CompileCommand &Command,
+SignatureHelp signatureHelp(PathRef FileName, Position Pos,
                             const PreambleData &Preamble,
-                            llvm::StringRef Contents, Position Pos,
-                            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
-                            const SymbolIndex *Index) {
-  auto Offset = positionToOffset(Contents, Pos);
+                            const ParseInputs &ParseInput) {
+  auto Offset = positionToOffset(ParseInput.Contents, Pos);
   if (!Offset) {
     elog("Signature help position was invalid {0}", Offset.takeError());
     return SignatureHelp();
   }
   SignatureHelp Result;
   clang::CodeCompleteOptions Options;
   Options.IncludeGlobals = false;
   Options.IncludeMacros = false;
   Options.IncludeCodePatterns = false;
   Options.IncludeBriefComments = false;
-
-  ParseInputs PI;
-  PI.CompileCommand = Command;
-  PI.Contents = Contents.str();
-  PI.FS = std::move(VFS);
   semaCodeComplete(
-      std::make_unique<SignatureHelpCollector>(Options, Index, Result), Options,
-      {FileName, Command, Preamble,
-       PreamblePatch::create(FileName, PI, Preamble), Contents, *Offset,
-       std::move(PI.FS)});
+      std::make_unique<SignatureHelpCollector>(Options, ParseInput.Index,
+                                               Result),
+      Options,
+      {FileName, *Offset, Preamble,
+       PreamblePatch::create(FileName, ParseInput, Preamble), ParseInput});
   return Result;
 }
 
 bool isIndexedForCodeCompletion(const NamedDecl &ND, ASTContext &ASTCtx) {
   auto InTopLevelScope = [](const NamedDecl &ND) {
     switch (ND.getDeclContext()->getDeclKind()) {
     case Decl::TranslationUnit:
     case Decl::Namespace:

clang-tools-extra/clangd/Compiler.h

struct ParseInputs {
   std::string Contents;
   // Version identifier for Contents, provided by the client and opaque to us.
   std::string Version = "null";
   // Prevent reuse of the cached preamble/AST. Slow! Useful to workaround
   // clangd's assumption that missing header files will stay missing.
   bool ForceRebuild = false;
   // Used to recover from diagnostics (e.g. find missing includes for symbol).
   const SymbolIndex *Index = nullptr;
-  ParseOptions Opts;
+  ParseOptions Opts = ParseOptions();
 };
 
 /// Builds compiler invocation that could be used to build AST or preamble.
 std::unique_ptr<CompilerInvocation>
 buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
                         std::vector<std::string> *CC1Args = nullptr);
 
 /// Creates a compiler instance, configured so that:

clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

CodeCompleteResult completions(const TestTU &TU, Position Point,
   IgnoreDiagnostics Diags;
   auto CI = buildCompilerInvocation(Inputs, Diags);
   if (!CI) {
     ADD_FAILURE() << "Couldn't build CompilerInvocation";
     return {};
   }
   auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
                                 /*InMemory=*/true, /*Callback=*/nullptr);
-  return codeComplete(testPath(TU.Filename), Inputs.CompileCommand,
-                      Preamble.get(), TU.Code, Point, Inputs.FS, Opts);
+  ParseInputs ParseInput{Inputs.CompileCommand, Inputs.FS, TU.Code};
+  ParseInput.Opts.BuildRecoveryAST = true;
+  ParseInput.Opts.PreserveRecoveryASTType = true;
+  return codeComplete(testPath(TU.Filename), Point, Preamble.get(), ParseInput,
+                      Opts);
 }
 
 // Runs code completion.
 CodeCompleteResult completions(llvm::StringRef Text,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {},
                                PathRef FilePath = "foo.cpp") {
   Annotations Test(Text);
CodeCompleteResult completionsNoCompile(llvm::StringRef Text,
   if (!IndexSymbols.empty()) {
     assert(!Opts.Index && "both Index and IndexSymbols given!");
     OverrideIndex = memIndex(std::move(IndexSymbols));
     Opts.Index = OverrideIndex.get();
   }
 
   MockFSProvider FS;
   Annotations Test(Text);
-  return codeComplete(FilePath, tooling::CompileCommand(), /*Preamble=*/nullptr,
-                      Test.code(), Test.point(), FS.getFileSystem(), Opts);
+  ParseInputs ParseInput{tooling::CompileCommand(), FS.getFileSystem(),
+                         Test.code().str()};
+  return codeComplete(FilePath, Test.point(), /*Preamble=*/nullptr, ParseInput,
+                      Opts);
 }
 
 Symbol withReferences(int N, Symbol S) {
   S.References = N;
   return S;
 }
 
 TEST(CompletionTest, Limit) {
auto Results = completions(R"cpp(
     #include <bar.h>
     int foo() {}
     #endif
     )cpp")
                      .Completions;
   EXPECT_THAT(Results, ElementsAre(Named("ifndef")));
 }
 
-// FIXME: enable it.
-TEST(CompletionTest, DISABLED_CompletionRecoveryASTType) {
+TEST(CompletionTest, CompletionRecoveryASTType) {
   auto Results = completions(R"cpp(
     struct S { int member; };
     S overloaded(int);
     void foo() {
       // No overload matches, but we have recovery-expr with the correct type.
       overloaded().^
     })cpp")
                      .Completions;
if (!CI) {
     return {};
   }
   auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
                                 /*InMemory=*/true, /*Callback=*/nullptr);
   if (!Preamble) {
     ADD_FAILURE() << "Couldn't build Preamble";
     return {};
   }
-  return signatureHelp(testPath(TU.Filename), Inputs.CompileCommand, *Preamble,
-                       Text, Point, Inputs.FS, Index.get());
+  ParseInputs ParseInput{Inputs.CompileCommand, Inputs.FS, Text.str()};
+  ParseInput.Index = Index.get();
+  ParseInput.Opts.BuildRecoveryAST = true;
+  ParseInput.Opts.PreserveRecoveryASTType = true;
+  return signatureHelp(testPath(TU.Filename), Point, *Preamble, ParseInput);
 }
 
 SignatureHelp signatures(llvm::StringRef Text,
                          std::vector<Symbol> IndexSymbols = {}) {
   Annotations Test(Text);
   return signatures(Test.code(), Test.point(), std::move(IndexSymbols));
 }
 
TEST(SignatureHelpTest, StalePreamble) {
   ASSERT_TRUE(EmptyPreamble);
 
   TU.AdditionalFiles["a.h"] = "int foo(int x);";
   const Annotations Test(R"cpp(
     #include "a.h"
     void bar() { foo(^2); })cpp");
   TU.Code = Test.code().str();
   Inputs = TU.inputs();
-  auto Results =
-      signatureHelp(testPath(TU.Filename), Inputs.CompileCommand,
-                    *EmptyPreamble, TU.Code, Test.point(), Inputs.FS, nullptr);
+
+  ParseInputs ParseInput{Inputs.CompileCommand, Inputs.FS, TU.Code};
+  auto Results = signatureHelp(testPath(TU.Filename), Test.point(),
+                               *EmptyPreamble, ParseInput);
   EXPECT_THAT(Results.signatures, ElementsAre(Sig("foo([[int x]]) -> int")));
   EXPECT_EQ(0, Results.activeSignature);
   EXPECT_EQ(0, Results.activeParameter);
 }
 
 class IndexRequestCollector : public SymbolIndex {
 public:
   bool