[clangd] Use AST-based signals in CodeCompletion.

[usaxena95]

Edge cases fix.

[usaxena95]

Rebase.

[adamcz]

clang-tools-extra/clangd/Quality.cpp

363

Maybe it's just me, but using a lambda like this seems quite confusing. I don't think it improves readability. Maybe just make it a separate method? Or inline here, but without lambda?

[usaxena95]

Addressed comments.

clang-tools-extra/clangd/ClangdServer.cpp

if (!IP->Preamble) {
         }
       }
     }
     ParseInputs ParseInput{IP->Command, &TFS, IP->Contents.str()};
     ParseInput.Index = Index;
     ParseInput.Opts.BuildRecoveryAST = BuildRecoveryAST;
     ParseInput.Opts.PreserveRecoveryASTType = PreserveRecoveryASTType;
 
+    CodeCompleteOpts.MainFileSignals = IP->Signals;
     // FIXME(ibiryukov): even if Preamble is non-null, we may want to check
     // both the old and the new version in case only one of them matches.
     CodeCompleteResult Result = clangd::codeComplete(
         File, Pos, IP->Preamble, ParseInput, CodeCompleteOpts,
         SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr);
     {
       clang::clangd::trace::Span Tracer("Completion results callback");
       CB(std::move(Result));

clang-tools-extra/clangd/CodeComplete.h

 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 
 #include "Compiler.h"
 #include "Headers.h"
 #include "Protocol.h"
 #include "Quality.h"
+#include "TUScheduler.h"
 #include "index/Index.h"
 #include "index/Symbol.h"
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
 #include "support/Markup.h"
 #include "support/Path.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/CodeCompleteOptions.h"
struct CodeCompleteOptions {
 
   // Populated internally by clangd, do not set.
   /// If `Index` is set, it is used to augment the code completion
   /// results.
   /// FIXME(ioeric): we might want a better way to pass the index around inside
   /// clangd.
   const SymbolIndex *Index = nullptr;
 
+  const ASTSignals *MainFileSignals = nullptr;
   /// Include completions that require small corrections, e.g. change '.' to
   /// '->' on member access etc.
   bool IncludeFixIts = false;
 
   /// Whether to generate snippets for function arguments on code-completion.
   /// Needs snippets to be enabled as well.
   bool EnableFunctionArgSnippets = true;
 

clang-tools-extra/clangd/CodeComplete.cpp

auto RecorderOwner = std::make_unique<CompletionRecorder>(Opts, [&]() {
           llvm::join(QueryScopes.begin(), QueryScopes.end(), ","), AllScopes,
           PreferredType ? Recorder->CCContext.getPreferredType().getAsString()
                         : "<none>",
           IsUsingDeclaration ? ", inside using declaration" : "");
     });
 
     Recorder = RecorderOwner.get();
 
     semaCodeComplete(std::move(RecorderOwner), Opts.getClangCompleteOpts(),
                      SemaCCInput, &Includes);
     logResults(Output, Tracer);
     return Output;
   }
 
   void logResults(const CodeCompleteResult &Output, const trace::Span &Tracer) {
     SPAN_ATTACH(Tracer, "sema_results", NSema);
     SPAN_ATTACH(Tracer, "index_results", NIndex);
void addCandidate(TopN<ScoredBundle, ScoredBundleGreater> &Candidates,
     Relevance.FilterLength = HeuristicPrefix.Name.size();
     Relevance.Query = SymbolRelevanceSignals::CodeComplete;
     Relevance.FileProximityMatch = FileProximity.getPointer();
     if (ScopeProximity)
       Relevance.ScopeProximityMatch = ScopeProximity.getPointer();
     if (PreferredType)
       Relevance.HadContextType = true;
     Relevance.ContextWords = &ContextWords;
+    Relevance.MainFileSignals = Opts.MainFileSignals;
 
     auto &First = Bundle.front();
     if (auto FuzzyScore = fuzzyScore(First))
       Relevance.NameMatch = *FuzzyScore;
     else
       return;
     SymbolOrigin Origin = SymbolOrigin::Unknown;
     bool FromIndex = false;

clang-tools-extra/clangd/Quality.h

 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_QUALITY_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_QUALITY_H
 
 #include "ExpectedTypes.h"
 #include "FileDistance.h"
+#include "TUScheduler.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include <algorithm>
 #include <functional>
 #include <vector>
 
struct SymbolRelevanceSignals {
   bool HadSymbolType = false;
   // Whether the item matches the type expected in the completion context.
   bool TypeMatchesPreferred = false;
 
   /// Length of the unqualified partial name of Symbol typed in
   /// CompletionPrefix.
   unsigned FilterLength = 0;
 
+  const ASTSignals *MainFileSignals = nullptr;
+  unsigned MainFileRefs = 0;
+  unsigned ScopeRefsInFile = 0;
+
   /// Set of derived signals computed by calculateDerivedSignals(). Must not be
   /// set explicitly.
   struct DerivedSignals {
     /// Whether Name contains some word from context.
     bool NameMatchesContext = false;
     /// Min distance between SymbolURI and all the headers included by the TU.
     unsigned FileProximityDistance = FileDistance::Unreachable;
     /// Min distance between SymbolScope and all the available scopes.

clang-tools-extra/clangd/Quality.cpp

 
 void SymbolRelevanceSignals::merge(const Symbol &IndexResult) {
   SymbolURI = IndexResult.CanonicalDeclaration.FileURI;
   SymbolScope = IndexResult.Scope;
   IsInstanceMember |= isInstanceMember(IndexResult.SymInfo);
   if (!(IndexResult.Flags & Symbol::VisibleOutsideFile)) {
     Scope = AccessibleScope::FileScope;
   }
+  if (MainFileSignals) {
+    MainFileRefs =
+        std::max(MainFileRefs, MainFileSignals->Symbols.lookup(IndexResult.ID));
+    ScopeRefsInFile =
+        std::max(ScopeRefsInFile,
+                 MainFileSignals->RelatedNamespaces.lookup(IndexResult.Scope));
+  }
 }
 
 void SymbolRelevanceSignals::merge(const CodeCompletionResult &SemaCCResult) {
   if (SemaCCResult.Availability == CXAvailability_NotAvailable ||
       SemaCCResult.Availability == CXAvailability_NotAccessible)
     Forbidden = true;
 
   if (SemaCCResult.Declaration) {
     SemaSaysInScope = true;
     // We boost things that have decls in the main file. We give a fixed score
     // for all other declarations in sema as they are already included in the
     // translation unit.
     float DeclProximity = (hasDeclInMainFile(*SemaCCResult.Declaration) ||
                            hasUsingDeclInMainFile(SemaCCResult))
                               ? 1.0
                               : 0.6;
     SemaFileProximityScore = std::max(DeclProximity, SemaFileProximityScore);
     IsInstanceMember |= isInstanceMember(SemaCCResult.Declaration);
     InBaseClass |= SemaCCResult.InBaseClass;
   }
+  if (MainFileSignals) {
+    if ((SemaCCResult.Kind == CodeCompletionResult::RK_Declaration) ||
+        (SemaCCResult.Kind == CodeCompletionResult::RK_Pattern)) {
+      if (const NamedDecl *ND = SemaCCResult.getDeclaration()) {
+        MainFileRefs = std::max(
+            MainFileRefs, MainFileSignals->Symbols.lookup(getSymbolID(ND)));
+        std::string Qual;
+        llvm::raw_string_ostream OS(Qual);
+        ND->printNestedNameSpecifier(OS);
+        if (!Qual.empty())
+          ScopeRefsInFile = std::max(
+              ScopeRefsInFile, MainFileSignals->RelatedNamespaces.lookup(Qual));
+      }
+    }
+  }
 
   // Declarations are scoped, others (like macros) are assumed global.
   if (SemaCCResult.Declaration)
     Scope = std::min(Scope, computeScope(SemaCCResult.Declaration));
 
   NeedsFixIts = !SemaCCResult.FixIts.empty();
 }
 
 static float fileProximityScore(unsigned FileDistance) {
   // Range: [0, 1]
   // FileDistance = [0, 1, 2, 3, 4, .., FileDistance::Unreachable]
   // Score = [1, 0.82, 0.67, 0.55, 0.45, .., 0]
   if (FileDistance == FileDistance::Unreachable)
     return 0;
   // Assume approximately default options are used for sensible scoring.
   return std::exp(FileDistance * -0.4f / FileDistanceOptions().UpCost);
 }
 
 static float scopeProximityScore(unsigned ScopeDistance) {
   // Range: [0.6, 2].
   // ScopeDistance = [0, 1, 2, 3, 4, 5, 6, 7, .., FileDistance::Unreachable]
   // Score = [2.0, 1.55, 1.2, 0.93, 0.72, 0.65, 0.65, 0.65, .., 0.6]
   if (ScopeDistance == FileDistance::Unreachable)
     return 0.6f;
   return std::max(0.65, 2.0 * std::pow(0.6, ScopeDistance / 2.0));
 }
 
 static llvm::Optional<llvm::StringRef>
 wordMatching(llvm::StringRef Name, const llvm::StringSet<> *ContextWords) {
   if (ContextWords)
     for (const auto &Word : ContextWords->keys())
       if (Name.contains_lower(Word))
         return Word;

clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

 #include "ClangdServer.h"
 #include "CodeComplete.h"
 #include "Compiler.h"
 #include "Matchers.h"
 #include "Protocol.h"
 #include "Quality.h"
 #include "SourceCode.h"
 #include "SyncAPI.h"
+#include "TUScheduler.h"
 #include "TestFS.h"
 #include "TestIndex.h"
 #include "TestTU.h"
 #include "index/Index.h"
 #include "index/MemIndex.h"
 #include "support/Threading.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Tooling/CompilationDatabase.h"

 using ::testing::HasSubstr;
 using ::testing::IsEmpty;
 using ::testing::Not;
 using ::testing::UnorderedElementsAre;
 using ContextKind = CodeCompletionContext::Kind;
 
 // GMock helpers for matching completion items.
 MATCHER_P(Named, Name, "") { return arg.Name == Name; }
+MATCHER_P(MainFileRefs, Refs, "") { return arg.MainFileRefs == Refs; }
+MATCHER_P(ScopeRefs, Refs, "") { return arg.ScopeRefsInFile == Refs; }
 MATCHER_P(NameStartsWith, Prefix, "") {
   return llvm::StringRef(arg.Name).startswith(Prefix);
 }
 MATCHER_P(Scope, S, "") { return arg.Scope == S; }
 MATCHER_P(Qualifier, Q, "") { return arg.RequiredQualifier == Q; }
 MATCHER_P(Labeled, Label, "") {
   return arg.RequiredQualifier + arg.Name + arg.Signature == Label;
 }
for (const auto &Sym : Symbols)
     Slab.insert(Sym);
   return MemIndex::build(std::move(Slab).build(), RefSlab(), RelationSlab());
 }
 
 // Runs code completion.
 // If IndexSymbols is non-empty, an index will be built and passed to opts.
 CodeCompleteResult completions(const TestTU &TU, Position Point,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {}) {
   std::unique_ptr<SymbolIndex> OverrideIndex;
   if (!IndexSymbols.empty()) {
     assert(!Opts.Index && "both Index and IndexSymbols given!");
     OverrideIndex = memIndex(std::move(IndexSymbols));
     Opts.Index = OverrideIndex.get();
   }
 
   MockFS FS;
   auto Inputs = TU.inputs(FS);
   IgnoreDiagnostics Diags;
   auto CI = buildCompilerInvocation(Inputs, Diags);
   if (!CI) {
     ADD_FAILURE() << "Couldn't build CompilerInvocation";
     return {};
   }
   auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
                                 /*InMemory=*/true, /*Callback=*/nullptr);
   return codeComplete(testPath(TU.Filename), Point, Preamble.get(), Inputs,
                       Opts);
 }
 
 // Runs code completion.
 CodeCompleteResult completions(llvm::StringRef Text,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {},
                                PathRef FilePath = "foo.cpp") {
   Annotations Test(Text);
   auto TU = TestTU::withCode(Test.code());
   // To make sure our tests for completiopns inside templates work on Windows.
   TU.Filename = FilePath.str();
   return completions(TU, Test.point(), std::move(IndexSymbols),
                      std::move(Opts));
 }
 
 // Runs code completion without the clang parser.
 CodeCompleteResult completionsNoCompile(llvm::StringRef Text,
                                         std::vector<Symbol> IndexSymbols = {},
                                         clangd::CodeCompleteOptions Opts = {},
                                         PathRef FilePath = "foo.cpp") {
   std::unique_ptr<SymbolIndex> OverrideIndex;
   if (!IndexSymbols.empty()) {
     assert(!Opts.Index && "both Index and IndexSymbols given!");
     OverrideIndex = memIndex(std::move(IndexSymbols));
     Opts.Index = OverrideIndex.get();
   }
 
std::string Body = R"cpp(
     struct S {
       int FooBar;
       int FooBaz;
       int Qux;
     };
   )cpp";
 
   // Only items matching the fuzzy query are returned.
   EXPECT_THAT(completions(Body + "int main() { S().Foba^ }").Completions,
               AllOf(Has("FooBar"), Has("FooBaz"), Not(Has("Qux"))));
 
   // Macros require prefix match, either from index or AST.
   Symbol Sym = var("MotorCarIndex");
   Sym.SymInfo.Kind = index::SymbolKind::Macro;
   EXPECT_THAT(
       completions(Body + "int main() { C^ }", {Sym}).Completions,
       AllOf(Has("Car"), Not(Has("MotorCar")), Not(Has("MotorCarIndex"))));
   EXPECT_THAT(completions(Body + "int main() { M^ }", {Sym}).Completions,
               AllOf(Has("MotorCar"), Has("MotorCarIndex")));
 }
 
 void testAfterDotCompletion(clangd::CodeCompleteOptions Opts) {
   auto Results = completions(
       R"cpp(
       int global_var;
 
TEST(CompletionTest, CompletionOptions) {
   // We used to test every combination of options, but that got too slow (2^N).
   auto Flags = {
       &clangd::CodeCompleteOptions::IncludeMacros,
       &clangd::CodeCompleteOptions::IncludeComments,
       &clangd::CodeCompleteOptions::IncludeCodePatterns,
       &clangd::CodeCompleteOptions::IncludeIneligibleResults,
   };
   // Test default options.
   Test({});
   // Test with one flag flipped.
   for (auto &F : Flags) {
     clangd::CodeCompleteOptions O;
     O.*F ^= true;
     Test(O);
   }
 }
 
 TEST(CompletionTest, Accessible) {
   auto Internal = completions(R"cpp(
       class Foo {
         public: void pub();
         protected: void prot();
         private: void priv();
       };
       void Foo::pub() { this->^ }
   )cpp");
   EXPECT_THAT(Internal.Completions,
               AllOf(Has("priv"), Has("prot"), Has("pub")));
 
   auto External = completions(R"cpp(
       class Foo {
         public: void pub();
         protected: void prot();
         private: void priv();
       };
       void test() {
         Foo F;
         F.^
       }
   )cpp");
   EXPECT_THAT(External.Completions,
               AllOf(Has("pub"), Not(Has("prot")), Not(Has("priv"))));
 }
 
 TEST(CompletionTest, Qualifiers) {
   auto Results = completions(R"cpp(
       class Foo {
         public: int foo() const;
         int bar() const;
       };
       class Bar : public Foo {
         int foo() const;
       };
       void test() { Bar().^ }
   )cpp");
   EXPECT_THAT(Results.Completions,
               Contains(AllOf(Qualifier(""), Named("bar"))));
   // Hidden members are not shown.
   EXPECT_THAT(Results.Completions,
               Not(Contains(AllOf(Qualifier("Foo::"), Named("foo")))));
   // Private members are not shown.
   EXPECT_THAT(Results.Completions,
               Not(Contains(AllOf(Qualifier(""), Named("foo")))));
 }
 
 TEST(CompletionTest, InjectedTypename) {
   // These are suppressed when accessed as a member...
   EXPECT_THAT(completions("struct X{}; void foo(){ X().^ }").Completions,
               Not(Has("X")));
   EXPECT_THAT(completions("struct X{ void foo(){ this->^ } };").Completions,
               Not(Has("X")));
   // ...but accessible in other, more useful cases.
   EXPECT_THAT(completions("struct X{ void foo(){ ^ } };").Completions,
               Has("X"));
   EXPECT_THAT(
       completions("struct Y{}; struct X:Y{ void foo(){ ^ } };").Completions,
       Has("Y"));
   EXPECT_THAT(
       completions(
           "template<class> struct Y{}; struct X:Y<int>{ void foo(){ ^ } };")
           .Completions,
       Has("Y"));
   // This case is marginal (`using X::X` is useful), we allow it for now.
   EXPECT_THAT(completions("struct X{}; void foo(){ X::^ }").Completions,
               Has("X"));
 }
 
 TEST(CompletionTest, SkipInjectedWhenUnqualified) {
   EXPECT_THAT(completions("struct X { void f() { X^ }};").Completions,
               ElementsAre(Named("X"), Named("~X")));
 }
 
 TEST(CompletionTest, Snippets) {
   clangd::CodeCompleteOptions Opts;
   auto Results = completions(
       R"cpp(
       struct fake {
Opts.RecordCCResult = [&RecordedCompletions](const CodeCompletion &CC,
     RecordedCompletions.push_back(CC);
   };
 
   completions("int xy1, xy2; int a = xy^", /*IndexSymbols=*/{}, Opts);
   EXPECT_THAT(RecordedCompletions,
               UnorderedElementsAre(Named("xy1"), Named("xy2")));
 }
 
+TEST(CompletionTest, ASTSignals) {
+  struct Completion {
+    std::string Name;
+    unsigned MainFileRefs;
+    unsigned ScopeRefsInFile;
+  };
+  CodeCompleteOptions Opts;
+  std::vector<Completion> RecordedCompletions;
+  Opts.RecordCCResult = [&RecordedCompletions](const CodeCompletion &CC,
+                                               const SymbolQualitySignals &,
+                                               const SymbolRelevanceSignals &R,
+                                               float Score) {
+    RecordedCompletions.push_back({CC.Name, R.MainFileRefs, R.ScopeRefsInFile});
+  };
+  ASTSignals MainFileSignals;
+  MainFileSignals.Symbols[var("xy1").ID] = 3;
+  MainFileSignals.Symbols[var("xy2").ID] = 1;
+  MainFileSignals.Symbols[var("xyindex").ID] = 10;
+  MainFileSignals.RelatedNamespaces["tar::"] = 5;
+  MainFileSignals.RelatedNamespaces["clang::"] = 3;
+  Opts.MainFileSignals = &MainFileSignals;
+  Opts.AllScopes = true;
+  completions(R"cpp(
+              int xy1;
+              int xy2;
+              namespace clang {
+              const int xybar = 1;
+              int a = xy^
+              }
+              )cpp",
+              /*IndexSymbols=*/{var("xyindex"), var("tar::xyfoo")}, Opts);
+  EXPECT_THAT(RecordedCompletions,
+              UnorderedElementsAre(
+                  AllOf(Named("xy1"), MainFileRefs(3u), ScopeRefs(0u)),
+                  AllOf(Named("xy2"), MainFileRefs(1u), ScopeRefs(0u)),
+                  AllOf(Named("xyindex"), MainFileRefs(10u), ScopeRefs(0u)),
+                  AllOf(Named("xyfoo"), MainFileRefs(0u), ScopeRefs(5u)),
+                  AllOf(Named("xybar"), MainFileRefs(0u), ScopeRefs(3u))));
+  //   Annotations Test(R"cpp(
+  //     #include "foo.h"
+  //     Fun^
+  //   )cpp");
+  //   auto TU = TestTU::withCode(Test.code());
+  //   TU.AdditionalFiles["foo.h"] = "";
+
+  //   std::string DeclFile = URI::create(testPath("foo")).toString();
+  //   Symbol Sym = func("Func");
+  //   Sym.CanonicalDeclaration.FileURI = DeclFile.c_str();
+  //   Sym.IncludeHeaders.emplace_back("\"foo.h\"", 2);
+  //   Sym.IncludeHeaders.emplace_back("\"bar.h\"", 1000);
+
+  //   EXPECT_THAT(completions(TU, Test.point(), {Sym}).Completions,
+  //               UnorderedElementsAre(AllOf(Named("Func"),
+  //               HasInclude("\"foo.h\""),
+  //                                          Not(InsertInclude()))));
+}
+
 SignatureHelp signatures(llvm::StringRef Text, Position Point,
                          std::vector<Symbol> IndexSymbols = {}) {
   std::unique_ptr<SymbolIndex> Index;
   if (!IndexSymbols.empty())
     Index = memIndex(IndexSymbols);
 
   auto TU = TestTU::withCode(Text);
   MockFS FS;