[clangd] Use AST-based signals in CodeCompletion.

[usaxena95]

Edge cases fix.

[usaxena95]

Rebase.

[adamcz]

clang-tools-extra/clangd/Quality.cpp

373

Maybe it's just me, but using a lambda like this seems quite confusing. I don't think it improves readability. Maybe just make it a separate method? Or inline here, but without lambda?

[usaxena95]

Addressed comments.

clang-tools-extra/clangd/ClangdServer.cpp

if (!IP->Preamble) {
         SpecFuzzyFind->CachedReq = CachedCompletionFuzzyFindRequestByFile[File];
       }
     }
     ParseInputs ParseInput{IP->Command, &TFS, IP->Contents.str()};
     ParseInput.Index = Index;
     ParseInput.Opts.BuildRecoveryAST = BuildRecoveryAST;
     ParseInput.Opts.PreserveRecoveryASTType = PreserveRecoveryASTType;
 
+    CodeCompleteOpts.MainFileSignals = IP->Signals;
     // FIXME(ibiryukov): even if Preamble is non-null, we may want to check
     // both the old and the new version in case only one of them matches.
     CodeCompleteResult Result = clangd::codeComplete(
         File, Pos, IP->Preamble, ParseInput, CodeCompleteOpts,
         SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr);
     {
       clang::clangd::trace::Span Tracer("Completion results callback");
       CB(std::move(Result));

clang-tools-extra/clangd/CodeComplete.h

 // After "std::string S; S." we might suggest members of std::string.
 // Signature help describes the parameters of a function as you type them.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 
+#include "ASTSignals.h"
 #include "Compiler.h"
 #include "Headers.h"
 #include "Protocol.h"
 #include "Quality.h"
 #include "index/Index.h"
 #include "index/Symbol.h"
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
struct CodeCompleteOptions {
 
   // Populated internally by clangd, do not set.
   /// If `Index` is set, it is used to augment the code completion
   /// results.
   /// FIXME(ioeric): we might want a better way to pass the index around inside
   /// clangd.
   const SymbolIndex *Index = nullptr;
 
+  const ASTSignals *MainFileSignals = nullptr;
   /// Include completions that require small corrections, e.g. change '.' to
   /// '->' on member access etc.
   bool IncludeFixIts = false;
 
   /// Whether to generate snippets for function arguments on code-completion.
   /// Needs snippets to be enabled as well.
   bool EnableFunctionArgSnippets = true;
 

clang-tools-extra/clangd/CodeComplete.cpp

void addCandidate(TopN<ScoredBundle, ScoredBundleGreater> &Candidates,
     Relevance.FilterLength = HeuristicPrefix.Name.size();
     Relevance.Query = SymbolRelevanceSignals::CodeComplete;
     Relevance.FileProximityMatch = FileProximity.getPointer();
     if (ScopeProximity)
       Relevance.ScopeProximityMatch = ScopeProximity.getPointer();
     if (PreferredType)
       Relevance.HadContextType = true;
     Relevance.ContextWords = &ContextWords;
+    Relevance.MainFileSignals = Opts.MainFileSignals;
 
     auto &First = Bundle.front();
     if (auto FuzzyScore = fuzzyScore(First))
       Relevance.NameMatch = *FuzzyScore;
     else
       return;
     SymbolOrigin Origin = SymbolOrigin::Unknown;
     bool FromIndex = false;

clang-tools-extra/clangd/Quality.h

 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_QUALITY_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_QUALITY_H
 
 #include "ExpectedTypes.h"
 #include "FileDistance.h"
+#include "TUScheduler.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include <algorithm>
 #include <functional>
 #include <vector>
 
struct SymbolRelevanceSignals {
   bool HadSymbolType = false;
   // Whether the item matches the type expected in the completion context.
   bool TypeMatchesPreferred = false;
 
   /// Length of the unqualified partial name of Symbol typed in
   /// CompletionPrefix.
   unsigned FilterLength = 0;
 
+  const ASTSignals *MainFileSignals = nullptr;
+  /// Number of references to the candidate in the main file.
+  unsigned MainFileRefs = 0;
+  /// Number of unique symbols in the main file which belongs to candidate's
+  /// namespace. This indicates how relevant the namespace is in the current
+  /// file.
+  unsigned ScopeRefsInFile = 0;
+
   /// Set of derived signals computed by calculateDerivedSignals(). Must not be
   /// set explicitly.
   struct DerivedSignals {
     /// Whether Name contains some word from context.
     bool NameMatchesContext = false;
     /// Min distance between SymbolURI and all the headers included by the TU.
     unsigned FileProximityDistance = FileDistance::Unreachable;
     /// Min distance between SymbolScope and all the available scopes.
     unsigned ScopeProximityDistance = FileDistance::Unreachable;
   };
 
   DerivedSignals calculateDerivedSignals() const;
 
   void merge(const CodeCompletionResult &SemaResult);
   void merge(const Symbol &IndexResult);
+  void computeASTSignals(const CodeCompletionResult &SemaResult);
 
   // Condense these signals down to a single number, higher is better.
   float evaluateHeuristics() const;
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                               const SymbolRelevanceSignals &);
 
 /// Combine symbol quality and relevance into a single score.

clang-tools-extra/clangd/Quality.cpp

 
 void SymbolRelevanceSignals::merge(const Symbol &IndexResult) {
   SymbolURI = IndexResult.CanonicalDeclaration.FileURI;
   SymbolScope = IndexResult.Scope;
   IsInstanceMember |= isInstanceMember(IndexResult.SymInfo);
   if (!(IndexResult.Flags & Symbol::VisibleOutsideFile)) {
     Scope = AccessibleScope::FileScope;
   }
+  if (MainFileSignals) {
+    MainFileRefs =
+        std::max(MainFileRefs,
+                 MainFileSignals->ReferencedSymbols.lookup(IndexResult.ID));
+    ScopeRefsInFile =
+        std::max(ScopeRefsInFile,
+                 MainFileSignals->RelatedNamespaces.lookup(IndexResult.Scope));
+  }
+}
+
+void SymbolRelevanceSignals::computeASTSignals(
+    const CodeCompletionResult &SemaResult) {
+  if (!MainFileSignals)
+    return;
+  if ((SemaResult.Kind != CodeCompletionResult::RK_Declaration) &&
+      (SemaResult.Kind != CodeCompletionResult::RK_Pattern))
+    return;
+  if (const NamedDecl *ND = SemaResult.getDeclaration()) {
+    auto ID = getSymbolID(ND);
+    if (!ID)
+      return;
+    MainFileRefs =
+        std::max(MainFileRefs, MainFileSignals->ReferencedSymbols.lookup(ID));
+    if (const auto *NSD = dyn_cast<NamespaceDecl>(ND->getDeclContext())) {
+      if (NSD->isAnonymousNamespace())
+        return;
+      std::string Scope = printNamespaceScope(*NSD);
+      if (!Scope.empty())
+        ScopeRefsInFile = std::max(
+            ScopeRefsInFile, MainFileSignals->RelatedNamespaces.lookup(Scope));
+    }
+  }
 }
 
 void SymbolRelevanceSignals::merge(const CodeCompletionResult &SemaCCResult) {
   if (SemaCCResult.Availability == CXAvailability_NotAvailable ||
       SemaCCResult.Availability == CXAvailability_NotAccessible)
     Forbidden = true;
 
   if (SemaCCResult.Declaration) {
     SemaSaysInScope = true;
     // We boost things that have decls in the main file. We give a fixed score
     // for all other declarations in sema as they are already included in the
     // translation unit.
     float DeclProximity = (hasDeclInMainFile(*SemaCCResult.Declaration) ||
                            hasUsingDeclInMainFile(SemaCCResult))
                               ? 1.0
                               : 0.6;
     SemaFileProximityScore = std::max(DeclProximity, SemaFileProximityScore);
     IsInstanceMember |= isInstanceMember(SemaCCResult.Declaration);
     InBaseClass |= SemaCCResult.InBaseClass;
   }
 
+  computeASTSignals(SemaCCResult);
   // Declarations are scoped, others (like macros) are assumed global.
   if (SemaCCResult.Declaration)
     Scope = std::min(Scope, computeScope(SemaCCResult.Declaration));
 
   NeedsFixIts = !SemaCCResult.FixIts.empty();
 }
 
 static float fileProximityScore(unsigned FileDistance) {
   // Range: [0, 1]
   // FileDistance = [0, 1, 2, 3, 4, .., FileDistance::Unreachable]
   // Score = [1, 0.82, 0.67, 0.55, 0.45, .., 0]
   if (FileDistance == FileDistance::Unreachable)
     return 0;
   // Assume approximately default options are used for sensible scoring.
   return std::exp(FileDistance * -0.4f / FileDistanceOptions().UpCost);
 }
 
 static float scopeProximityScore(unsigned ScopeDistance) {
   // Range: [0.6, 2].
   // ScopeDistance = [0, 1, 2, 3, 4, 5, 6, 7, .., FileDistance::Unreachable]
   // Score = [2.0, 1.55, 1.2, 0.93, 0.72, 0.65, 0.65, 0.65, .., 0.6]
   if (ScopeDistance == FileDistance::Unreachable)
     return 0.6f;
   return std::max(0.65, 2.0 * std::pow(0.6, ScopeDistance / 2.0));
 }
 
 static llvm::Optional<llvm::StringRef>
 wordMatching(llvm::StringRef Name, const llvm::StringSet<> *ContextWords) {
   if (ContextWords)
     for (const auto &Word : ContextWords->keys())
       if (Name.contains_lower(Word))

clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

 //===-- CodeCompleteTests.cpp -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+#include "ASTSignals.h"
 #include "Annotations.h"
 #include "ClangdServer.h"
 #include "CodeComplete.h"
 #include "Compiler.h"
 #include "Matchers.h"
 #include "Protocol.h"
 #include "Quality.h"
 #include "SourceCode.h"

 using ::testing::HasSubstr;
 using ::testing::IsEmpty;
 using ::testing::Not;
 using ::testing::UnorderedElementsAre;
 using ContextKind = CodeCompletionContext::Kind;
 
 // GMock helpers for matching completion items.
 MATCHER_P(Named, Name, "") { return arg.Name == Name; }
+MATCHER_P(MainFileRefs, Refs, "") { return arg.MainFileRefs == Refs; }
+MATCHER_P(ScopeRefs, Refs, "") { return arg.ScopeRefsInFile == Refs; }
 MATCHER_P(NameStartsWith, Prefix, "") {
   return llvm::StringRef(arg.Name).startswith(Prefix);
 }
 MATCHER_P(Scope, S, "") { return arg.Scope == S; }
 MATCHER_P(Qualifier, Q, "") { return arg.RequiredQualifier == Q; }
 MATCHER_P(Labeled, Label, "") {
   return arg.RequiredQualifier + arg.Name + arg.Signature == Label;
 }
Opts.RecordCCResult = [&RecordedCompletions](const CodeCompletion &CC,
     RecordedCompletions.push_back(CC);
   };
 
   completions("int xy1, xy2; int a = xy^", /*IndexSymbols=*/{}, Opts);
   EXPECT_THAT(RecordedCompletions,
               UnorderedElementsAre(Named("xy1"), Named("xy2")));
 }
 
+TEST(CompletionTest, ASTSignals) {
+  struct Completion {
+    std::string Name;
+    unsigned MainFileRefs;
+    unsigned ScopeRefsInFile;
+  };
+  CodeCompleteOptions Opts;
+  std::vector<Completion> RecordedCompletions;
+  Opts.RecordCCResult = [&RecordedCompletions](const CodeCompletion &CC,
+                                               const SymbolQualitySignals &,
+                                               const SymbolRelevanceSignals &R,
+                                               float Score) {
+    RecordedCompletions.push_back({CC.Name, R.MainFileRefs, R.ScopeRefsInFile});
+  };
+  ASTSignals MainFileSignals;
+  MainFileSignals.ReferencedSymbols[var("xy1").ID] = 3;
+  MainFileSignals.ReferencedSymbols[var("xy2").ID] = 1;
+  MainFileSignals.ReferencedSymbols[var("xyindex").ID] = 10;
+  MainFileSignals.RelatedNamespaces["tar::"] = 5;
+  MainFileSignals.RelatedNamespaces["bar::"] = 3;
+  Opts.MainFileSignals = &MainFileSignals;
+  Opts.AllScopes = true;
+  completions(
+      R"cpp(
+      int xy1;
+      int xy2;
+      namespace bar {
+      int xybar = 1;
+      int a = xy^
+      }
+      )cpp",
+      /*IndexSymbols=*/{var("xyindex"), var("tar::xytar"), var("bar::xybar")},
+      Opts);
+  EXPECT_THAT(RecordedCompletions,
+              UnorderedElementsAre(
+                  AllOf(Named("xy1"), MainFileRefs(3u), ScopeRefs(0u)),
+                  AllOf(Named("xy2"), MainFileRefs(1u), ScopeRefs(0u)),
+                  AllOf(Named("xyindex"), MainFileRefs(10u), ScopeRefs(0u)),
+                  AllOf(Named("xytar"), MainFileRefs(0u), ScopeRefs(5u)),
+                  AllOf(/*both from sema and index*/ Named("xybar"),
+                        MainFileRefs(0u), ScopeRefs(3u))));
+}
+
 SignatureHelp signatures(llvm::StringRef Text, Position Point,
                          std::vector<Symbol> IndexSymbols = {}) {
   std::unique_ptr<SymbolIndex> Index;
   if (!IndexSymbols.empty())
     Index = memIndex(IndexSymbols);
 
   auto TU = TestTU::withCode(Text);
   MockFS FS;