[clangd] Prototype for postfix completion.

[hokein]

Summary

This is a quick proof-of-concept demo for postfix completion.

demo (ignore the wrong highlightings in VSCode):

The big question: Where should we implement? in clangd, or in Sema?

[sammccall]

Neat!

The big question: Where should we implement? in clangd, or in Sema?

Advantages of Sema:

• better access to AST/context so more flexible
• available to tools other than clangd

Advantages of clangd:

• fewer layer of abstraction to traverse
• access to the index, config etc
• more comfortable making calls about how much to assume about stdlib etc
• maybe a bit easier to understand/test (depends on what you're used to)

We could get access to the index or other things from clangd by adding a hook to provide these "extended" member completions, and having clangd implement it... Not sure if that's best or worst of both worlds.

One question with Sema is how to make it produce the right text edit. Can we model it as a RK_Pattern completion with a typo fix to delete the foo. before the if?

This choice would be clearer if we had a set of examples we wanted to implement. If it's just if/for, then Sema seems like the right call...

clang-tools-extra/clangd/CodeComplete.cpp

 }
 
 // Identifier code completion result.
 struct RawIdentifier {
   llvm::StringRef Name;
   unsigned References; // # of usages in file.
 };
 
+struct PostfixCompletionResult {
+  // Display string, not inserted.
+  std::string Signature;
+  // The range being replaced by ReplacedText.
+  // e.g.
+  //  is_success.if
+  //  ~~~~~~~~~~~~~
+  Range ReplacedRange;
+  // The snippet being inserted when doing the completion.
+  // e.g. if (is_success) {}
+  std::string SnippetText;
+};
+
 /// A code completion result, in clang-native form.
 /// It may be promoted to a CompletionItem if it's among the top-ranked results.
 struct CompletionCandidate {
   llvm::StringRef Name; // Used for filtering and sorting.
   // We may have a result from Sema, from the index, or both.
   const CodeCompletionResult *SemaResult = nullptr;
   const Symbol *IndexResult = nullptr;
   const RawIdentifier *IdentifierResult = nullptr;
+  const PostfixCompletionResult *PostfixResult = nullptr;
   llvm::SmallVector<llvm::StringRef, 1> RankedIncludeHeaders;
 
   // Returns a token identifying the overload set this is part of.
   // 0 indicates it's not part of any overload set.
   size_t overloadSet(const CodeCompleteOptions &Opts) const {
     if (!Opts.BundleOverloads.getValueOr(false))
       return 0;
     llvm::SmallString<256> Scratch;
if (SemaResult) {
         return 0;
       {
         llvm::raw_svector_ostream OS(Scratch);
         D->printQualifiedName(OS);
       }
       return llvm::hash_combine(Scratch,
                                 headerToInsertIfAllowed(Opts).getValueOr(""));
     }
-    assert(IdentifierResult);
+    assert(IdentifierResult || PostfixResult);
     return 0;
   }
 
   // The best header to include if include insertion is allowed.
   llvm::Optional<llvm::StringRef>
   headerToInsertIfAllowed(const CodeCompleteOptions &Opts) const {
     if (Opts.InsertIncludes == CodeCompleteOptions::NeverInsert ||
         RankedIncludeHeaders.empty())
if (C.IndexResult) {
       }
       Completion.Deprecated |= (C.IndexResult->Flags & Symbol::Deprecated);
     }
     if (C.IdentifierResult) {
       Completion.Origin |= SymbolOrigin::Identifier;
       Completion.Kind = CompletionItemKind::Text;
       Completion.Name = std::string(C.IdentifierResult->Name);
     }
+    if (C.PostfixResult) {
+      Completion.Signature = C.PostfixResult->Signature;
+      Completion.Kind = CompletionItemKind::Snippet;
+      Completion.SnippetSuffix = C.PostfixResult->SnippetText;
+    }
 
     // Turn absolute path into a literal string that can be #included.
     auto Inserted = [&](llvm::StringRef Header)
         -> llvm::Expected<std::pair<std::string, bool>> {
       auto ResolvedDeclaring =
           URI::resolve(C.IndexResult->CanonicalDeclaration.FileURI, FileName);
       if (!ResolvedDeclaring)
         return ResolvedDeclaring.takeError();
if (C.SemaResult) {
       bool IsPattern = C.SemaResult->Kind == CodeCompletionResult::RK_Pattern;
       getSignature(*SemaCCS, &S.Signature, &S.SnippetSuffix,
                    &Completion.RequiredQualifier, IsPattern);
       S.ReturnType = getReturnType(*SemaCCS);
     } else if (C.IndexResult) {
       S.Signature = std::string(C.IndexResult->Signature);
       S.SnippetSuffix = std::string(C.IndexResult->CompletionSnippetSuffix);
       S.ReturnType = std::string(C.IndexResult->ReturnType);
+    } else if (C.PostfixResult) {
+      S.SnippetSuffix = C.PostfixResult->SnippetText;
+      S.Signature = C.PostfixResult->Signature;
     }
     if (ExtractDocumentation && !Completion.Documentation) {
       auto SetDoc = [&](llvm::StringRef Doc) {
         if (!Doc.empty()) {
           Completion.Documentation.emplace();
           parseDocumentation(Doc, *Completion.Documentation);
         }
       };
getQueryScopes(CodeCompletionContext &CCContext, const Sema &CCSema,
   Scopes.UnresolvedQualifier = std::string(SpelledSpecifier);
   // Sema excludes the trailing "::".
   if (!Scopes.UnresolvedQualifier->empty())
     *Scopes.UnresolvedQualifier += "::";
 
   return {Scopes.scopesForIndexQuery(), false};
 }
 
+bool contextAllowsPostfix(enum CodeCompletionContext::Kind K) {
+  switch (K) {
+  case CodeCompletionContext::CCC_DotMemberAccess:
+    return true;
+  default:
+    return false;
+  }
+}
+
 // Should we perform index-based completion in a context of the specified kind?
 // FIXME: consider allowing completion, but restricting the result types.
 bool contextAllowsIndex(enum CodeCompletionContext::Kind K) {
   switch (K) {
   case CodeCompletionContext::CCC_TopLevel:
   case CodeCompletionContext::CCC_ObjCInterface:
   case CodeCompletionContext::CCC_ObjCImplementation:
   case CodeCompletionContext::CCC_ObjCIvarList:
void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context,
     if (Context.getKind() == CodeCompletionContext::CCC_Recovery) {
       log("Code complete: Ignoring sema code complete callback with Recovery "
           "context.");
       return;
     }
     // If a callback is called without any sema result and the context does not
     // support index-based completion, we simply skip it to give way to
     // potential future callbacks with results.
-    if (NumResults == 0 && !contextAllowsIndex(Context.getKind()))
+    if (NumResults == 0 && !contextAllowsIndex(Context.getKind()) &&
+        !contextAllowsPostfix(Context.getKind())) {
       return;
+    }
     if (CCSema) {
       log("Multiple code complete callbacks (parser backtracked?). "
           "Dropping results from context {0}, keeping results from {1}.",
           getCompletionKindString(Context.getKind()),
           getCompletionKindString(this->CCContext.getKind()));
       return;
     }
     // Record the completion context.
else if (HeuristicPrefix.Qualifier.startswith("::")) {
       Scopes.UnresolvedQualifier = std::string(HeuristicPrefix.Qualifier);
     // First scope is the (modified) enclosing scope.
     QueryScopes = Scopes.scopesForIndexQuery();
     ScopeProximity.emplace(QueryScopes);
 
     SymbolSlab IndexResults = Opts.Index ? queryIndex() : SymbolSlab();
 
     CodeCompleteResult Output = toCodeCompleteResult(mergeResults(
-        /*SemaResults=*/{}, IndexResults, IdentifierResults));
+        /*SemaResults=*/{}, IndexResults, IdentifierResults, {}));
     Output.RanParser = false;
     logResults(Output, Tracer);
     return Output;
   }
 
 private:
   void populateContextWords(llvm::StringRef Content) {
     // Take last 3 lines before the completion point.
     unsigned RangeEnd = HeuristicPrefix.Qualifier.begin() - Content.data(),
              RangeBegin = RangeEnd;
     for (size_t I = 0; I < 3 && RangeBegin > 0; ++I) {
       auto PrevNL = Content.rfind('\n', RangeBegin);
       if (PrevNL == StringRef::npos) {
         RangeBegin = 0;
         break;
       }
       RangeBegin = PrevNL;
     }
 
     ContextWords = collectWords(Content.slice(RangeBegin, RangeEnd));
     dlog("Completion context words: {0}",
          llvm::join(ContextWords.keys(), ", "));
   }
 
+  std::vector<PostfixCompletionResult> getPostfixResults() {
+    // Postfix completion requires snippet support.
+    if (!Opts.EnableSnippets)
+      return {};
+    // Only trigger on ".".
+    if (Recorder->CCContext.getKind() !=
+        CodeCompletionContext::CCC_DotMemberAccess)
+      return {};
+    const auto &CCContext = Recorder->CCContext;
+    auto *CCSema = Recorder->CCSema;
+    const auto *BaseExpr = CCContext.getBaseExpr();
+    if (!BaseExpr)
+      return {};
+
+    // Setup the CodeCompletionRange range.
+    // FIXME: ReplacedRange is a global variable used by all CompletionItems,
+    // refine it to support CompletionItems with variant completion ranges.
+    ReplacedRange =
+        halfOpenToRange(CCSema->SourceMgr,
+                        CharSourceRange::getTokenRange(
+                            BaseExpr->getBeginLoc(),
+                            CCSema->getPreprocessor().getCodeCompletionLoc()));
+    llvm::StringRef SpelledExpr = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(BaseExpr->getSourceRange()),
+        CCSema->SourceMgr, CCSema->getASTContext().getLangOpts());
+    if (CCContext.getBaseType()->isBooleanType()) {
+      return {{/*Signature=*/"if (expressions) { statements }", ReplacedRange,
+               /*Snippet=*/
+               llvm::formatv("if ({0}) { {1} }", SpelledExpr, "${0:statements}")
+                   .str()}};
+    }
+    return {};
+  }
+
   // This is called by run() once Sema code completion is done, but before the
   // Sema data structures are torn down. It does all the real work.
   CodeCompleteResult runWithSema() {
     const auto &CodeCompletionRange = CharSourceRange::getCharRange(
         Recorder->CCSema->getPreprocessor().getCodeCompletionTokenRange());
     // When we are getting completions with an empty identifier, for example
     //    std::vector<int> asdf;
     //    asdf.^;
CodeCompleteResult runWithSema() {
     // FIXME: in addition to querying for extra/overlapping symbols, we should
     //        explicitly request symbols corresponding to Sema results.
     //        We can use their signals even if the index can't suggest them.
     // We must copy index results to preserve them, but there are at most Limit.
     auto IndexResults = (Opts.Index && allowIndex(Recorder->CCContext))
                             ? queryIndex()
                             : SymbolSlab();
     trace::Span Tracer("Populate CodeCompleteResult");
+    std::vector<PostfixCompletionResult> PostfixResults;
+    if (Recorder->Results.empty() &&
+        contextAllowsPostfix(Recorder->CCContext.getKind())) {
+      PostfixResults = getPostfixResults();
+    }
     // Merge Sema and Index results, score them, and pick the winners.
-    auto Top =
-        mergeResults(Recorder->Results, IndexResults, /*Identifiers*/ {});
+    auto Top = mergeResults(Recorder->Results, IndexResults, /*Identifiers*/ {},
+                            std::move(PostfixResults));
     return toCodeCompleteResult(Top);
   }
 
   CodeCompleteResult
   toCodeCompleteResult(const std::vector<ScoredBundle> &Scored) {
     CodeCompleteResult Output;
 
     // Convert the results to final form, assembling the expensive strings.
private:
   // Merges Sema and Index results where possible, to form CompletionCandidates.
   // \p Identifiers is raw identifiers that can also be completion candidates.
   // Identifiers are not merged with results from index or sema.
   // Groups overloads if desired, to form CompletionCandidate::Bundles. The
   // bundles are scored and top results are returned, best to worst.
   std::vector<ScoredBundle>
   mergeResults(const std::vector<CodeCompletionResult> &SemaResults,
                const SymbolSlab &IndexResults,
-               const std::vector<RawIdentifier> &IdentifierResults) {
+               const std::vector<RawIdentifier> &IdentifierResults,
+               const std::vector<PostfixCompletionResult> &PostfixResults) {
     trace::Span Tracer("Merge and score results");
     std::vector<CompletionCandidate::Bundle> Bundles;
     llvm::DenseMap<size_t, size_t> BundleLookup;
     auto AddToBundles = [&](const CodeCompletionResult *SemaResult,
                             const Symbol *IndexResult,
-                            const RawIdentifier *IdentifierResult) {
+                            const RawIdentifier *IdentifierResult,
+                            const PostfixCompletionResult *PostfixResult) {
       CompletionCandidate C;
       C.SemaResult = SemaResult;
       C.IndexResult = IndexResult;
       C.IdentifierResult = IdentifierResult;
+      C.PostfixResult = PostfixResult;
       if (C.IndexResult) {
         C.Name = IndexResult->Name;
         C.RankedIncludeHeaders = getRankedIncludes(*C.IndexResult);
       } else if (C.SemaResult) {
         C.Name = Recorder->getName(*SemaResult);
-      } else {
+      } else if (C.IdentifierResult) {
         assert(IdentifierResult);
         C.Name = IdentifierResult->Name;
+      } else {
+        assert(PostfixResult);
       }
       if (auto OverloadSet = C.overloadSet(Opts)) {
         auto Ret = BundleLookup.try_emplace(OverloadSet, Bundles.size());
         if (Ret.second)
           Bundles.emplace_back();
         Bundles[Ret.first->second].push_back(std::move(C));
       } else {
         Bundles.emplace_back();
auto CorrespondingIndexResult =
           UsedIndexResults.insert(&*I);
           return &*I;
         }
       }
       return nullptr;
     };
     // Emit all Sema results, merging them with Index results if possible.
     for (auto &SemaResult : SemaResults)
-      AddToBundles(&SemaResult, CorrespondingIndexResult(SemaResult), nullptr);
+      AddToBundles(&SemaResult, CorrespondingIndexResult(SemaResult), nullptr,
+                   nullptr);
     // Now emit any Index-only results.
     for (const auto &IndexResult : IndexResults) {
       if (UsedIndexResults.count(&IndexResult))
         continue;
-      AddToBundles(/*SemaResult=*/nullptr, &IndexResult, nullptr);
+      AddToBundles(/*SemaResult=*/nullptr, &IndexResult, nullptr, nullptr);
     }
     // Emit identifier results.
     for (const auto &Ident : IdentifierResults)
-      AddToBundles(/*SemaResult=*/nullptr, /*IndexResult=*/nullptr, &Ident);
+      AddToBundles(/*SemaResult=*/nullptr, /*IndexResult=*/nullptr, &Ident,
+                   nullptr);
+    for (const auto &PositFix : PostfixResults)
+      AddToBundles(/*SemaResult=*/nullptr, /*IndexResult=*/nullptr, nullptr,
+                   &PositFix);
     // We only keep the best N results at any time, in "native" format.
     TopN<ScoredBundle, ScoredBundleGreater> Top(
         Opts.Limit == 0 ? std::numeric_limits<size_t>::max() : Opts.Limit);
     for (auto &Bundle : Bundles)
       addCandidate(Top, std::move(Bundle));
     return std::move(Top).items();
   }
 
   llvm::Optional<float> fuzzyScore(const CompletionCandidate &C) {
     // Macros can be very spammy, so we only support prefix completion.
     // We won't end up with underfull index results, as macros are sema-only.
     if (C.SemaResult && C.SemaResult->Kind == CodeCompletionResult::RK_Macro &&
         !C.Name.startswith_lower(Filter->pattern()))
       return None;
+    // Name in PostfixResult is empty, use the replaced text to do the match.
+    if (C.PostfixResult && llvm::StringRef(C.PostfixResult->SnippetText)
+                               .startswith_lower(Filter->pattern()))
+      return Filter->match(C.PostfixResult->SnippetText);
     return Filter->match(C.Name);
   }
 
   // Scores a candidate and adds it to the TopN structure.
   void addCandidate(TopN<ScoredBundle, ScoredBundleGreater> &Candidates,
                     CompletionCandidate::Bundle Bundle) {
     SymbolQualitySignals Quality;
     SymbolRelevanceSignals Relevance;
for (const auto &Candidate : Bundle) {
         }
         Origin |= SymbolOrigin::AST;
       }
       if (Candidate.IdentifierResult) {
         Quality.References = Candidate.IdentifierResult->References;
         Relevance.Scope = SymbolRelevanceSignals::FileScope;
         Origin |= SymbolOrigin::Identifier;
       }
+      if (Candidate.PostfixResult)
+        Origin |= SymbolOrigin::AST;
     }
 
     CodeCompletion::Scores Scores;
     Scores.Quality = Quality.evaluate();
     Scores.Relevance = Relevance.evaluate();
     Scores.Total = evaluateSymbolAndRelevance(Scores.Quality, Scores.Relevance);
     // NameMatch is in fact a multiplier on total score, so rescoring is sound.
     Scores.ExcludingName = Relevance.NameMatch

clang/include/clang/Sema/CodeCompleteConsumer.h

private:
 
   /// The type that would prefer to see at this point (e.g., the type
   /// of an initializer or function parameter).
   QualType PreferredType;
 
   /// The type of the base object in a member access expression.
   QualType BaseType;
 
+  Expr *BaseExpr = nullptr;
+
   /// The identifiers for Objective-C selector parts.
   ArrayRef<IdentifierInfo *> SelIdents;
 
   /// The scope specifier that comes before the completion token e.g.
   /// "a::b::"
   llvm::Optional<CXXScopeSpec> ScopeSpecifier;
 
   /// A set of declaration contexts visited by Sema when doing lookup for
public:
   /// if the expression is a variable initializer or a function argument, the
   /// type of the corresponding variable or function parameter.
   QualType getPreferredType() const { return PreferredType; }
   void setPreferredType(QualType T) { PreferredType = T; }
 
   /// Retrieve the type of the base object in a member-access
   /// expression.
   QualType getBaseType() const { return BaseType; }
+  void setBaseExpr(Expr *E) { BaseExpr = E; }
+  Expr *getBaseExpr() const { return BaseExpr; }
 
   /// Retrieve the Objective-C selector identifiers.
   ArrayRef<IdentifierInfo *> getSelIdents() const { return SelIdents; }
 
   /// Determines whether we want C++ constructors as results within this
   /// context.
   bool wantConstructorResults() const;
 

clang/lib/Sema/SemaCodeComplete.cpp

if (ConvertedBaseType->isObjCObjectPointerType() ||
         ConvertedBaseType->isObjCObjectOrInterfaceType()) {
       contextKind = CodeCompletionContext::CCC_ObjCPropertyAccess;
     } else {
       contextKind = CodeCompletionContext::CCC_DotMemberAccess;
     }
   }
 
   CodeCompletionContext CCContext(contextKind, ConvertedBaseType);
+  CCContext.setBaseExpr(Base);
   CCContext.setPreferredType(PreferredType);
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CCContext,
                         &ResultBuilder::IsMember);
 
   auto DoCompletion = [&](Expr *Base, bool IsArrow,
                           Optional<FixItHint> AccessOpFixIt) -> bool {
     if (!Base)