[clangd] Collect and store expected types in the index

[ilya-biryukov]

Summary

And add a hidden option to control whether the types are collected.
For experiments, will be removed when expected types implementation
is stabilized.

The index size is almost unchanged, e.g. the YAML index for all clangd
sources increased from 53MB to 54MB.

[ilya-biryukov]

Need to store the types in RIFF format too

[ilya-biryukov]

• Rebase and update after dependent changes.

[sammccall]

clangd/index/Index.cpp

118 ↗

(On Diff #173340)

why these changes?

clangd/index/SymbolCollector.h

80 ↗

(On Diff #173340)

Why make this an option? Is it expensive in time/size?

clangd/index/YAMLSerialization.cpp

201

also need to update the binary serialization?

[ilya-biryukov]

• Remove accidental changes
• Add types to binary serialization

[ilya-biryukov]

• Remove another accidental change

[ilya-biryukov]

clangd/index/Index.cpp

118 ↗

(On Diff #173340)

Sorry, leftovers from the times we had arrays of sha hashes.

clangd/index/SymbolCollector.h

80 ↗

(On Diff #173340)

Was there for experiments. They're not expensive, removed it.

[sammccall]

clangd/index/Index.h

283

either call this OpaqueType or point at in in the comment?

I'd put this below the ReturnType string too, it seems out of place here. But up to you.

clangd/index/SymbolCollector.cpp

591

nit: like this?

if (indexed for code completion)
 if (auto T = ...)
   S.Type = T->raw();

[ilya-biryukov]

clangd/index/Index.h

283

• Added a comment. OpaqueType would clash with the type name, something I'd prefer to avoid.
• Moved to live after ReturnType. I initially thought it'd be nice to keep them apart, since ReturnType is for showing it to the users and Type is for scoring. Now that I think about it does not look like a good idea, since keeping them apart means the user might read the comment for one of them, but not the other and end up inferring the other one on their own.

[ilya-biryukov]

• Bump the RIFF version number
• Place the Type field after the ReturnType
• Address other comments

[ilya-biryukov]

• Remove unused #include

clangd/index/Index.h

 //===--- Index.h -------------------------------------------------*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 
+#include "ExpectedTypes.h"
 #include "clang/Index/IndexSymbol.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/StringSaver.h"
 #include <array>
 #include <limits>
 #include <mutex>
enum SymbolFlag : uint8_t {
     /// See also isIndexedForCodeCompletion().
     IndexedForCodeCompletion = 1 << 0,
     /// Indicates if the symbol is deprecated.
     Deprecated = 1 << 1,
     // Symbol is an implementation detail.
     ImplementationDetail = 1 << 2,
   };
 
+  /// Type of the symbol, used for scoring purposes.
+  llvm::StringRef Type;
+
   SymbolFlag Flags = SymbolFlag::None;
   /// FIXME: also add deprecation message and fixit?
 };
 inline Symbol::SymbolFlag  operator|(Symbol::SymbolFlag A, Symbol::SymbolFlag  B) {
   return static_cast<Symbol::SymbolFlag>(static_cast<uint8_t>(A) |
                                          static_cast<uint8_t>(B));
 }
 inline Symbol::SymbolFlag &operator|=(Symbol::SymbolFlag &A, Symbol::SymbolFlag B) {
auto RawCharPointerCB = [&CB](const char *&P) {
     assert(!S.data()[S.size()] && "Visited StringRef must be null-terminated");
     P = S.data();
   };
   RawCharPointerCB(S.CanonicalDeclaration.FileURI);
   RawCharPointerCB(S.Definition.FileURI);
 
   for (auto &Include : S.IncludeHeaders)
     CB(Include.IncludeHeader);
+  CB(S.Type);
 }
 
 // Computes query-independent quality score for a Symbol.
 // This currently falls in the range [1, ln(#indexed documents)].
 // FIXME: this should probably be split into symbol -> signals
 //        and signals -> score, so it can be reused for Sema completions.
 float quality(const Symbol &S);
 

clangd/index/Serialization.cpp

void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,
 
   auto WriteInclude = [&](const Symbol::IncludeHeaderWithReferences &Include) {
     writeVar(Strings.index(Include.IncludeHeader), OS);
     writeVar(Include.References, OS);
   };
   writeVar(Sym.IncludeHeaders.size(), OS);
   for (const auto &Include : Sym.IncludeHeaders)
     WriteInclude(Include);
+  writeVar(Strings.index(Sym.Type), OS);
 }
 
 Symbol readSymbol(Reader &Data, ArrayRef<StringRef> Strings) {
   Symbol Sym;
   Sym.ID = Data.consumeID();
   Sym.SymInfo.Kind = static_cast<index::SymbolKind>(Data.consume8());
   Sym.SymInfo.Lang = static_cast<index::SymbolLanguage>(Data.consume8());
   Sym.Name = Data.consumeString(Strings);
   Sym.Scope = Data.consumeString(Strings);
   Sym.Definition = readLocation(Data, Strings);
   Sym.CanonicalDeclaration = readLocation(Data, Strings);
   Sym.References = Data.consumeVar();
   Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consumeVar());
   Sym.Origin = static_cast<SymbolOrigin>(Data.consumeVar());
   Sym.Signature = Data.consumeString(Strings);
   Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
   Sym.Documentation = Data.consumeString(Strings);
   Sym.ReturnType = Data.consumeString(Strings);
   Sym.IncludeHeaders.resize(Data.consumeVar());
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);
     I.References = Data.consumeVar();
   }
+  Sym.Type = Data.consumeString(Strings);
   return Sym;
 }
 
 // REFS ENCODING
 // A refs section has data grouped by Symbol. Each symbol has:
 //  - SymbolID: 8 bytes
 //  - NumRefs: varint
 //  - Ref[NumRefs]

clangd/index/SymbolCollector.cpp

const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND,
   }
   S.Signature = Signature;
   S.CompletionSnippetSuffix = SnippetSuffix;
   S.Documentation = Documentation;
   S.ReturnType = ReturnType;
   if (!Include.empty())
     S.IncludeHeaders.emplace_back(Include, 1);
 
+  llvm::Optional<OpaqueType> Type;
+  if (S.Flags & Symbol::IndexedForCodeCompletion)
+    Type = OpaqueType::fromCompletionResult(*ASTCtx, SymbolCompletion);
+  S.Type = Type ? Type->raw() : "";
+
   S.Origin = Opts.Origin;
   if (ND.getAvailability() == AR_Deprecated)
     S.Flags |= Symbol::Deprecated;
   Symbols.insert(S);
   return Symbols.find(S.ID);
 }
 
 void SymbolCollector::addDefinition(const NamedDecl &ND,

clangd/index/YAMLSerialization.cpp

static void mapping(IO &IO, Symbol &Sym) {
     IO.mapOptional("References", Sym.References, 0u);
     IO.mapOptional("Origin", NSymbolOrigin->Origin);
     IO.mapOptional("Flags", NSymbolFlag->Flag);
     IO.mapOptional("Signature", Sym.Signature);
     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
     IO.mapOptional("Documentation", Sym.Documentation);
     IO.mapOptional("ReturnType", Sym.ReturnType);
     IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
+    IO.mapOptional("Type", Sym.Type);
   }
 };
 
 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
   static void enumeration(IO &IO, SymbolLanguage &Value) {
     IO.enumCase(Value, "C", SymbolLanguage::C);
     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);

clangd/indexer/IndexerMain.cpp

Format("format", cl::desc("Format of the index to be written"),
                                  "binary RIFF format")),
            cl::init(IndexFileFormat::RIFF));
 
 class IndexActionFactory : public tooling::FrontendActionFactory {
 public:
   IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
 
   clang::FrontendAction *create() override {
-    SymbolCollector::Options Opts;
     return createStaticIndexingAction(
-               Opts,
+               SymbolCollector::Options(),
                [&](SymbolSlab S) {
                  // Merge as we go.
                  std::lock_guard<std::mutex> Lock(SymbolsMu);
                  for (const auto &Sym : S) {
                    if (const auto *Existing = Symbols.find(Sym.ID))
                      Symbols.insert(mergeSymbol(*Existing, Sym));
                    else
                      Symbols.insert(Sym);