[clangd] Collect and store expected types in the index

[ilya-biryukov]

Summary

And add a hidden option to control whether the types are collected.
For experiments, will be removed when expected types implementation
is stabilized.

The index size is almost unchanged, e.g. the YAML index for all clangd
sources increased from 53MB to 54MB.

[ilya-biryukov]

Need to store the types in RIFF format too

[ilya-biryukov]

• Rebase and update after dependent changes.

[sammccall]

clangd/index/Index.cpp

118 ↗

(On Diff #173340)

why these changes?

clangd/index/SymbolCollector.h

80 ↗

(On Diff #173340)

Why make this an option? Is it expensive in time/size?

clangd/index/YAMLSerialization.cpp

202

also need to update the binary serialization?

[ilya-biryukov]

• Remove accidental changes
• Add types to binary serialization

[ilya-biryukov]

• Remove another accidental change

[ilya-biryukov]

clangd/index/Index.cpp

118 ↗

(On Diff #173340)

Sorry, leftovers from the times we had arrays of sha hashes.

clangd/index/SymbolCollector.h

80 ↗

(On Diff #173340)

Was there for experiments. They're not expensive, removed it.

[sammccall]

clangd/index/Index.h

287

either call this OpaqueType or point at in in the comment?

I'd put this below the ReturnType string too, it seems out of place here. But up to you.

clangd/index/SymbolCollector.cpp

591

nit: like this?

if (indexed for code completion)
 if (auto T = ...)
   S.Type = T->raw();

[ilya-biryukov]

clangd/index/Index.h

287

• Added a comment. OpaqueType would clash with the type name, something I'd prefer to avoid.
• Moved to live after ReturnType. I initially thought it'd be nice to keep them apart, since ReturnType is for showing it to the users and Type is for scoring. Now that I think about it does not look like a good idea, since keeping them apart means the user might read the comment for one of them, but not the other and end up inferring the other one on their own.

[ilya-biryukov]

• Bump the RIFF version number
• Place the Type field after the ReturnType
• Address other comments

[ilya-biryukov]

• Remove unused #include

clangd/index/Index.h

 //===--- Index.h -------------------------------------------------*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 
+#include "ExpectedTypes.h"
 #include "clang/Index/IndexSymbol.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/StringSaver.h"
 #include <array>
 #include <limits>
 #include <mutex>
struct Symbol {
   /// (When snippets are disabled, the symbol name alone is used).
   llvm::StringRef CompletionSnippetSuffix;
   /// Documentation including comment for the symbol declaration.
   llvm::StringRef Documentation;
   /// Type when this symbol is used in an expression. (Short display form).
   /// e.g. return type of a function, or type of a variable.
   llvm::StringRef ReturnType;
 
+  /// Raw representation of the OpaqueType of the symbol, used for scoring
+  /// purposes.
+  llvm::StringRef Type;
+
   struct IncludeHeaderWithReferences {
     IncludeHeaderWithReferences() = default;
 
     IncludeHeaderWithReferences(llvm::StringRef IncludeHeader,
                                 unsigned References)
         : IncludeHeader(IncludeHeader), References(References) {}
 
     /// This can be either a URI of the header to be #include'd
enum SymbolFlag : uint8_t {
     /// See also isIndexedForCodeCompletion().
     IndexedForCodeCompletion = 1 << 0,
     /// Indicates if the symbol is deprecated.
     Deprecated = 1 << 1,
     // Symbol is an implementation detail.
     ImplementationDetail = 1 << 2,
   };
 
   SymbolFlag Flags = SymbolFlag::None;
   /// FIXME: also add deprecation message and fixit?
 };
 inline Symbol::SymbolFlag  operator|(Symbol::SymbolFlag A, Symbol::SymbolFlag  B) {
   return static_cast<Symbol::SymbolFlag>(static_cast<uint8_t>(A) |
                                          static_cast<uint8_t>(B));
 }
 inline Symbol::SymbolFlag &operator|=(Symbol::SymbolFlag &A, Symbol::SymbolFlag B) {
   return A = A | B;
 }
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Symbol &S);
 raw_ostream &operator<<(raw_ostream &, Symbol::SymbolFlag);
 
 // Invokes Callback with each StringRef& contained in the Symbol.
 // Useful for deduplicating backing strings.
 template <typename Callback> void visitStrings(Symbol &S, const Callback &CB) {
   CB(S.Name);
   CB(S.Scope);
   CB(S.Signature);
   CB(S.CompletionSnippetSuffix);
   CB(S.Documentation);
   CB(S.ReturnType);
+  CB(S.Type);
   auto RawCharPointerCB = [&CB](const char *&P) {
     llvm::StringRef S(P);
     CB(S);
     assert(!S.data()[S.size()] && "Visited StringRef must be null-terminated");
     P = S.data();
   };
   RawCharPointerCB(S.CanonicalDeclaration.FileURI);
   RawCharPointerCB(S.Definition.FileURI);

clangd/index/Serialization.cpp

void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,
   writeLocation(Sym.CanonicalDeclaration, Strings, OS);
   writeVar(Sym.References, OS);
   OS.write(static_cast<uint8_t>(Sym.Flags));
   OS.write(static_cast<uint8_t>(Sym.Origin));
   writeVar(Strings.index(Sym.Signature), OS);
   writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);
   writeVar(Strings.index(Sym.Documentation), OS);
   writeVar(Strings.index(Sym.ReturnType), OS);
+  writeVar(Strings.index(Sym.Type), OS);
 
   auto WriteInclude = [&](const Symbol::IncludeHeaderWithReferences &Include) {
     writeVar(Strings.index(Include.IncludeHeader), OS);
     writeVar(Include.References, OS);
   };
   writeVar(Sym.IncludeHeaders.size(), OS);
   for (const auto &Include : Sym.IncludeHeaders)
     WriteInclude(Include);
Symbol readSymbol(Reader &Data, ArrayRef<StringRef> Strings) {
   Sym.CanonicalDeclaration = readLocation(Data, Strings);
   Sym.References = Data.consumeVar();
   Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consumeVar());
   Sym.Origin = static_cast<SymbolOrigin>(Data.consumeVar());
   Sym.Signature = Data.consumeString(Strings);
   Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
   Sym.Documentation = Data.consumeString(Strings);
   Sym.ReturnType = Data.consumeString(Strings);
+  Sym.Type = Data.consumeString(Strings);
   Sym.IncludeHeaders.resize(Data.consumeVar());
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);
     I.References = Data.consumeVar();
   }
   return Sym;
 }
 

 //   - srcs: checksum of the source file
 //   - stri: string table
 //   - symb: symbols
 //   - refs: references to symbols
 
 // The current versioning scheme is simple - non-current versions are rejected.
 // If you make a breaking change, bump this version number to invalidate stored
 // data. Later we may want to support some backward compatibility.
-constexpr static uint32_t Version = 7;
+constexpr static uint32_t Version = 8;
 
 Expected<IndexFileIn> readRIFF(StringRef Data) {
   auto RIFF = riff::readFile(Data);
   if (!RIFF)
     return RIFF.takeError();
   if (RIFF->Type != riff::fourCC("CdIx"))
     return makeError("wrong RIFF type");
   StringMap<StringRef> Chunks;

clangd/index/SymbolCollector.cpp

const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND,
   }
   S.Signature = Signature;
   S.CompletionSnippetSuffix = SnippetSuffix;
   S.Documentation = Documentation;
   S.ReturnType = ReturnType;
   if (!Include.empty())
     S.IncludeHeaders.emplace_back(Include, 1);
 
+  if (S.Flags & Symbol::IndexedForCodeCompletion) {
+    if (auto T = OpaqueType::fromCompletionResult(*ASTCtx, SymbolCompletion))
+      S.Type = T->raw();
+  }
+
   S.Origin = Opts.Origin;
   if (ND.getAvailability() == AR_Deprecated)
     S.Flags |= Symbol::Deprecated;
   Symbols.insert(S);
   return Symbols.find(S.ID);
 }
 
 void SymbolCollector::addDefinition(const NamedDecl &ND,

clangd/index/YAMLSerialization.cpp

static void mapping(IO &IO, Symbol &Sym) {
     IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
     IO.mapOptional("References", Sym.References, 0u);
     IO.mapOptional("Origin", NSymbolOrigin->Origin);
     IO.mapOptional("Flags", NSymbolFlag->Flag);
     IO.mapOptional("Signature", Sym.Signature);
     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
     IO.mapOptional("Documentation", Sym.Documentation);
     IO.mapOptional("ReturnType", Sym.ReturnType);
+    IO.mapOptional("Type", Sym.Type);
     IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
   }
 };
 
 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
   static void enumeration(IO &IO, SymbolLanguage &Value) {
     IO.enumCase(Value, "C", SymbolLanguage::C);
     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
     IO.enumCase(Value, "Swift", SymbolLanguage::Swift);