[clangd] Merge binary + YAML serialization behind a (mostly) common interface.

[sammccall]

Summary

Interface is in one file, implementation in two as they have little in common.
A couple of ad-hoc YAML functions left exposed:

• symbol -> YAML I expect to keep for tools like dexp
• YAML -> symbol is used for the MR-style indexer, I think we can eliminate this (merge-on-the-fly, else use a different serialization)

[sammccall]

Update tests.

[kbobyrev]

clangd/index/Serialization.cpp

407

nit: probably omit the last else so that it's easier to read? This is not exactly [[ https://llvm.org/docs/CodingStandards.html#don-t-use-else-after-a-return | Early Exits & else after return ]] case, but looks rather similar.

412

Would llvm::Option<SymbolIndex> be better here? I used this in the benchmark patch, that might be semantically better than returning nullptr.

clangd/indexer/IndexerMain.cpp

68

Maybe change default value to IndexFileFormat::RIFF? IndexFileOut::Format is set to IndexFileFormat::RIFF; in Serialization.h by default.

[sammccall]

clangd/index/Serialization.cpp

407

If we remove the else, then YAMLContents is no longer in scope.

412

SymbolIndex is abstract.

clangd/indexer/IndexerMain.cpp

68

I agree we should switch it, but this refactoring patch isn't the place to change the behavior of binaries.

[kbobyrev]

Ah, I see. Thanks for clarifying! I was wondering if we should also rename -yaml-symbol-file flag to something like -static-symbol-file (but, as you said, it's something to do in another patch).

LGTM

clangd/CMakeLists.txt

add_clang_library(clangDaemon
 
   index/CanonicalIncludes.cpp
   index/FileIndex.cpp
   index/Index.cpp
   index/MemIndex.cpp
   index/Merge.cpp
   index/Serialization.cpp
   index/SymbolCollector.cpp
-  index/SymbolYAML.cpp
+  index/YAMLSerialization.cpp
 
   index/dex/Dex.cpp
   index/dex/Iterator.cpp
   index/dex/PostingList.cpp
   index/dex/Trigram.cpp
 
   LINK_LIBS
   clangAST

clangd/benchmarks/IndexBenchmark.cpp

 //===--- IndexBenchmark.cpp - Clangd index benchmarks -----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
-#include "../index/SymbolYAML.h"
+#include "../index/Serialization.h"
 #include "../index/dex/Dex.h"
 #include "benchmark/benchmark.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
 #include <fstream>
 #include <streambuf>

clangd/index/Serialization.h

 //===--- Serialization.h - Binary serialization of index data ----*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
-// This file provides a compact binary serialization of indexed symbols.
+// This file provides serialization of indexed symbols and other data.
 //
-// It writes two sections:
+// It writes sections:
+//  - metadata such as version info
 //  - a string table (which is compressed)
 //  - lists of encoded symbols
 //
-// The format has a simple versioning scheme: the version is embedded in the
-// data and non-current versions are rejected when reading.
+// The format has a simple versioning scheme: the format version number is
+// written in the file and non-current versions are rejected when reading.
+//
+// Human-readable YAML serialization is also supported, and recommended for
+// debugging and experiments only.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_RIFF_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_RIFF_H
 #include "Index.h"
 #include "llvm/Support/Error.h"
 
+namespace llvm {
+namespace yaml {
+class Input;
+}
+} // namespace llvm
 namespace clang {
 namespace clangd {
 
+enum class IndexFileFormat {
+  RIFF, // Versioned binary format, suitable for production use.
+  YAML, // Human-readable format, suitable for experiments and debugging.
+};
+
 // Specifies the contents of an index file to be written.
 struct IndexFileOut {
   const SymbolSlab *Symbols;
   // TODO: Support serializing symbol occurrences.
   // TODO: Support serializing Dex posting lists.
+  IndexFileFormat Format = IndexFileFormat::RIFF;
 };
 // Serializes an index file. (This is a RIFF container chunk).
-llvm::raw_ostream &operator<<(llvm::raw_ostream &, const IndexFileOut &);
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O);
 
 // Holds the contents of an index file that was read.
 struct IndexFileIn {
   llvm::Optional<SymbolSlab> Symbols;
+  IndexFileFormat Format;
 };
 // Parse an index file. The input must be a RIFF container chunk.
 llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
 
+std::string toYAML(const Symbol &);
+// Returned symbol is backed by the YAML input.
+// FIXME: this is only needed for IndexerMain, find a better solution.
+llvm::Expected<Symbol> symbolFromYAML(llvm::yaml::Input &);
+
+// Build an in-memory static index from an index file.
+// The size should be relatively small, so data can be managed in memory.
+std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename,
+                                       llvm::ArrayRef<std::string> URISchemes,
+                                       bool UseDex = true);
+
 } // namespace clangd
 } // namespace clang
 
 #endif

clangd/index/Serialization.cpp

 //===-- Serialization.cpp - Binary serialization of index data ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 #include "Serialization.h"
 #include "Index.h"
 #include "RIFF.h"
+#include "Trace.h"
+#include "dex/Dex.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
 using namespace llvm;
 namespace clang {
 namespace clangd {
 namespace {
Symbol readSymbol(Reader &Data, ArrayRef<StringRef> Strings) {
   Sym.IncludeHeaders.resize(Data.consumeVar());
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);
     I.References = Data.consumeVar();
   }
   return Sym;
 }
 
-} // namespace
-
 // FILE ENCODING
 // A file is a RIFF chunk with type 'CdIx'.
 // It contains the sections:
 //   - meta: version number
 //   - stri: string table
 //   - symb: symbols
 
 // The current versioning scheme is simple - non-current versions are rejected.
 // If you make a breaking change, bump this version number to invalidate stored
 // data. Later we may want to support some backward compatibility.
 constexpr static uint32_t Version = 4;
 
-Expected<IndexFileIn> readIndexFile(StringRef Data) {
+Expected<IndexFileIn> readRIFF(StringRef Data) {
   auto RIFF = riff::readFile(Data);
   if (!RIFF)
     return RIFF.takeError();
   if (RIFF->Type != riff::fourCC("CdIx"))
     return makeError("wrong RIFF type");
   StringMap<StringRef> Chunks;
   for (const auto &Chunk : RIFF->Chunks)
     Chunks.try_emplace(StringRef(Chunk.ID.data(), Chunk.ID.size()), Chunk.Data);
while (!SymbolReader.eof())
       Symbols.insert(readSymbol(SymbolReader, Strings->Strings));
     if (SymbolReader.err())
       return makeError("malformed or truncated symbol");
     Result.Symbols = std::move(Symbols).build();
   }
   return std::move(Result);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const IndexFileOut &Data) {
+void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) {
   assert(Data.Symbols && "An index file without symbols makes no sense!");
   riff::File RIFF;
   RIFF.Type = riff::fourCC("CdIx");
 
   SmallString<4> Meta;
   {
     raw_svector_ostream MetaOS(Meta);
     write32(Version, MetaOS);
void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) {
   std::string SymbolSection;
   {
     raw_string_ostream SymbolOS(SymbolSection);
     for (const auto &Sym : Symbols)
       writeSymbol(Sym, Strings, SymbolOS);
   }
   RIFF.Chunks.push_back({riff::fourCC("symb"), SymbolSection});
 
-  return OS << RIFF;
+  OS << RIFF;
+}
+
+} // namespace
+
+// Defined in YAMLSerialization.cpp.
+void writeYAML(const IndexFileOut &, raw_ostream &);
+Expected<IndexFileIn> readYAML(StringRef);
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
+  switch (O.Format) {
+  case IndexFileFormat::RIFF:
+    writeYAML(O, OS);
+    break;
+  case IndexFileFormat::YAML:
+    writeRIFF(O, OS);
+    break;
+  }
+  return OS;
+}
+
+Expected<IndexFileIn> readIndexFile(StringRef Data) {
+  if (Data.startswith("RIFF")) {
+    return readRIFF(Data);
+  } else if (auto YAMLContents = readYAML(Data)) {
+    return std::move(*YAMLContents);
+  } else {
+    return makeError("Not a RIFF file and failed to parse as YAML: " +
+                     llvm::toString(YAMLContents.takeError()));
+  }
+}
+
+std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
+                                       llvm::ArrayRef<std::string> URISchemes,
+                                       bool UseDex) {
+  trace::Span OverallTracer("LoadIndex");
+  auto Buffer = MemoryBuffer::getFile(SymbolFilename);
+  if (!Buffer) {
+    llvm::errs() << "Can't open " << SymbolFilename << "\n";
+    return nullptr;
+  }
+
+  SymbolSlab Symbols;
+  RefSlab Refs;
+  {
+    trace::Span Tracer("ParseIndex");
+    if (auto I = readIndexFile(Buffer->get()->getBuffer())) {
+      if (I->Symbols)
+        Symbols = std::move(*I->Symbols);
+    } else {
+      llvm::errs() << "Bad Index: " << llvm::toString(I.takeError()) << "\n";
+      return nullptr;
+    }
+  }
+
+  trace::Span Tracer("BuildIndex");
+  return UseDex ? dex::Dex::build(std::move(Symbols), URISchemes)
+                : MemIndex::build(std::move(Symbols), std::move(Refs));
 }
 
 } // namespace clangd
 } // namespace clang

clangd/index/SymbolYAML.h

-//===--- SymbolYAML.h --------------------------------------------*- C++-*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// SymbolYAML provides facilities to convert Symbol to YAML, and vice versa.
-// The YAML format of Symbol is designed for simplicity and experiment, but
-// isn't a suitable/efficient store.
-//
-// This is for **experimental** only. Don't use it in the production code.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
-#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
-
-#include "Index.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/YAMLTraits.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace clang {
-namespace clangd {
-
-// Read symbols from a YAML-format string.
-SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent);
-
-// Read one symbol from a YAML-stream.
-// The returned symbol is backed by Input.
-Symbol SymbolFromYAML(llvm::yaml::Input &Input);
-
-// Convert a single symbol to YAML-format string.
-// The YAML result is safe to concatenate.
-std::string SymbolToYAML(Symbol Sym);
-
-// Convert symbols to a YAML-format string.
-// The YAML result is safe to concatenate if you have multiple symbol slabs.
-void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS);
-
-// Build an in-memory static index for global symbols from a symbol file.
-// The size of global symbols should be relatively small, so that all symbols
-// can be managed in memory.
-std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
-                                       llvm::ArrayRef<std::string> URISchemes,
-                                       bool UseDex = true);
-
-} // namespace clangd
-} // namespace clang
-
-#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H

clangd/index/SymbolYAML.cpp

-//===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "SymbolYAML.h"
-#include "Index.h"
-#include "Serialization.h"
-#include "Trace.h"
-#include "dex/Dex.h"
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cstdint>
-
-LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(clang::clangd::Symbol)
-LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
-
-namespace llvm {
-namespace yaml {
-
-using clang::clangd::Symbol;
-using clang::clangd::SymbolID;
-using clang::clangd::SymbolOrigin;
-using clang::clangd::SymbolLocation;
-using clang::index::SymbolInfo;
-using clang::index::SymbolKind;
-using clang::index::SymbolLanguage;
-
-// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
-struct NormalizedSymbolID {
-  NormalizedSymbolID(IO &) {}
-  NormalizedSymbolID(IO &, const SymbolID &ID) {
-    llvm::raw_string_ostream OS(HexString);
-    OS << ID;
-  }
-
-  SymbolID denormalize(IO &I) {
-    auto ID = SymbolID::fromStr(HexString);
-    if (!ID) {
-      I.setError(llvm::toString(ID.takeError()));
-      return SymbolID();
-    }
-    return *ID;
-  }
-
-  std::string HexString;
-};
-
-struct NormalizedSymbolFlag {
-  NormalizedSymbolFlag(IO &) {}
-  NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
-    Flag = static_cast<uint8_t>(F);
-  }
-
-  Symbol::SymbolFlag denormalize(IO &) {
-    return static_cast<Symbol::SymbolFlag>(Flag);
-  }
-
-  uint8_t Flag = 0;
-};
-
-struct NormalizedSymbolOrigin {
-  NormalizedSymbolOrigin(IO &) {}
-  NormalizedSymbolOrigin(IO &, SymbolOrigin O) {
-    Origin = static_cast<uint8_t>(O);
-  }
-
-  SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
-
-  uint8_t Origin = 0;
-};
-
-template <> struct MappingTraits<SymbolLocation::Position> {
-  static void mapping(IO &IO, SymbolLocation::Position &Value) {
-    IO.mapRequired("Line", Value.Line);
-    IO.mapRequired("Column", Value.Column);
-  }
-};
-
-template <> struct MappingTraits<SymbolLocation> {
-  static void mapping(IO &IO, SymbolLocation &Value) {
-    IO.mapRequired("FileURI", Value.FileURI);
-    IO.mapRequired("Start", Value.Start);
-    IO.mapRequired("End", Value.End);
-  }
-};
-
-template <> struct MappingTraits<SymbolInfo> {
-  static void mapping(IO &io, SymbolInfo &SymInfo) {
-    // FIXME: expose other fields?
-    io.mapRequired("Kind", SymInfo.Kind);
-    io.mapRequired("Lang", SymInfo.Lang);
-  }
-};
-
-template <>
-struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
-  static void mapping(IO &io,
-                      clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
-    io.mapRequired("Header", Inc.IncludeHeader);
-    io.mapRequired("References", Inc.References);
-  }
-};
-
-template <> struct MappingTraits<Symbol> {
-  static void mapping(IO &IO, Symbol &Sym) {
-    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
-    MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
-        IO, Sym.Flags);
-    MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
-        IO, Sym.Origin);
-    IO.mapRequired("ID", NSymbolID->HexString);
-    IO.mapRequired("Name", Sym.Name);
-    IO.mapRequired("Scope", Sym.Scope);
-    IO.mapRequired("SymInfo", Sym.SymInfo);
-    IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
-                   SymbolLocation());
-    IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
-    IO.mapOptional("References", Sym.References, 0u);
-    IO.mapOptional("Origin", NSymbolOrigin->Origin);
-    IO.mapOptional("Flags", NSymbolFlag->Flag);
-    IO.mapOptional("Signature", Sym.Signature);
-    IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
-    IO.mapOptional("Documentation", Sym.Documentation);
-    IO.mapOptional("ReturnType", Sym.ReturnType);
-    IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<SymbolLanguage> {
-  static void enumeration(IO &IO, SymbolLanguage &Value) {
-    IO.enumCase(Value, "C", SymbolLanguage::C);
-    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
-    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
-    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<SymbolKind> {
-  static void enumeration(IO &IO, SymbolKind &Value) {
-#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
-
-    DEFINE_ENUM(Unknown);
-    DEFINE_ENUM(Function);
-    DEFINE_ENUM(Module);
-    DEFINE_ENUM(Namespace);
-    DEFINE_ENUM(NamespaceAlias);
-    DEFINE_ENUM(Macro);
-    DEFINE_ENUM(Enum);
-    DEFINE_ENUM(Struct);
-    DEFINE_ENUM(Class);
-    DEFINE_ENUM(Protocol);
-    DEFINE_ENUM(Extension);
-    DEFINE_ENUM(Union);
-    DEFINE_ENUM(TypeAlias);
-    DEFINE_ENUM(Function);
-    DEFINE_ENUM(Variable);
-    DEFINE_ENUM(Field);
-    DEFINE_ENUM(EnumConstant);
-    DEFINE_ENUM(InstanceMethod);
-    DEFINE_ENUM(ClassMethod);
-    DEFINE_ENUM(StaticMethod);
-    DEFINE_ENUM(InstanceProperty);
-    DEFINE_ENUM(ClassProperty);
-    DEFINE_ENUM(StaticProperty);
-    DEFINE_ENUM(Constructor);
-    DEFINE_ENUM(Destructor);
-    DEFINE_ENUM(ConversionFunction);
-    DEFINE_ENUM(Parameter);
-    DEFINE_ENUM(Using);
-
-#undef DEFINE_ENUM
-  }
-};
-
-} // namespace yaml
-} // namespace llvm
-
-namespace clang {
-namespace clangd {
-
-SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent) {
-  llvm::yaml::Input Yin(YAMLContent);
-  std::vector<Symbol> S;
-  Yin >> S;
-
-  SymbolSlab::Builder Syms;
-  for (auto &Sym : S)
-    Syms.insert(Sym);
-  return std::move(Syms).build();
-}
-
-Symbol SymbolFromYAML(llvm::yaml::Input &Input) {
-  Symbol S;
-  Input >> S;
-  return S;
-}
-
-void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS) {
-  llvm::yaml::Output Yout(OS);
-  for (Symbol S : Symbols) // copy: Yout<< requires mutability.
-    Yout << S;
-}
-
-std::string SymbolToYAML(Symbol Sym) {
-  std::string Str;
-  llvm::raw_string_ostream OS(Str);
-  llvm::yaml::Output Yout(OS);
-  Yout << Sym;
-  return OS.str();
-}
-
-std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
-                                       llvm::ArrayRef<std::string> URISchemes,
-                                       bool UseDex) {
-  trace::Span OverallTracer("LoadIndex");
-  auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
-  if (!Buffer) {
-    llvm::errs() << "Can't open " << SymbolFilename << "\n";
-    return nullptr;
-  }
-  StringRef Data = Buffer->get()->getBuffer();
-
-  llvm::Optional<SymbolSlab> Slab;
-  if (Data.startswith("RIFF")) { // Magic for binary index file.
-    trace::Span Tracer("ParseRIFF");
-    if (auto RIFF = readIndexFile(Data))
-      Slab = std::move(RIFF->Symbols);
-    else
-      llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n";
-  } else {
-    trace::Span Tracer("ParseYAML");
-    Slab = symbolsFromYAML(Data);
-  }
-
-  if (!Slab)
-    return nullptr;
-  trace::Span Tracer("BuildIndex");
-  return UseDex ? dex::Dex::build(std::move(*Slab), URISchemes)
-                : MemIndex::build(std::move(*Slab), RefSlab());
-}
-
-} // namespace clangd
-} // namespace clang

clangd/index/YAMLSerialization.cpp

+//===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Index.h"
+#include "Serialization.h"
+#include "Trace.h"
+#include "dex/Dex.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdint>
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
+
+namespace llvm {
+namespace yaml {
+
+using clang::clangd::Symbol;
+using clang::clangd::SymbolID;
+using clang::clangd::SymbolLocation;
+using clang::clangd::SymbolOrigin;
+using clang::index::SymbolInfo;
+using clang::index::SymbolKind;
+using clang::index::SymbolLanguage;
+
+// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
+struct NormalizedSymbolID {
+  NormalizedSymbolID(IO &) {}
+  NormalizedSymbolID(IO &, const SymbolID &ID) {
+    llvm::raw_string_ostream OS(HexString);
+    OS << ID;
+  }
+
+  SymbolID denormalize(IO &I) {
+    auto ID = SymbolID::fromStr(HexString);
+    if (!ID) {
+      I.setError(llvm::toString(ID.takeError()));
+      return SymbolID();
+    }
+    return *ID;
+  }
+
+  std::string HexString;
+};
+
+struct NormalizedSymbolFlag {
+  NormalizedSymbolFlag(IO &) {}
+  NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
+    Flag = static_cast<uint8_t>(F);
+  }
+
+  Symbol::SymbolFlag denormalize(IO &) {
+    return static_cast<Symbol::SymbolFlag>(Flag);
+  }
+
+  uint8_t Flag = 0;
+};
+
+struct NormalizedSymbolOrigin {
+  NormalizedSymbolOrigin(IO &) {}
+  NormalizedSymbolOrigin(IO &, SymbolOrigin O) {
+    Origin = static_cast<uint8_t>(O);
+  }
+
+  SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
+
+  uint8_t Origin = 0;
+};
+
+template <> struct MappingTraits<SymbolLocation::Position> {
+  static void mapping(IO &IO, SymbolLocation::Position &Value) {
+    IO.mapRequired("Line", Value.Line);
+    IO.mapRequired("Column", Value.Column);
+  }
+};
+
+template <> struct MappingTraits<SymbolLocation> {
+  static void mapping(IO &IO, SymbolLocation &Value) {
+    IO.mapRequired("FileURI", Value.FileURI);
+    IO.mapRequired("Start", Value.Start);
+    IO.mapRequired("End", Value.End);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo> {
+  static void mapping(IO &io, SymbolInfo &SymInfo) {
+    // FIXME: expose other fields?
+    io.mapRequired("Kind", SymInfo.Kind);
+    io.mapRequired("Lang", SymInfo.Lang);
+  }
+};
+
+template <>
+struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
+  static void mapping(IO &io,
+                      clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
+    io.mapRequired("Header", Inc.IncludeHeader);
+    io.mapRequired("References", Inc.References);
+  }
+};
+
+template <> struct MappingTraits<Symbol> {
+  static void mapping(IO &IO, Symbol &Sym) {
+    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
+    MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
+        IO, Sym.Flags);
+    MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
+        IO, Sym.Origin);
+    IO.mapRequired("ID", NSymbolID->HexString);
+    IO.mapRequired("Name", Sym.Name);
+    IO.mapRequired("Scope", Sym.Scope);
+    IO.mapRequired("SymInfo", Sym.SymInfo);
+    IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
+                   SymbolLocation());
+    IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
+    IO.mapOptional("References", Sym.References, 0u);
+    IO.mapOptional("Origin", NSymbolOrigin->Origin);
+    IO.mapOptional("Flags", NSymbolFlag->Flag);
+    IO.mapOptional("Signature", Sym.Signature);
+    IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
+    IO.mapOptional("Documentation", Sym.Documentation);
+    IO.mapOptional("ReturnType", Sym.ReturnType);
+    IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolLanguage> {
+  static void enumeration(IO &IO, SymbolLanguage &Value) {
+    IO.enumCase(Value, "C", SymbolLanguage::C);
+    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
+    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
+    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolKind> {
+  static void enumeration(IO &IO, SymbolKind &Value) {
+#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
+
+    DEFINE_ENUM(Unknown);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Module);
+    DEFINE_ENUM(Namespace);
+    DEFINE_ENUM(NamespaceAlias);
+    DEFINE_ENUM(Macro);
+    DEFINE_ENUM(Enum);
+    DEFINE_ENUM(Struct);
+    DEFINE_ENUM(Class);
+    DEFINE_ENUM(Protocol);
+    DEFINE_ENUM(Extension);
+    DEFINE_ENUM(Union);
+    DEFINE_ENUM(TypeAlias);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Variable);
+    DEFINE_ENUM(Field);
+    DEFINE_ENUM(EnumConstant);
+    DEFINE_ENUM(InstanceMethod);
+    DEFINE_ENUM(ClassMethod);
+    DEFINE_ENUM(StaticMethod);
+    DEFINE_ENUM(InstanceProperty);
+    DEFINE_ENUM(ClassProperty);
+    DEFINE_ENUM(StaticProperty);
+    DEFINE_ENUM(Constructor);
+    DEFINE_ENUM(Destructor);
+    DEFINE_ENUM(ConversionFunction);
+    DEFINE_ENUM(Parameter);
+    DEFINE_ENUM(Using);
+
+#undef DEFINE_ENUM
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace clangd {
+
+void writeYAML(const IndexFileOut &O, raw_ostream &OS) {
+  llvm::yaml::Output Yout(OS);
+  for (Symbol Sym : *O.Symbols) // copy: Yout<< requires mutability.
+    Yout << Sym;
+}
+
+Expected<IndexFileIn> readYAML(StringRef Data) {
+  SymbolSlab::Builder Symbols;
+  llvm::yaml::Input Yin(Data);
+  do {
+    Symbol S;
+    Yin >> S;
+    if (Yin.error())
+      return llvm::errorCodeToError(Yin.error());
+    Symbols.insert(S);
+  } while (Yin.nextDocument());
+
+  IndexFileIn Result;
+  Result.Symbols.emplace(std::move(Symbols).build());
+  return std::move(Result);
+}
+
+std::string toYAML(const Symbol &S) {
+  std::string Buf;
+  {
+    llvm::raw_string_ostream OS(Buf);
+    llvm::yaml::Output Yout(OS);
+    Symbol Sym = S; // copy: Yout<< requires mutability.
+    OS << Sym;
+  }
+  return Buf;
+}
+
+Expected<Symbol> symbolFromYAML(llvm::yaml::Input &Yin) {
+  Symbol S;
+  Yin >> S;
+  if (Yin.error())
+    return llvm::errorCodeToError(Yin.error());
+  return S;
+}
+
+} // namespace clangd
+} // namespace clang

clangd/index/dex/dexp/Dexp.cpp

 //===--- Dexp.cpp - Dex EXPloration tool ------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a simple interactive tool which can be used to manually
 // evaluate symbol search quality of Clangd index.
 //
 //===----------------------------------------------------------------------===//
 
-#include "../../../index/SymbolYAML.h"
+#include "../../Serialization.h"
 #include "../Dex.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/LineEditor/LineEditor.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Signals.h"
 
if (!SID) {
       return;
     }
 
     clang::clangd::LookupRequest Request;
     Request.IDs = {*SID};
     bool FoundSymbol = false;
     Index->lookup(Request, [&](const Symbol &Sym) {
       FoundSymbol = true;
-      llvm::outs() << SymbolToYAML(Sym);
+      llvm::outs() << toYAML(Sym);
     });
     if (!FoundSymbol)
       llvm::outs() << "not found\n";
   }
 };
 
 struct {
   const char *Name;

clangd/indexer/IndexerMain.cpp

 //===----------------------------------------------------------------------===//
 
 #include "RIFF.h"
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "index/Merge.h"
 #include "index/Serialization.h"
 #include "index/SymbolCollector.h"
-#include "index/SymbolYAML.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Index/IndexDataConsumer.h"
 #include "clang/Index/IndexingAction.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Execution.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/CommandLine.h"
static llvm::cl::opt<bool> MergeOnTheFly(
     llvm::cl::desc(
         "Merges symbols for each processed translation unit as soon "
         "they become available. This results in a smaller memory "
         "usage and an almost instant reduce stage. Optimal for running as a "
         "standalone tool, but cannot be used with multi-process executors like "
         "MapReduce."),
     llvm::cl::init(true), llvm::cl::Hidden);
 
-enum IndexFormat { YAML, Binary };
-static llvm::cl::opt<IndexFormat> Format(
-    "format", llvm::cl::desc("Format of the index to be written"),
-    llvm::cl::values(clEnumValN(YAML, "yaml", "human-readable YAML format"),
-                     clEnumValN(Binary, "binary", "binary RIFF format")),
-    llvm::cl::init(YAML));
+static llvm::cl::opt<IndexFileFormat>
+    Format("format", llvm::cl::desc("Format of the index to be written"),
+           llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml",
+                                       "human-readable YAML format"),
+                            clEnumValN(IndexFileFormat::RIFF, "binary",
+                                       "binary RIFF format")),
+           llvm::cl::init(IndexFileFormat::YAML));
 
 /// Responsible for aggregating symbols from each processed file and producing
 /// the final results. All methods in this class must be thread-safe,
 /// 'consumeSymbols' may be called from multiple threads.
 class SymbolsConsumer {
 public:
   virtual ~SymbolsConsumer() = default;
 

 /// end. Useful for running on MapReduce infrastructure to avoid keeping symbols
 /// from multiple files in memory.
 class ToolExecutorConsumer : public SymbolsConsumer {
 public:
   ToolExecutorConsumer(ToolExecutor &Executor) : Executor(Executor) {}
 
   void consumeSymbols(SymbolSlab Symbols) override {
     for (const auto &Sym : Symbols)
-      Executor.getExecutionContext()->reportResult(Sym.ID.str(),
-                                                   SymbolToYAML(Sym));
+      Executor.getExecutionContext()->reportResult(Sym.ID.str(), toYAML(Sym));
   }
 
   SymbolSlab mergeResults() override {
     SymbolSlab::Builder UniqueSymbols;
     Executor.getToolResults()->forEachResult(
         [&](llvm::StringRef Key, llvm::StringRef Value) {
           llvm::yaml::Input Yin(Value);
-          auto Sym = clang::clangd::SymbolFromYAML(Yin);
+          auto Sym = cantFail(clang::clangd::symbolFromYAML(Yin));
           auto ID = cantFail(clang::clangd::SymbolID::fromStr(Key));
           if (const auto *Existing = UniqueSymbols.find(ID))
             UniqueSymbols.insert(mergeSymbol(*Existing, Sym));
           else
             UniqueSymbols.insert(Sym);
         });
     return std::move(UniqueSymbols).build();
   }
int main(int argc, const char **argv) {
   auto Err = Executor->get()->execute(
       llvm::make_unique<clang::clangd::SymbolIndexActionFactory>(*Consumer));
   if (Err) {
     llvm::errs() << llvm::toString(std::move(Err)) << "\n";
   }
   // Reduce phase: combine symbols with the same IDs.
   auto UniqueSymbols = Consumer->mergeResults();
   // Output phase: emit result symbols.
-  switch (clang::clangd::Format) {
-  case clang::clangd::IndexFormat::YAML:
-    SymbolsToYAML(UniqueSymbols, llvm::outs());
-    break;
-  case clang::clangd::IndexFormat::Binary: {
-    clang::clangd::IndexFileOut Out;
-    Out.Symbols = &UniqueSymbols;
-    llvm::outs() << Out;
-  }
-  }
+  Out.Format = clang::clangd::Format;
   return 0;
 }

clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangdLSPServer.h"
 #include "JSONRPCDispatcher.h"
 #include "Path.h"
 #include "RIFF.h"
 #include "Trace.h"
-#include "index/SymbolYAML.h"
+#include "index/Serialization.h"
 #include "clang/Basic/Version.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>

unittests/clangd/SerializationTests.cpp

 //===-- SerializationTests.cpp - Binary and YAML serialization unit tests -===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "index/Index.h"
 #include "index/Serialization.h"
-#include "index/SymbolYAML.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+using testing::AllOf;
 using testing::UnorderedElementsAre;
 using testing::UnorderedElementsAreArray;
 namespace clang {
 namespace clangd {
 namespace {
 
-const char *YAML1 = R"(
+const char *YAML = R"(
 ---
 ID: 057557CEBF6E6B2DD437FBF60CC58F352D1DF856
 Name:   'Foo1'
 Scope:   'clang::'
 SymInfo:
   Kind:            Function
   Lang:            Cpp
 CanonicalDeclaration:

 Documentation:    'Foo doc'
 ReturnType:    'int'
 IncludeHeaders:
   - Header:    'include1'
     References:    7
   - Header:    'include2'
     References:    3
 ...
-)";
-
-const char *YAML2 = R"(
 ---
 ID: 057557CEBF6E6B2DD437FBF60CC58F352D1DF858
 Name:   'Foo2'
 Scope:   'clang::'
 SymInfo:
   Kind:            Function
   Lang:            Cpp
 CanonicalDeclaration:
   FileURI:        file:///path/bar.h
   Start:
     Line: 1
     Column: 0
   End:
     Line: 1
     Column: 1
 Flags:    2
 Signature:    '-sig'
 CompletionSnippetSuffix:    '-snippet'
 ...
 )";
 
+MATCHER_P(ID, I, "") { return arg.ID == cantFail(SymbolID::fromStr(I)); }
 MATCHER_P(QName, Name, "") { return (arg.Scope + arg.Name).str() == Name; }
 MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") {
   return (arg.IncludeHeader == IncludeHeader) && (arg.References == References);
 }
 
 TEST(SerializationTest, YAMLConversions) {
-  auto Symbols1 = symbolsFromYAML(YAML1);
-  ASSERT_EQ(Symbols1.size(), 1u);
-  const auto &Sym1 = *Symbols1.begin();
+  auto In = readIndexFile(YAML);
+  EXPECT_TRUE(bool(In)) << In.takeError();
+
+  auto ParsedYAML = readIndexFile(YAML);
+  ASSERT_TRUE(bool(ParsedYAML)) << ParsedYAML.takeError();
+  ASSERT_TRUE(bool(ParsedYAML->Symbols));
+  EXPECT_THAT(
+      *ParsedYAML->Symbols,
+      UnorderedElementsAre(ID("057557CEBF6E6B2DD437FBF60CC58F352D1DF856"),
+                           ID("057557CEBF6E6B2DD437FBF60CC58F352D1DF858")));
+
+  auto Sym1 = *ParsedYAML->Symbols->find(
+      cantFail(SymbolID::fromStr("057557CEBF6E6B2DD437FBF60CC58F352D1DF856")));
+  auto Sym2 = *ParsedYAML->Symbols->find(
+      cantFail(SymbolID::fromStr("057557CEBF6E6B2DD437FBF60CC58F352D1DF858")));
+
   EXPECT_THAT(Sym1, QName("clang::Foo1"));
   EXPECT_EQ(Sym1.Signature, "");
   EXPECT_EQ(Sym1.Documentation, "Foo doc");
   EXPECT_EQ(Sym1.ReturnType, "int");
   EXPECT_EQ(Sym1.CanonicalDeclaration.FileURI, "file:///path/foo.h");
   EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static);
   EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion);
   EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);
   EXPECT_THAT(Sym1.IncludeHeaders,
               UnorderedElementsAre(IncludeHeaderWithRef("include1", 7u),
                                    IncludeHeaderWithRef("include2", 3u)));
 
-  auto Symbols2 = symbolsFromYAML(YAML2);
-  ASSERT_EQ(Symbols2.size(), 1u);
-  const auto &Sym2 = *Symbols2.begin();
   EXPECT_THAT(Sym2, QName("clang::Foo2"));
   EXPECT_EQ(Sym2.Signature, "-sig");
   EXPECT_EQ(Sym2.ReturnType, "");
   EXPECT_EQ(Sym2.CanonicalDeclaration.FileURI, "file:///path/bar.h");
   EXPECT_FALSE(Sym2.Flags & Symbol::IndexedForCodeCompletion);
   EXPECT_TRUE(Sym2.Flags & Symbol::Deprecated);
-
-  std::string ConcatenatedYAML;
-  {
-    llvm::raw_string_ostream OS(ConcatenatedYAML);
-    SymbolsToYAML(Symbols1, OS);
-    SymbolsToYAML(Symbols2, OS);
-  }
-  auto ConcatenatedSymbols = symbolsFromYAML(ConcatenatedYAML);
-  EXPECT_THAT(ConcatenatedSymbols,
-              UnorderedElementsAre(QName("clang::Foo1"), QName("clang::Foo2")));
 }
 
 std::vector<std::string> YAMLFromSymbols(const SymbolSlab &Slab) {
   std::vector<std::string> Result;
   for (const auto &Sym : Slab)
-    Result.push_back(SymbolToYAML(Sym));
+    Result.push_back(toYAML(Sym));
   return Result;
 }
 
 TEST(SerializationTest, BinaryConversions) {
-  // We reuse the test symbols from YAML.
-  auto Slab = symbolsFromYAML(std::string(YAML1) + YAML2);
-  ASSERT_EQ(Slab.size(), 2u);
+  auto In = readIndexFile(YAML);
+  EXPECT_TRUE(bool(In)) << In.takeError();
 
   // Write to binary format, and parse again.
   IndexFileOut Out;
-  Out.Symbols = &Slab;
+  Out.Symbols = In->Symbols.getPointer();
+  Out.Format = IndexFileFormat::RIFF;
   std::string Serialized = llvm::to_string(Out);
 
-  auto In = readIndexFile(Serialized);
-  ASSERT_TRUE(bool(In)) << In.takeError();
+  auto In2 = readIndexFile(Serialized);
+  ASSERT_TRUE(bool(In2)) << In.takeError();
   ASSERT_TRUE(In->Symbols);
 
   // Assert the YAML serializations match, for nice comparisons and diffs.
-  EXPECT_THAT(YAMLFromSymbols(*In->Symbols),
-              UnorderedElementsAreArray(YAMLFromSymbols(Slab)));
+  EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
+              UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
 }
 
 } // namespace
 } // namespace clangd
 } // namespace clang

unittests/clangd/SymbolCollectorTests.cpp

 //===-- SymbolCollectorTests.cpp  -------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Annotations.h"
 #include "TestFS.h"
 #include "TestTU.h"
 #include "index/SymbolCollector.h"
-#include "index/SymbolYAML.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/VirtualFileSystem.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Index/IndexingAction.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"