[clangd] Construct SymbolSlab from YAML format.

[hokein]

Summary

This will be used together with D40548 for the global index source (experimental).

[sammccall]

Nice! just nits

clangd/index/SymbolFromYAML.cpp

32

what does NP stand for?

44

Nit: for readability, I suggest you encode as a hex string instead.

We don't yet have operator<< for SymbolID, but I think we should, and it should be consistent.
So preferably define operator<< for SymbolID, and use it here, rather than just defining everything here.

clangd/index/SymbolFromYAML.h

1

nit: SymbolFromYAML.h seems slightly off for the file, because you support both conversions (and also multiple symbols)

Consider just YAML.h?

8

File comment, describing what's going on here, use cases.

In particular, I'd note that the format is designed for simplicity but isn't a suitable/efficient store and isn't intended for real use by end-users.

23

We may have multiple slabs, e.g. if we want to dump the dynamic index.
If we dump each slab and concatenate the resulting strings, is the result valid YAML (containing all symbols)?
If so, then this interface is good, but it may be worth a comment // The YAML is safe to concatenate if you have multiple slabs.

unittests/clangd/SymbolCollectorTests.cpp

108

round-trip is good to test the serialization, but can you add a separate test case loading a simple symbol from an actual YAML string?

[hokein]

Address the review comments.

[hokein]

Thanks for the comments!

clangd/index/SymbolFromYAML.cpp

32

Ah, P is a typo here. N stands for Normalized. Renamed it to NormalizedXXX.

44

Good idea. Done.

clangd/index/SymbolFromYAML.h

1

YAML seems too general. Named it SymbolYAML indicating the conversion between Symbol and YAML.

[sammccall]

clangd/index/Index.h

17

This isn't needed anymore (if you remove the friend below)

57

I think this no longer needs to be a friend.

[hokein]

MappingTraits is not Symbol's friend.

clangd/CMakeLists.txt

add_clang_library(clangDaemon
   JSONExpr.cpp
   JSONRPCDispatcher.cpp
   Logger.cpp
   Protocol.cpp
   ProtocolHandlers.cpp
   Trace.cpp
   index/Index.cpp
   index/SymbolCollector.cpp
+  index/SymbolFromYAML.cpp
 
   LINK_LIBS
   clangAST
   clangBasic
   clangFormat
   clangFrontend
   clangIndex
   clangLex

clangd/index/Index.h

 //===--- Symbol.h -----------------------------------------------*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===---------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
 
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/YAMLTraits.h"
 
 #include <array>
 #include <string>
 
 namespace clang {
 namespace clangd {
 
 struct SymbolLocation {
public:
   bool operator==(const SymbolID& Sym) const {
     return HashValue == Sym.HashValue;
   }
 
 private:
   friend llvm::hash_code hash_value(const SymbolID &ID) {
     return hash_value(ArrayRef<uint8_t>(ID.HashValue));
   }
+  friend struct llvm::yaml::MappingTraits<class SymbolID>;
 
   std::array<uint8_t, 20> HashValue;
 };
 
 // The class presents a C++ symbol, e.g. class, function.
 //
 // FIXME: instead of having own copy fields for each symbol, we can share
 // storage from SymbolSlab.

clangd/index/SymbolFromYAML.h

+//===--- SymbolFromYAML.h ----------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+
+#include "Index.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace clangd {
+
+// Read symbols from a YAML-format string.
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent);
+
+// Convert symbols to a YAML-format string.
+std::string SymbolToYAML(const SymbolSlab& Symbols);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H

clangd/index/SymbolFromYAML.cpp

+//===--- SymbolFromYAML.cpp --------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolFromYAML.h"
+
+#include "Index.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Errc.h"
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(clang::clangd::Symbol)
+
+namespace llvm {
+namespace yaml {
+
+using clang::clangd::Symbol;
+using clang::clangd::SymbolID;
+using clang::clangd::SymbolLocation;
+using clang::index::SymbolInfo;
+using clang::index::SymbolLanguage;
+using clang::index::SymbolKind;
+
+// Helpers to (de)serialize the SymbolID underlying data as std::array is not
+// supported in YAML I/O.
+struct NPArray {
+  NPArray(IO &) {}
+  NPArray(IO &, std::array<uint8_t, 20> HashValue) {
+    Value = {HashValue.begin(), HashValue.end()};
+  }
+  std::array<uint8_t, 20> denormalize(IO &) {
+    assert(Value.size() == 20);
+    std::array<uint8_t, 20> Result;
+    std::copy(Value.begin(), Value.end(), Result.begin());
+    return Result;
+  }
+
+  std::vector<uint8_t> Value;
+};
+
+template <> struct MappingTraits<class SymbolID> {
+  static void mapping(IO &IO, SymbolID &ID) {
+    MappingNormalization<NPArray, std::array<uint8_t, 20>> NPArry(
+        IO, ID.HashValue);
+    IO.mapOptional("Value", NPArry->Value);
+  }
+};
+
+template <> struct MappingTraits<SymbolLocation> {
+  static void mapping(IO &IO, SymbolLocation &Value) {
+    IO.mapRequired("StartOffset", Value.StartOffset);
+    IO.mapRequired("EndOffset", Value.EndOffset);
+    IO.mapRequired("FilePath", Value.FilePath);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo> {
+  static void mapping(IO &io, SymbolInfo &SymInfo) {
+    // FIXME: expose other fields?
+    io.mapRequired("Kind", SymInfo.Kind);
+    io.mapRequired("Lang", SymInfo.Lang);
+  }
+};
+
+template<> struct MappingTraits<Symbol> {
+  static void mapping(IO &IO, Symbol &Sym) {
+    IO.mapRequired("ID", Sym.ID);
+    IO.mapRequired("QualifiedName", Sym.QualifiedName);
+    IO.mapRequired("SymInfo", Sym.SymInfo);
+    IO.mapRequired("CanonicalDeclaration", Sym.CanonicalDeclaration);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolLanguage> {
+  static void enumeration(IO &IO, SymbolLanguage &Value) {
+    IO.enumCase(Value, "C", SymbolLanguage::C);
+    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
+    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
+    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolKind> {
+  static void enumeration(IO &IO, SymbolKind &Value) {
+#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
+
+    DEFINE_ENUM(Unknown);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Module);
+    DEFINE_ENUM(Namespace);
+    DEFINE_ENUM(NamespaceAlias);
+    DEFINE_ENUM(Macro);
+    DEFINE_ENUM(Enum);
+    DEFINE_ENUM(Struct);
+    DEFINE_ENUM(Class);
+    DEFINE_ENUM(Protocol);
+    DEFINE_ENUM(Extension);
+    DEFINE_ENUM(Union);
+    DEFINE_ENUM(TypeAlias);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Variable);
+    DEFINE_ENUM(Field);
+    DEFINE_ENUM(EnumConstant);
+    DEFINE_ENUM(InstanceMethod);
+    DEFINE_ENUM(ClassMethod);
+    DEFINE_ENUM(StaticMethod);
+    DEFINE_ENUM(InstanceProperty);
+    DEFINE_ENUM(ClassProperty);
+    DEFINE_ENUM(StaticProperty);
+    DEFINE_ENUM(Constructor);
+    DEFINE_ENUM(Destructor);
+    DEFINE_ENUM(ConversionFunction);
+    DEFINE_ENUM(Parameter);
+    DEFINE_ENUM(Using);
+
+#undef DEFINE_ENUM
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace clangd {
+
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent) {
+  std::vector<Symbol> S;
+  llvm::yaml::Input Yin(YAMLContent);
+  Yin >> S;
+  SymbolSlab Syms;
+  for (auto& Sym : S)
+    Syms.insert(std::move(Sym));
+  return Syms;
+}
+
+std::string SymbolToYAML(const SymbolSlab& Symbols) {
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  llvm::yaml::Output Yout(OS);
+  for (auto &Pair : Symbols) {
+    Symbol MutableSymbol = Pair.second;
+    Yout<< MutableSymbol;
+  }
+  return OS.str();
+}
+
+} // namespace clangd
+} // namespace clang

unittests/clangd/SymbolCollectorTests.cpp

 //===-- SymbolCollectorTests.cpp  -------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "index/SymbolCollector.h"
+#include "index/SymbolFromYAML.h"
 #include "clang/Index/IndexingAction.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/VirtualFileSystem.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
const std::string Main = R"(
     namespace {
     void ff() {} // ignore
     }
     void f1() {}
   )";
   runSymbolCollector(Header, Main);
   EXPECT_THAT(Symbols, UnorderedElementsAre(QName("Foo"), QName("Foo::f"),
                                             QName("f1"), QName("f2")));
+
+  std::string YAMLContent = SymbolToYAML(Symbols);
+  auto SymbolsReadFromYAML = SymbolFromYAML(YAMLContent);
+  EXPECT_THAT(SymbolsReadFromYAML,
+              UnorderedElementsAre(QName("Foo"), QName("Foo::f"), QName("f1"),
+                                   QName("f2")));
 }
 
 } // namespace
 } // namespace clangd
 } // namespace clang