[clangd] Extend YAML Serialization

[mnauw]

This was previously part of D77385. Rather than a separate file (which seems a bit tricky), the test YAML has simply been included in the unit test itself.

[sammccall]

Thanks for doing this!

clang-tools-extra/clangd/unittests/YAMLTests.cpp

1 ↗

(On Diff #256757)

Ack, sorry for not mentioning this in the previous review.
The YAML stuff is tested mixed together with the binary format stuff in SerializationTests.cpp.
Is it possible to extend the YAML literal in that file with a synthetic Source and Cmd entry, and test it in a similar way?

[mnauw]

As suggested, existing (YAML) Serialization test has been extended.

[sammccall]

Thank you!

Do you have commit access, or would you like me to land this and D78038 for you?

[mnauw]

I do not have commit access, so it would be helpful that you land this (and the other) ;-)

[sammccall]

Done, thanks for the contributions!
Feel free to apply for commit access at any point if you plan to work more on clangd or other LLVM stuff.

clang-tools-extra/clangd/index/YAMLSerialization.cpp

 namespace {
 using RefBundle =
     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
 struct VariantEntry {
   llvm::Optional<clang::clangd::Symbol> Symbol;
   llvm::Optional<RefBundle> Refs;
   llvm::Optional<clang::clangd::Relation> Relation;
+  llvm::Optional<clang::clangd::IncludeGraphNode> Source;
+  llvm::Optional<clang::tooling::CompileCommand> Cmd;
 };
 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
 // as YAMLIO can't directly map bitfields.
 struct YPosition {
   uint32_t Line;
   uint32_t Column;
 };
 
+// avoid ODR violation of specialization for non-owned CompileCommand
+struct CompileCommandYAML : clang::tooling::CompileCommand {};
+
 } // namespace
 namespace llvm {
 namespace yaml {
 
+using clang::clangd::FileDigest;
+using clang::clangd::IncludeGraph;
+using clang::clangd::IncludeGraphNode;
 using clang::clangd::Ref;
 using clang::clangd::RefKind;
 using clang::clangd::Relation;
 using clang::clangd::RelationKind;
 using clang::clangd::Symbol;
 using clang::clangd::SymbolID;
 using clang::clangd::SymbolLocation;
 using clang::clangd::SymbolOrigin;
 using clang::index::SymbolInfo;
 using clang::index::SymbolKind;
 using clang::index::SymbolLanguage;
 using clang::index::SymbolRole;
+using clang::tooling::CompileCommand;
 
 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
 struct NormalizedSymbolID {
   NormalizedSymbolID(IO &) {}
   NormalizedSymbolID(IO &, const SymbolID &ID) {
     llvm::raw_string_ostream OS(HexString);
     OS << ID;
   }
static void mapping(IO &IO, Relation &Relation) {
     MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
         IO, Relation.Predicate);
     IO.mapRequired("Subject", Relation.Subject);
     IO.mapRequired("Predicate", NRole->Kind);
     IO.mapRequired("Object", Relation.Object);
   }
 };
 
+struct NormalizedSourceFlag {
+  NormalizedSourceFlag(IO &) {}
+  NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
+    Flag = static_cast<uint8_t>(O);
+  }
+
+  IncludeGraphNode::SourceFlag denormalize(IO &) {
+    return static_cast<IncludeGraphNode::SourceFlag>(Flag);
+  }
+
+  uint8_t Flag = 0;
+};
+
+struct NormalizedFileDigest {
+  NormalizedFileDigest(IO &) {}
+  NormalizedFileDigest(IO &, const FileDigest &Digest) {
+    HexString = llvm::toHex(Digest);
+  }
+
+  FileDigest denormalize(IO &I) {
+    FileDigest Digest;
+    if (HexString.size() == Digest.size() * 2 &&
+        llvm::all_of(HexString, llvm::isHexDigit)) {
+      memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
+    } else {
+      I.setError(std::string("Bad hex file digest: ") + HexString);
+    }
+    return Digest;
+  }
+
+  std::string HexString;
+};
+
+template <> struct MappingTraits<IncludeGraphNode> {
+  static void mapping(IO &IO, IncludeGraphNode &Node) {
+    IO.mapRequired("URI", Node.URI);
+    MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
+        NSourceFlag(IO, Node.Flags);
+    IO.mapRequired("Flags", NSourceFlag->Flag);
+    MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
+                                                                   Node.Digest);
+    IO.mapRequired("Digest", NDigest->HexString);
+    IO.mapRequired("DirectIncludes", Node.DirectIncludes);
+  }
+};
+
+template <> struct MappingTraits<CompileCommandYAML> {
+  static void mapping(IO &IO, CompileCommandYAML &Cmd) {
+    IO.mapRequired("Directory", Cmd.Directory);
+    IO.mapRequired("CommandLine", Cmd.CommandLine);
+  }
+};
+
 template <> struct MappingTraits<VariantEntry> {
   static void mapping(IO &IO, VariantEntry &Variant) {
     if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) {
       if (!IO.outputting())
         Variant.Symbol.emplace();
       MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
     } else if (IO.mapTag("!Refs", Variant.Refs.hasValue())) {
       if (!IO.outputting())
         Variant.Refs.emplace();
       MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
     } else if (IO.mapTag("!Relations", Variant.Relation.hasValue())) {
       if (!IO.outputting())
         Variant.Relation.emplace();
       MappingTraits<Relation>::mapping(IO, *Variant.Relation);
+    } else if (IO.mapTag("!Source", Variant.Source.hasValue())) {
+      if (!IO.outputting())
+        Variant.Source.emplace();
+      MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
+    } else if (IO.mapTag("!Cmd", Variant.Cmd.hasValue())) {
+      if (!IO.outputting())
+        Variant.Cmd.emplace();
+      MappingTraits<CompileCommandYAML>::mapping(
+          IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
     }
   }
 };
 
 } // namespace yaml
 } // namespace llvm
 
 namespace clang {
for (auto &Sym : *O.Refs) {
       Yout << Entry;
     }
   if (O.Relations)
     for (auto &R : *O.Relations) {
       VariantEntry Entry;
       Entry.Relation = R;
       Yout << Entry;
     }
+  if (O.Sources) {
+    for (const auto &Source : *O.Sources) {
+      VariantEntry Entry;
+      Entry.Source = Source.getValue();
+      Yout << Entry;
+    }
+  }
+  if (O.Cmd) {
+    VariantEntry Entry;
+    Entry.Cmd = *O.Cmd;
+    Yout << Entry;
+  }
 }
 
 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
   SymbolSlab::Builder Symbols;
   RefSlab::Builder Refs;
   RelationSlab::Builder Relations;
   llvm::BumpPtrAllocator
       Arena; // store the underlying data of Position::FileURI.
   llvm::UniqueStringSaver Strings(Arena);
   llvm::yaml::Input Yin(Data, &Strings);
+  IncludeGraph Sources;
+  llvm::Optional<tooling::CompileCommand> Cmd;
   while (Yin.setCurrentDocument()) {
     llvm::yaml::EmptyContext Ctx;
     VariantEntry Variant;
     yamlize(Yin, Variant, true, Ctx);
     if (Yin.error())
       return llvm::errorCodeToError(Yin.error());
 
     if (Variant.Symbol)
       Symbols.insert(*Variant.Symbol);
     if (Variant.Refs)
       for (const auto &Ref : Variant.Refs->second)
         Refs.insert(Variant.Refs->first, Ref);
     if (Variant.Relation)
       Relations.insert(*Variant.Relation);
+    if (Variant.Source) {
+      auto &IGN = Variant.Source.getValue();
+      auto Entry = Sources.try_emplace(IGN.URI).first;
+      Entry->getValue() = std::move(IGN);
+      // Fixup refs to refer to map keys which will live on
+      Entry->getValue().URI = Entry->getKey();
+      for (auto &Include : Entry->getValue().DirectIncludes)
+        Include = Sources.try_emplace(Include).first->getKey();
+    }
+    if (Variant.Cmd)
+      Cmd = *Variant.Cmd;
     Yin.nextDocument();
   }
 
   IndexFileIn Result;
   Result.Symbols.emplace(std::move(Symbols).build());
   Result.Refs.emplace(std::move(Refs).build());
   Result.Relations.emplace(std::move(Relations).build());
+  if (Sources.size())
+    Result.Sources = std::move(Sources);
+  Result.Cmd = std::move(Cmd);
   return std::move(Result);
 }
 
 std::string toYAML(const Symbol &S) {
   std::string Buf;
   {
     llvm::raw_string_ostream OS(Buf);
     llvm::yaml::Output Yout(OS);

clang-tools-extra/clangd/unittests/SerializationTests.cpp

 #include "index/Serialization.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 using ::testing::_;
 using ::testing::AllOf;
+using ::testing::ElementsAre;
 using ::testing::Pair;
 using ::testing::UnorderedElementsAre;
 using ::testing::UnorderedElementsAreArray;
 
 namespace clang {
 namespace clangd {
 namespace {
 

 ...
 --- !Relations
 Subject:
   ID:              6481EE7AF2841756
 Predicate:       0
 Object:
   ID:              6512AEC512EA3A2D
 ...
+--- !Cmd
+Directory:       'testdir'
+CommandLine:
+  - 'cmd1'
+  - 'cmd2'
+...
+--- !Source
+URI:             'file:///path/source1.cpp'
+Flags:           1
+Digest:          EED8F5EAF25C453C
+DirectIncludes:
+  - 'file:///path/inc1.h'
+  - 'file:///path/inc2.h'
+...
 )";
 
 MATCHER_P(ID, I, "") { return arg.ID == cantFail(SymbolID::fromStr(I)); }
 MATCHER_P(QName, Name, "") { return (arg.Scope + arg.Name).str() == Name; }
 MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") {
   return (arg.IncludeHeader == IncludeHeader) && (arg.References == References);
 }
 
TEST(SerializationTest, YAMLConversions) {
   EXPECT_EQ(StringRef(Ref1.Location.FileURI), "file:///path/foo.cc");
 
   SymbolID Base = cantFail(SymbolID::fromStr("6481EE7AF2841756"));
   SymbolID Derived = cantFail(SymbolID::fromStr("6512AEC512EA3A2D"));
   ASSERT_TRUE(bool(ParsedYAML->Relations));
   EXPECT_THAT(
       *ParsedYAML->Relations,
       UnorderedElementsAre(Relation{Base, RelationKind::BaseOf, Derived}));
+
+  ASSERT_TRUE(bool(ParsedYAML->Cmd));
+  auto &Cmd = *ParsedYAML->Cmd;
+  ASSERT_EQ(Cmd.Directory, "testdir");
+  EXPECT_THAT(Cmd.CommandLine, ElementsAre("cmd1", "cmd2"));
+
+  ASSERT_TRUE(bool(ParsedYAML->Sources));
+  const auto *URI = "file:///path/source1.cpp";
+  ASSERT_TRUE(ParsedYAML->Sources->count(URI));
+  auto IGNDeserialized = ParsedYAML->Sources->lookup(URI);
+  EXPECT_EQ(llvm::toHex(IGNDeserialized.Digest), "EED8F5EAF25C453C");
+  EXPECT_THAT(IGNDeserialized.DirectIncludes,
+              ElementsAre("file:///path/inc1.h", "file:///path/inc2.h"));
+  EXPECT_EQ(IGNDeserialized.URI, URI);
+  EXPECT_EQ(IGNDeserialized.Flags, IncludeGraphNode::SourceFlag(1));
 }
 
 std::vector<std::string> YAMLFromSymbols(const SymbolSlab &Slab) {
   std::vector<std::string> Result;
   for (const auto &Sym : Slab)
     Result.push_back(toYAML(Sym));
   return Result;
 }