YAML output for index-while-building

[yvvan]

Summary

Should be compatible with the current index format accepted by clangd

Depends on https://reviews.llvm.org/D49009

[ilya-biryukov]

JSON is a subset of YAML, do we really need both outputs? I've noticed the schema for the output is actually different between YAML and JSON, wouldn't that be confusing?

Should be compatible with the current index format accepted by clangd

Do we really want that?
Note that we were not planning to keep the YAML index in clangd long-term, it was merely a convenient serialization routine available in LLVM land, so our implementation uses it for now. I would discourage to build tools that rely on the YAML format staying the same.

Maybe we could teach clangd to consume the index-while-build output (in binary format) when building the index instead? WDYT?

[yvvan]

@ilya-biryukov I would name this revision "for test purpose". It works at some extent with the current clangd version and can be used by someone else. But since we had a discussion about YAML being a temporary solution I never planned to commit this revision.
So yes, I totally agree that using a binary format would be ideal for the "index while build' use case.

include/clang/Index/IndexRecordReader.h

 namespace index {
 
 struct IndexRecordDecl {
   unsigned DeclID;
   SymbolInfo SymInfo;
   SymbolRoleSet Roles;
   SymbolRoleSet RelatedRoles;
   StringRef Name;
+  StringRef Scope;
   StringRef USR;
   StringRef CodeGenName;
 };
 
 struct IndexRecordRelation {
   SymbolRoleSet Roles;
   const IndexRecordDecl *Dcl = nullptr;
 
   IndexRecordRelation() = default;
   IndexRecordRelation(SymbolRoleSet Roles, const IndexRecordDecl *Dcl)
       : Roles(Roles), Dcl(Dcl) {}
 };
 
 struct IndexRecordOccurrence {
   const IndexRecordDecl *Dcl;
   SmallVector<IndexRecordRelation, 4> Relations;
   SymbolRoleSet Roles;
   unsigned Line;
   unsigned Column;
+  unsigned EndLine;
+  unsigned EndColumn;
 };
 
 class IndexRecordReader {
   IndexRecordReader();
 
 public:
   static std::unique_ptr<IndexRecordReader>
   createWithRecordFilename(StringRef RecordFilename, StringRef StorePath,

include/clang/Index/IndexRecordWriter.h

 /// except for roles, which are synthesized by looking at all the occurrences.
 ///
 /// \seealso IndexRecordDecl
 /// \note this struct is generally accompanied by a buffer that owns the string
 /// storage.  It should not be stored permanently.
 struct Symbol {
   SymbolInfo SymInfo;
   StringRef Name;
+  StringRef Scope;
   StringRef USR;
   StringRef CodeGenName;
 };
 
 /// A relation to an opaque symbol.
 /// \seealso IndexRecordRelation
 struct SymbolRelation {
   OpaqueDecl RelatedSymbol;
public:
   /// any necessary storage.
   ///
   /// \return Success, or Failure and sets \p Error.
   Result endRecord(std::string &Error,
                    writer::SymbolWriterCallback GetSymbolForDecl);
 
   /// Add an occurrence of the symbol \p D with the given \p Roles and location.
   void addOccurrence(writer::OpaqueDecl D, SymbolRoleSet Roles, unsigned Line,
-                     unsigned Column, ArrayRef<writer::SymbolRelation> Related);
+                     unsigned Column, unsigned EndLine, unsigned EndColumn,
+                     ArrayRef<writer::SymbolRelation> Related);
 };
 
 } // end namespace index
 } // end namespace clang
 
 #endif // LLVM_CLANG_INDEX_INDEXRECORDWRITER_H

include/indexstore/IndexStoreCXX.h

public:
   uint64_t getProperties() { return indexstore_symbol_get_properties(obj); }
   uint64_t getRoles() { return indexstore_symbol_get_roles(obj); }
   uint64_t getRelatedRoles() {
     return indexstore_symbol_get_related_roles(obj);
   }
   StringRef getName() {
     return stringFromIndexStoreStringRef(indexstore_symbol_get_name(obj));
   }
+  StringRef getScope() {
+    return stringFromIndexStoreStringRef(indexstore_symbol_get_scope(obj));
+  }
   StringRef getUSR() {
     return stringFromIndexStoreStringRef(indexstore_symbol_get_usr(obj));
   }
   StringRef getCodegenName() {
     return stringFromIndexStoreStringRef(
         indexstore_symbol_get_codegen_name(obj));
   }
 };
#endif
         });
   }
 
   std::pair<unsigned, unsigned> getLineCol() {
     unsigned line, col;
     indexstore_occurrence_get_line_col(obj, &line, &col);
     return std::make_pair(line, col);
   }
+
+  std::pair<unsigned, unsigned> getEndLineCol() {
+    unsigned start, end;
+    indexstore_occurrence_get_end_line_col(obj, &start, &end);
+    return std::make_pair(start, end);
+  }
 };
 
 class IndexStore;
 typedef std::shared_ptr<IndexStore> IndexStoreRef;
 
 class IndexStore {
   indexstore_t obj;
   friend class IndexRecordReader;

include/indexstore/indexstore.h

 
 INDEXSTORE_PUBLIC uint64_t
 indexstore_symbol_get_related_roles(indexstore_symbol_t symbol);
 
 INDEXSTORE_PUBLIC indexstore_string_ref_t
 indexstore_symbol_get_name(indexstore_symbol_t symbol);
 
 INDEXSTORE_PUBLIC indexstore_string_ref_t
+indexstore_symbol_get_scope(indexstore_symbol_t symbol);
+
+INDEXSTORE_PUBLIC indexstore_string_ref_t
 indexstore_symbol_get_usr(indexstore_symbol_t symbol);
 
 INDEXSTORE_PUBLIC indexstore_string_ref_t
 indexstore_symbol_get_codegen_name(indexstore_symbol_t symbol);
 
 typedef void *indexstore_symbol_relation_t;
 
 INDEXSTORE_PUBLIC uint64_t

 
 INDEXSTORE_PUBLIC uint64_t
 indexstore_occurrence_get_roles(indexstore_occurrence_t occur);
 
 INDEXSTORE_PUBLIC void
 indexstore_occurrence_get_line_col(indexstore_occurrence_t occur,
                                    unsigned *line, unsigned *column);
 
+INDEXSTORE_PUBLIC void
+indexstore_occurrence_get_end_line_col(indexstore_occurrence_t occur,
+                                       unsigned *end_line, unsigned *end_column);
+
 typedef void *indexstore_record_reader_t;
 
 INDEXSTORE_PUBLIC indexstore_record_reader_t indexstore_record_reader_create(
     indexstore_t store, const char *record_name, indexstore_error_t *error);
 
 INDEXSTORE_PUBLIC void
 indexstore_record_reader_dispose(indexstore_record_reader_t reader);
 

lib/Index/ClangIndexRecordWriter.cpp

 //===--- ClangIndexRecordWriter.cpp - Index record serialization ----------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangIndexRecordWriter.h"
 #include "FileIndexData.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/QualTypeNames.h"
 #include "clang/Index/IndexSymbol.h"
 #include "clang/Index/USRGeneration.h"
+#include "clang/Lex/Lexer.h"
 
 using namespace clang;
 using namespace clang::index;
 
 StringRef ClangIndexRecordWriter::getUSR(const Decl *D) {
   assert(D->isCanonicalDecl());
   auto Insert = USRByDecl.insert(std::make_pair(D, StringRef()));
   if (Insert.second) {
bool ClangIndexRecordWriter::writeRecord(StringRef Filename, FileID FID,
   for (auto &Occur : Occurs) {
     unsigned Line, Col;
     std::tie(Line, Col) = getLineCol(Occur.Offset);
     SmallVector<writer::SymbolRelation, 3> Related;
     Related.reserve(Occur.Relations.size());
     for (auto &Rel : Occur.Relations)
       Related.push_back(writer::SymbolRelation{Rel.RelatedSymbol, Rel.Roles});
 
-    Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col, Related);
+    SourceLocation Loc = SM.getComposedLoc(FID, Occur.Offset);
+    unsigned TokenLen = Lexer::MeasureTokenLength(Loc, SM, Ctx.getLangOpts());
+    unsigned EndLine, EndCol;
+    std::tie(EndLine, EndCol) = getLineCol(Occur.Offset + TokenLen);
+
+    Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col,
+                       EndLine, EndCol, Related);
   }
 
   PrintingPolicy Policy(Ctx.getLangOpts());
   Policy.SuppressTemplateArgsInCXXConstructors = true;
 
   auto Result = Impl.endRecord(
       Error, [&](writer::OpaqueDecl OD, SmallVectorImpl<char> &Scratch) {
         const Decl *D = static_cast<const Decl *>(OD);
         auto Info = getSymbolInfo(D);
 
         writer::Symbol Sym;
         Sym.SymInfo = Info;
 
         auto *ND = dyn_cast<NamedDecl>(D);
+        unsigned NameLen = 0;
+        unsigned QualLength = 0;
         if (ND) {
           llvm::raw_svector_ostream OS(Scratch);
           DeclarationName DeclName = ND->getDeclName();
           if (!DeclName.isEmpty())
             DeclName.print(OS, Policy);
+          NameLen = Scratch.size();
+
+          ND->printQualifiedName(OS);
+          QualLength = Scratch.size();
         }
-        unsigned NameLen = Scratch.size();
+
         Sym.Name = StringRef(Scratch.data(), NameLen);
+        Sym.Scope = StringRef(Scratch.data() + NameLen, QualLength - 2 * NameLen);
 
         Sym.USR = getUSR(D);
         assert(!Sym.USR.empty() && "Recorded decl without USR!");
 
         if (CGNameGen && ND) {
           llvm::raw_svector_ostream OS(Scratch);
           CGNameGen->writeName(ND, OS);
         }
-        unsigned CGNameLen = Scratch.size() - NameLen;
-        Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen);
+        unsigned CGNameLen = Scratch.size() - QualLength;
+        Sym.CodeGenName = StringRef(Scratch.data() + QualLength, CGNameLen);
         return Sym;
       });
 
   switch (Result) {
   case IndexRecordWriter::Result::Success:
   case IndexRecordWriter::Result::AlreadyExists:
     return false;
   case IndexRecordWriter::Result::Failure:
     return true;
   }
 }

lib/Index/IndexRecordReader.cpp

void readDecl(unsigned Index, IndexRecordDecl &RecD) {
     RecD.SymInfo.SubKind =
         getSymbolSubKind((indexstore_symbol_subkind_t)read(Record, I));
     RecD.SymInfo.Lang =
         getSymbolLanguage((indexstore_symbol_language_t)read(Record, I));
     RecD.SymInfo.Properties = getSymbolProperties(read(Record, I));
     RecD.Roles = getSymbolRoles(read(Record, I));
     RecD.RelatedRoles = getSymbolRoles(read(Record, I));
     size_t NameLen = read(Record, I);
+    size_t ScopeLen = read(Record, I);
     size_t USRLen = read(Record, I);
     RecD.Name = Blob.substr(0, NameLen);
-    RecD.USR = Blob.substr(NameLen, USRLen);
-    RecD.CodeGenName = Blob.substr(NameLen + USRLen);
+    RecD.Scope = Blob.substr(NameLen, ScopeLen);
+    RecD.USR = Blob.substr(NameLen + ScopeLen, USRLen);
+    RecD.CodeGenName = Blob.substr(NameLen + ScopeLen + USRLen);
   }
 
   /// Reads occurrence data.
   /// \param DeclsFilter if non-empty indicates the list of decls that we want
   /// to get occurrences for. If empty then indicates that we want occurrences
   /// for all decls.
   /// \param RelatedDeclsFilter Same as \c DeclsFilter but for related decls.
   /// \returns true if the occurrence info was filled out, false if occurrence
if (!RelatedDeclsFilter.empty()) {
       if (!FoundRelated)
         return false;
     }
 
     RecOccur.Dcl = getDeclByID(DeclID);
     RecOccur.Roles = getSymbolRoles(read(Record, I));
     RecOccur.Line = read(Record, I);
     RecOccur.Column = read(Record, I);
+    RecOccur.EndLine = read(Record, I);
+    RecOccur.EndColumn = read(Record, I);
 
     unsigned NumRelated = read(Record, I);
     while (NumRelated--) {
       SymbolRoleSet RelRoles = getSymbolRoles(read(Record, I));
       const IndexRecordDecl *RelD = getDeclByID(read(Record, I));
       RecOccur.Relations.emplace_back(RelRoles, RelD);
     }
 

lib/Index/IndexRecordWriter.cpp

 };
 
 struct OccurrenceInfo {
   unsigned DeclID;
   OpaqueDecl D;
   SymbolRoleSet Roles;
   unsigned Line;
   unsigned Column;
+  unsigned EndLine;
+  unsigned EndColumn;
   SmallVector<std::pair<writer::SymbolRelation, unsigned>, 4> Related;
 };
 } // end anonymous namespace
 
 static void writeBlockInfo(BitstreamWriter &Stream) {
   RecordData Record;
 
   Stream.EnterBlockInfoBlock();
for (auto &Info : Decls) {
     Blob.clear();
     Scratch.clear();
 
     writer::Symbol SymInfo = GetSymbolForDecl(Info.D, Scratch);
     assert(SymInfo.SymInfo.Kind != SymbolKind::Unknown);
     assert(!SymInfo.USR.empty() && "Recorded decl without USR!");
 
     Blob += SymInfo.Name;
+    Blob += SymInfo.Scope;
     Blob += SymInfo.USR;
     Blob += SymInfo.CodeGenName;
 
 #ifndef NDEBUG
     bool IsNew = USRSet.insert(SymInfo.USR).second;
     if (!IsNew) {
       llvm::errs() << "Index: Duplicate USR! " << SymInfo.USR << "\n";
       // FIXME: print more information so it's easier to find the declaration.
     }
 #endif
 
     Record.clear();
     Record.push_back(REC_DECLINFO);
     Record.push_back(getIndexStoreKind(SymInfo.SymInfo.Kind));
     Record.push_back(getIndexStoreSubKind(SymInfo.SymInfo.SubKind));
     Record.push_back(getIndexStoreLang(SymInfo.SymInfo.Lang));
     Record.push_back(getIndexStoreProperties(SymInfo.SymInfo.Properties));
     Record.push_back(getIndexStoreRoles(Info.Roles));
     Record.push_back(getIndexStoreRoles(Info.RelatedRoles));
     Record.push_back(SymInfo.Name.size());
+    Record.push_back(SymInfo.Scope.size());
     Record.push_back(SymInfo.USR.size());
     Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob);
   }
 
   Stream.ExitBlock();
 
   //===--------------------------------------------------------------------===//
   // DECLOFFSETS_BLOCK_ID
#endif
 
   for (auto &Occur : Occurrences) {
     Record.clear();
     Record.push_back(REC_DECLOCCURRENCE);
     Record.push_back(Occur.DeclID);
     Record.push_back(getIndexStoreRoles(Occur.Roles));
     Record.push_back(Occur.Line);
     Record.push_back(Occur.Column);
+    Record.push_back(Occur.EndLine);
+    Record.push_back(Occur.EndColumn);
     Record.push_back(Occur.Related.size());
     for (auto &Rel : Occur.Related) {
       Record.push_back(getIndexStoreRoles(Rel.first.Roles));
       Record.push_back(Rel.second);
     }
     Stream.EmitRecordWithAbbrev(AbbrevCode, Record);
   }
   Stream.ExitBlock();
if (std::error_code EC =
     return Result::Failure;
   }
 
   return Result::Success;
 }
 
 void IndexRecordWriter::addOccurrence(
     OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, unsigned Column,
+    unsigned EndLine, unsigned EndColumn,
     ArrayRef<writer::SymbolRelation> Related) {
   assert(Record && "called addOccurrence without calling beginRecord");
   auto &State = *Record;
 
   auto insertDecl = [&](OpaqueDecl D, SymbolRoleSet Roles,
                         SymbolRoleSet RelatedRoles) -> unsigned {
     auto Insert =
         State.IndexForDecl.insert(std::make_pair(D, State.Decls.size()));
void IndexRecordWriter::addOccurrence(
   decltype(OccurrenceInfo::Related) RelatedDecls;
   RelatedDecls.reserve(Related.size());
   for (auto &Rel : Related) {
     unsigned ID = insertDecl(Rel.RelatedSymbol, SymbolRoleSet(), Rel.Roles);
     RelatedDecls.emplace_back(Rel, ID);
   }
 
   State.Occurrences.push_back(
-      OccurrenceInfo{DeclID, D, Roles, Line, Column, std::move(RelatedDecls)});
+      OccurrenceInfo{DeclID, D, Roles, Line, Column, EndLine, EndColumn, std::move(RelatedDecls)});
 }

tools/IndexStore/IndexStore.cpp

uint64_t indexstore_symbol_get_related_roles(indexstore_symbol_t sym) {
   return getIndexStoreRoles(static_cast<IndexRecordDecl *>(sym)->RelatedRoles);
 }
 
 indexstore_string_ref_t indexstore_symbol_get_name(indexstore_symbol_t sym) {
   auto *D = static_cast<IndexRecordDecl *>(sym);
   return toIndexStoreString(D->Name);
 }
 
+indexstore_string_ref_t indexstore_symbol_get_scope(indexstore_symbol_t sym) {
+  auto *D = static_cast<IndexRecordDecl *>(sym);
+  return toIndexStoreString(D->Scope);
+}
+
 indexstore_string_ref_t indexstore_symbol_get_usr(indexstore_symbol_t sym) {
   auto *D = static_cast<IndexRecordDecl *>(sym);
   return toIndexStoreString(D->USR);
 }
 
 indexstore_string_ref_t
 indexstore_symbol_get_codegen_name(indexstore_symbol_t sym) {
   auto *D = static_cast<IndexRecordDecl *>(sym);
void indexstore_occurrence_get_line_col(indexstore_occurrence_t occur,
                                         unsigned *line, unsigned *column) {
   auto *recOccur = static_cast<IndexRecordOccurrence *>(occur);
   if (line)
     *line = recOccur->Line;
   if (column)
     *column = recOccur->Column;
 }
 
+void indexstore_occurrence_get_end_line_col(indexstore_occurrence_t occur,
+                                            unsigned *end_line, unsigned *end_column) {
+  auto *recOccur = static_cast<IndexRecordOccurrence *>(occur);
+  if (end_line)
+    *end_line = recOccur->EndLine;
+  if (end_column)
+    *end_column = recOccur->EndColumn;
+}
+
 typedef void *indexstore_record_reader_t;
 
 indexstore_record_reader_t
 indexstore_record_reader_create(indexstore_t c_store, const char *record_name,
                                 indexstore_error_t *c_error) {
   IndexDataStore *store = static_cast<IndexDataStore *>(c_store);
   std::unique_ptr<IndexRecordReader> reader;
   std::string error;

tools/c-index-test/JSONAggregation.h

 #include "clang/Basic/LLVM.h"
 
 namespace clang {
 namespace index {
 
 /// Returns true if an error occurred, false otherwise.
 bool aggregateDataAsJSON(StringRef StorePath, raw_ostream &OS);
 
+/// Returns true if an error occurred, false otherwise.
+bool aggregateDataAsYAML(StringRef StorePath, raw_ostream &OS);
+
 } // end namespace index
 } // end namespace clang
 
 #endif

tools/c-index-test/JSONAggregation.cpp

 //===--- JSONAggregation.cpp - Index data aggregation in JSON format ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "JSONAggregation.h"
 #include "indexstore/IndexStoreCXX.h"
 #include "clang/Index/IndexDataStoreSymbolUtils.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/SHA1.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <regex>
+#include <set>
+
 using namespace clang;
 using namespace clang::index;
 using namespace indexstore;
 using namespace llvm;
 
 namespace {
 
 typedef size_t FilePathIndex;
struct UnitInfo {
   StringRef Triple;
 };
 
 struct SymbolInfo {
   SymbolKind Kind;
   SymbolLanguage Lang;
   StringRef USR;
   StringRef Name;
+  StringRef Scope;
   StringRef CodegenName;
   SymbolRoleSet Roles = 0;
   SymbolRoleSet RelatedRoles = 0;
 };
 
 struct SymbolRelationInfo {
   SymbolIndex RelatedSymbol;
   SymbolRoleSet Roles;
   SymbolRelationInfo(SymbolIndex relSymbol, SymbolRoleSet roles)
       : RelatedSymbol(relSymbol), Roles(roles) {}
 };
 
 struct SymbolOccurrenceInfo {
   SymbolIndex Symbol;
   SymbolRoleSet Roles = 0;
   std::vector<SymbolRelationInfo> Relations;
   unsigned Line;
   unsigned Column;
+  unsigned EndLine;
+  unsigned EndColumn;
 };
 
 struct RecordInfo {
   SmallVector<SymbolOccurrenceInfo, 8> Occurrences;
 };
 
 class Aggregator {
   IndexStore Store;

 public:
   explicit Aggregator(IndexStore store)
       : Store(std::move(store)), FilePathIndices(Allocator), Triples(Allocator),
         RecordIndices(Allocator), SymbolIndices(Allocator) {}
 
   bool process();
   void processUnit(StringRef name, IndexUnitReader &UnitReader);
   void dumpJSON(raw_ostream &OS);
+  void dumpYAML(raw_ostream &OS);
 
 private:
   StringRef copyStr(StringRef str) {
     if (str.empty())
       return StringRef();
     char *buf = Allocator.Allocate<char>(str.size());
     std::copy(str.begin(), str.end(), buf);
     return StringRef(buf, str.size());
idxOccur.foreachRelation([&](IndexSymbolRelation rel) -> bool {
       SymbolIndex relsymIdx = getSymbolIndex(rel.getSymbol());
       SymbolInfo &relsymInfo = Symbols[relsymIdx];
       relsymInfo.RelatedRoles |= rel.getRoles();
       occurInfo.Relations.emplace_back(relsymIdx, rel.getRoles());
       return true;
     });
     occurInfo.Roles = idxOccur.getRoles();
     std::tie(occurInfo.Line, occurInfo.Column) = idxOccur.getLineCol();
+    std::tie(occurInfo.EndLine, occurInfo.EndColumn) = idxOccur.getEndLineCol();
     record->Occurrences.push_back(std::move(occurInfo));
     return true;
   });
   return record;
 }
 
 SymbolIndex Aggregator::getSymbolIndex(IndexRecordSymbol sym) {
   auto pair =
       SymbolIndices.insert(std::make_pair(sym.getUSR(), Symbols.size()));
   bool wasInserted = pair.second;
   if (wasInserted) {
     SymbolInfo symInfo;
     symInfo.Kind = getSymbolKind(sym.getKind());
     symInfo.Lang = getSymbolLanguage(sym.getLanguage());
     symInfo.USR = pair.first->first();
     symInfo.Name = copyStr(sym.getName());
+    symInfo.Scope = copyStr(sym.getScope());
     symInfo.CodegenName = copyStr(sym.getCodegenName());
     Symbols.push_back(std::move(symInfo));
   }
   return pair.first->second;
 }
 
 void Aggregator::dumpJSON(raw_ostream &OS) {
   OS << "{\n";
for (unsigned i = 0, e = Units.size(); i != e; ++i) {
     if (i < e - 1)
       OS << ',';
     OS << '\n';
   }
   OS.indent(2) << "]\n";
   OS << "}\n";
 }
 
+static StringRef getSymbolKindForYAML(SymbolKind K) {
+  switch (K) {
+  case SymbolKind::Unknown: return "<unknown>";
+  case SymbolKind::Module: return "Module";
+  case SymbolKind::Namespace: return "Namespace";
+  case SymbolKind::NamespaceAlias: return "NamespaceAlias";
+  case SymbolKind::Macro: return "Macro";
+  case SymbolKind::Enum: return "Enum";
+  case SymbolKind::Struct: return "Struct";
+  case SymbolKind::Class: return "Class";
+  case SymbolKind::Protocol: return "Protocol";
+  case SymbolKind::Extension: return "Extension";
+  case SymbolKind::Union: return "Union";
+  case SymbolKind::TypeAlias: return "TypeAlias";
+  case SymbolKind::Function: return "Function";
+  case SymbolKind::Variable: return "Variable";
+  case SymbolKind::Field: return "Field";
+  case SymbolKind::EnumConstant: return "EnumConstant";
+  case SymbolKind::InstanceMethod: return "InstanceMethod";
+  case SymbolKind::ClassMethod: return "ClassMethod";
+  case SymbolKind::StaticMethod: return "StaticMethod";
+  case SymbolKind::InstanceProperty: return "InstanceProperty";
+  case SymbolKind::ClassProperty: return "ClassProperty";
+  case SymbolKind::StaticProperty: return "StaticProperty";
+  case SymbolKind::Constructor: return "Constructor";
+  case SymbolKind::Destructor: return "Destructor";
+  case SymbolKind::ConversionFunction: return "ConversionFunction";
+  case SymbolKind::Parameter: return "Parameter";
+  case SymbolKind::Using: return "Using";
+  }
+  llvm_unreachable("invalid symbol kind");
+}
+
+void Aggregator::dumpYAML(raw_ostream &OS) {
+  std::set<size_t> files;
+  for (unsigned i = 0, e = Units.size(); i != e; ++i) {
+    UnitInfo &unit = *Units[i];
+    for (unsigned si = 0, se = unit.Sources.size(); si != se; ++si) {
+      UnitSourceInfo &source = unit.Sources[si];
+      if (source.FilePath >= FilePaths.size())
+        continue;
+      if (files.find(source.FilePath) == files.end())
+          files.insert(source.FilePath);
+      else
+          continue;
+
+      StringRef filename = FilePaths[source.FilePath];
+      if (filename.empty())
+        continue;
+
+      for (unsigned ri = 0, re = source.AssociatedRecords.size(); ri != re;
+           ++ri) {
+        size_t RecordNumber = source.AssociatedRecords[ri];
+        if (RecordNumber >= Records.size())
+          continue;
+        RecordInfo &recInfo = *Records[source.AssociatedRecords[ri]];
+        for (unsigned oi = 0, oe = recInfo.Occurrences.size(); oi != oe; ++oi) {
+          SymbolOccurrenceInfo &occurInfo = recInfo.Occurrences[oi];
+          if (occurInfo.Symbol >= Symbols.size())
+            continue;
+          StringRef role;
+          if (occurInfo.Roles & (unsigned)SymbolRole::Declaration)
+            role = "CanonicalDeclaration:\n";
+          else if (occurInfo.Roles & (unsigned)SymbolRole::Definition)
+            role = "Definition:\n";
+          else
+            continue;
+          SymbolInfo &symInfo = Symbols[occurInfo.Symbol];
+          OS << "---\n";
+          OS << "ID: "
+             << toHex(
+                    toStringRef(SHA1::hash(arrayRefFromStringRef(symInfo.USR))))
+             << "\n";
+          OS << "Name : '" << symInfo.Name << "'\n";
+          OS << "Scope: '" << symInfo.Scope << "'\n";
+          OS << "SymInfo:\n";
+          OS.indent(2) << "Kind: " << getSymbolKindForYAML(symInfo.Kind)
+                       << "\n";
+          OS.indent(2) << "Lang: Cpp\n";
+          OS << role;
+
+          OS.indent(2) << "FileURI: file://" << (filename[0] != '/' ? "/" : "")
+                       << filename << "\n";
+          OS.indent(2) << "Start:\n";
+          OS.indent(4) << "Line: " << occurInfo.Line << "\n";
+          OS.indent(4) << "Column: " << occurInfo.Column << "\n";
+          OS.indent(2) << "End:\n";
+          OS.indent(4) << "Line: " << occurInfo.EndLine << "\n";
+          OS.indent(4) << "Column: " << occurInfo.EndColumn << "\n";
+          OS << "...\n";
+        }
+      }
+    }
+  }
+}
+
 bool index::aggregateDataAsJSON(StringRef StorePath, raw_ostream &OS) {
   std::string error;
   auto dataStore = IndexStore(StorePath, error);
   if (!dataStore) {
     errs() << "error opening store path '" << StorePath << "': " << error
            << '\n';
     return true;
   }
 
   // Explicitely avoid doing any memory cleanup for aggregator since the process
   // is going to exit when we are done.
   Aggregator *aggregator = new Aggregator(std::move(dataStore));
   bool err = aggregator->process();
   if (err)
     return true;
   aggregator->dumpJSON(OS);
   return false;
 }
+
+bool index::aggregateDataAsYAML(StringRef StorePath, raw_ostream &OS) {
+    std::string error;
+    auto dataStore = IndexStore(StorePath, error);
+    if (!dataStore) {
+      errs() << "error opening store path '" << StorePath << "': " << error
+             << '\n';
+      return true;
+    }
+
+    // Explicitely avoid doing any memory cleanup for aggregator since the process
+    // is going to exit when we are done.
+    Aggregator *aggregator = new Aggregator(std::move(dataStore));
+    bool err = aggregator->process();
+    if (err)
+      return true;
+    aggregator->dumpYAML(OS);
+    return false;
+}

tools/c-index-test/core_main.cpp

 enum class ActionType {
   None,
   PrintSourceSymbols,
   PrintSourceUnit,
   PrintRecord,
   PrintUnit,
   PrintStoreFormatVersion,
   AggregateAsJSON,
+  AggregateAsYAML,
   WatchDir,
 };
 
 namespace options {
 
 static cl::OptionCategory IndexTestCoreCategory("index-test-core options");
 
 static cl::opt<ActionType> Action(
     cl::desc("Action:"), cl::init(ActionType::None),
     cl::values(
         clEnumValN(ActionType::PrintSourceSymbols, "print-source-symbols",
                    "Print symbols from source"),
         clEnumValN(ActionType::PrintSourceUnit, "print-source-unit",
                    "Print unit info from source"),
         clEnumValN(ActionType::PrintRecord, "print-record",
                    "Print record file info"),
         clEnumValN(ActionType::PrintUnit, "print-unit", "Print unit file info"),
         clEnumValN(ActionType::PrintStoreFormatVersion,
                    "print-store-format-version", "Print store format version"),
         clEnumValN(ActionType::AggregateAsJSON, "aggregate-json",
                    "Aggregate index data in JSON format"),
+        clEnumValN(ActionType::AggregateAsYAML, "aggregate-yaml",
+                   "Aggregate index data in YAML format"),
         clEnumValN(ActionType::WatchDir, "watch-dir",
                    "Watch directory for file events")),
     cl::cat(IndexTestCoreCategory));
 
 static cl::opt<std::string> OutputFile("o", cl::desc("output file"),
                                        cl::cat(IndexTestCoreCategory));
 
 static cl::list<std::string> InputFiles(cl::Positional,
if (options::Action == ActionType::PrintUnit) {
     else
       return printUnit(options::InputFiles[0], outs());
   }
 
   if (options::Action == ActionType::PrintStoreFormatVersion) {
     outs() << indexstore::IndexStore::formatVersion() << '\n';
   }
 
-  if (options::Action == ActionType::AggregateAsJSON) {
+  if (options::Action == ActionType::AggregateAsJSON ||
+      options::Action == ActionType::AggregateAsYAML) {
     if (options::InputFiles.empty()) {
       errs() << "error: missing input data store directory\n";
       return 1;
     }
     StringRef storePath = options::InputFiles[0];
     if (options::OutputFile.empty())
       return aggregateDataAsJSON(storePath, outs());
     std::error_code EC;
     raw_fd_ostream OS(options::OutputFile, EC, llvm::sys::fs::F_None);
     if (EC) {
       errs() << "failed to open output file: " << EC.message() << '\n';
       return 1;
     }
+    if (options::Action == ActionType::AggregateAsJSON)
       return aggregateDataAsJSON(storePath, OS);
+    else
+      return aggregateDataAsYAML(storePath, OS);
   }
 
   if (options::Action == ActionType::WatchDir) {
     if (options::InputFiles.empty()) {
       errs() << "error: missing directory path\n";
       return 1;
     }
     return watchDirectory(options::InputFiles[0]);
   }
 
   return 0;
 }