Index: include/clang/Index/IndexRecordReader.h =================================================================== --- include/clang/Index/IndexRecordReader.h +++ include/clang/Index/IndexRecordReader.h @@ -28,6 +28,7 @@ SymbolRoleSet Roles; SymbolRoleSet RelatedRoles; StringRef Name; + StringRef Scope; StringRef USR; StringRef CodeGenName; }; @@ -47,6 +48,8 @@ SymbolRoleSet Roles; unsigned Line; unsigned Column; + unsigned EndLine; + unsigned EndColumn; }; class IndexRecordReader { Index: include/clang/Index/IndexRecordWriter.h =================================================================== --- include/clang/Index/IndexRecordWriter.h +++ include/clang/Index/IndexRecordWriter.h @@ -33,6 +33,7 @@ struct Symbol { SymbolInfo SymInfo; StringRef Name; + StringRef Scope; StringRef USR; StringRef CodeGenName; }; @@ -99,7 +100,8 @@ /// Add an occurrence of the symbol \p D with the given \p Roles and location. void addOccurrence(writer::OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, - unsigned Column, ArrayRef Related); + unsigned Column, unsigned EndLine, unsigned EndColumn, + ArrayRef Related); }; } // end namespace index Index: include/indexstore/IndexStoreCXX.h =================================================================== --- include/indexstore/IndexStoreCXX.h +++ include/indexstore/IndexStoreCXX.h @@ -53,6 +53,9 @@ StringRef getName() { return stringFromIndexStoreStringRef(indexstore_symbol_get_name(obj)); } + StringRef getScope() { + return stringFromIndexStoreStringRef(indexstore_symbol_get_scope(obj)); + } StringRef getUSR() { return stringFromIndexStoreStringRef(indexstore_symbol_get_usr(obj)); } @@ -102,6 +105,12 @@ indexstore_occurrence_get_line_col(obj, &line, &col); return std::make_pair(line, col); } + + std::pair getEndLineCol() { + unsigned start, end; + indexstore_occurrence_get_end_line_col(obj, &start, &end); + return std::make_pair(start, end); + } }; class IndexStore; Index: include/indexstore/indexstore.h =================================================================== --- include/indexstore/indexstore.h +++ include/indexstore/indexstore.h @@ -291,6 +291,9 @@ indexstore_symbol_get_name(indexstore_symbol_t symbol); INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_symbol_get_scope(indexstore_symbol_t symbol); + +INDEXSTORE_PUBLIC indexstore_string_ref_t indexstore_symbol_get_usr(indexstore_symbol_t symbol); INDEXSTORE_PUBLIC indexstore_string_ref_t @@ -324,6 +327,10 @@ indexstore_occurrence_get_line_col(indexstore_occurrence_t occur, unsigned *line, unsigned *column); +INDEXSTORE_PUBLIC void +indexstore_occurrence_get_end_line_col(indexstore_occurrence_t occur, + unsigned *end_line, unsigned *end_column); + typedef void *indexstore_record_reader_t; INDEXSTORE_PUBLIC indexstore_record_reader_t indexstore_record_reader_create( Index: lib/Index/ClangIndexRecordWriter.cpp =================================================================== --- lib/Index/ClangIndexRecordWriter.cpp +++ lib/Index/ClangIndexRecordWriter.cpp @@ -12,8 +12,10 @@ #include "clang/AST/ASTContext.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/QualTypeNames.h" #include "clang/Index/IndexSymbol.h" #include "clang/Index/USRGeneration.h" +#include "clang/Lex/Lexer.h" using namespace clang; using namespace clang::index; @@ -80,7 +82,13 @@ for (auto &Rel : Occur.Relations) Related.push_back(writer::SymbolRelation{Rel.RelatedSymbol, Rel.Roles}); - Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col, Related); + SourceLocation Loc = SM.getComposedLoc(FID, Occur.Offset); + unsigned TokenLen = Lexer::MeasureTokenLength(Loc, SM, Ctx.getLangOpts()); + unsigned EndLine, EndCol; + std::tie(EndLine, EndCol) = getLineCol(Occur.Offset + TokenLen); + + Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col, + EndLine, EndCol, Related); } PrintingPolicy Policy(Ctx.getLangOpts()); @@ -95,14 +103,21 @@ Sym.SymInfo = Info; auto *ND = dyn_cast(D); + unsigned NameLen = 0; + unsigned QualLength = 0; if (ND) { llvm::raw_svector_ostream OS(Scratch); DeclarationName DeclName = ND->getDeclName(); if (!DeclName.isEmpty()) DeclName.print(OS, Policy); + NameLen = Scratch.size(); + + ND->printQualifiedName(OS); + QualLength = Scratch.size(); } - unsigned NameLen = Scratch.size(); + Sym.Name = StringRef(Scratch.data(), NameLen); + Sym.Scope = StringRef(Scratch.data() + NameLen, QualLength - 2 * NameLen); Sym.USR = getUSR(D); assert(!Sym.USR.empty() && "Recorded decl without USR!"); @@ -111,8 +126,8 @@ llvm::raw_svector_ostream OS(Scratch); CGNameGen->writeName(ND, OS); } - unsigned CGNameLen = Scratch.size() - NameLen; - Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen); + unsigned CGNameLen = Scratch.size() - QualLength; + Sym.CodeGenName = StringRef(Scratch.data() + QualLength, CGNameLen); return Sym; }); Index: lib/Index/IndexRecordReader.cpp =================================================================== --- lib/Index/IndexRecordReader.cpp +++ lib/Index/IndexRecordReader.cpp @@ -109,10 +109,12 @@ RecD.Roles = getSymbolRoles(read(Record, I)); RecD.RelatedRoles = getSymbolRoles(read(Record, I)); size_t NameLen = read(Record, I); + size_t ScopeLen = read(Record, I); size_t USRLen = read(Record, I); RecD.Name = Blob.substr(0, NameLen); - RecD.USR = Blob.substr(NameLen, USRLen); - RecD.CodeGenName = Blob.substr(NameLen + USRLen); + RecD.Scope = Blob.substr(NameLen, ScopeLen); + RecD.USR = Blob.substr(NameLen + ScopeLen, USRLen); + RecD.CodeGenName = Blob.substr(NameLen + ScopeLen + USRLen); } /// Reads occurrence data. @@ -162,6 +164,8 @@ RecOccur.Roles = getSymbolRoles(read(Record, I)); RecOccur.Line = read(Record, I); RecOccur.Column = read(Record, I); + RecOccur.EndLine = read(Record, I); + RecOccur.EndColumn = read(Record, I); unsigned NumRelated = read(Record, I); while (NumRelated--) { Index: lib/Index/IndexRecordWriter.cpp =================================================================== --- lib/Index/IndexRecordWriter.cpp +++ lib/Index/IndexRecordWriter.cpp @@ -40,6 +40,8 @@ SymbolRoleSet Roles; unsigned Line; unsigned Column; + unsigned EndLine; + unsigned EndColumn; SmallVector, 4> Related; }; } // end anonymous namespace @@ -146,6 +148,7 @@ assert(!SymInfo.USR.empty() && "Recorded decl without USR!"); Blob += SymInfo.Name; + Blob += SymInfo.Scope; Blob += SymInfo.USR; Blob += SymInfo.CodeGenName; @@ -166,6 +169,7 @@ Record.push_back(getIndexStoreRoles(Info.Roles)); Record.push_back(getIndexStoreRoles(Info.RelatedRoles)); Record.push_back(SymInfo.Name.size()); + Record.push_back(SymInfo.Scope.size()); Record.push_back(SymInfo.USR.size()); Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); } @@ -216,6 +220,8 @@ Record.push_back(getIndexStoreRoles(Occur.Roles)); Record.push_back(Occur.Line); Record.push_back(Occur.Column); + Record.push_back(Occur.EndLine); + Record.push_back(Occur.EndColumn); Record.push_back(Occur.Related.size()); for (auto &Rel : Occur.Related) { Record.push_back(getIndexStoreRoles(Rel.first.Roles)); @@ -337,6 +343,7 @@ void IndexRecordWriter::addOccurrence( OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, unsigned Column, + unsigned EndLine, unsigned EndColumn, ArrayRef Related) { assert(Record && "called addOccurrence without calling beginRecord"); auto &State = *Record; @@ -366,5 +373,5 @@ } State.Occurrences.push_back( - OccurrenceInfo{DeclID, D, Roles, Line, Column, std::move(RelatedDecls)}); + OccurrenceInfo{DeclID, D, Roles, Line, Column, EndLine, EndColumn, std::move(RelatedDecls)}); } Index: tools/IndexStore/IndexStore.cpp =================================================================== --- tools/IndexStore/IndexStore.cpp +++ tools/IndexStore/IndexStore.cpp @@ -264,6 +264,11 @@ return toIndexStoreString(D->Name); } +indexstore_string_ref_t indexstore_symbol_get_scope(indexstore_symbol_t sym) { + auto *D = static_cast(sym); + return toIndexStoreString(D->Scope); +} + indexstore_string_ref_t indexstore_symbol_get_usr(indexstore_symbol_t sym) { auto *D = static_cast(sym); return toIndexStoreString(D->USR); @@ -318,6 +323,15 @@ *column = recOccur->Column; } +void indexstore_occurrence_get_end_line_col(indexstore_occurrence_t occur, + unsigned *end_line, unsigned *end_column) { + auto *recOccur = static_cast(occur); + if (end_line) + *end_line = recOccur->EndLine; + if (end_column) + *end_column = recOccur->EndColumn; +} + typedef void *indexstore_record_reader_t; indexstore_record_reader_t Index: tools/c-index-test/JSONAggregation.h =================================================================== --- tools/c-index-test/JSONAggregation.h +++ tools/c-index-test/JSONAggregation.h @@ -18,6 +18,9 @@ /// Returns true if an error occurred, false otherwise. bool aggregateDataAsJSON(StringRef StorePath, raw_ostream &OS); +/// Returns true if an error occurred, false otherwise. +bool aggregateDataAsYAML(StringRef StorePath, raw_ostream &OS); + } // end namespace index } // end namespace clang Index: tools/c-index-test/JSONAggregation.cpp =================================================================== --- tools/c-index-test/JSONAggregation.cpp +++ tools/c-index-test/JSONAggregation.cpp @@ -10,11 +10,16 @@ #include "JSONAggregation.h" #include "indexstore/IndexStoreCXX.h" #include "clang/Index/IndexDataStoreSymbolUtils.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Path.h" +#include "llvm/Support/SHA1.h" #include "llvm/Support/raw_ostream.h" +#include +#include + using namespace clang; using namespace clang::index; using namespace indexstore; @@ -44,6 +49,7 @@ SymbolLanguage Lang; StringRef USR; StringRef Name; + StringRef Scope; StringRef CodegenName; SymbolRoleSet Roles = 0; SymbolRoleSet RelatedRoles = 0; @@ -62,6 +68,8 @@ std::vector Relations; unsigned Line; unsigned Column; + unsigned EndLine; + unsigned EndColumn; }; struct RecordInfo { @@ -93,6 +101,7 @@ bool process(); void processUnit(StringRef name, IndexUnitReader &UnitReader); void dumpJSON(raw_ostream &OS); + void dumpYAML(raw_ostream &OS); private: StringRef copyStr(StringRef str) { @@ -232,6 +241,7 @@ }); occurInfo.Roles = idxOccur.getRoles(); std::tie(occurInfo.Line, occurInfo.Column) = idxOccur.getLineCol(); + std::tie(occurInfo.EndLine, occurInfo.EndColumn) = idxOccur.getEndLineCol(); record->Occurrences.push_back(std::move(occurInfo)); return true; }); @@ -248,6 +258,7 @@ symInfo.Lang = getSymbolLanguage(sym.getLanguage()); symInfo.USR = pair.first->first(); symInfo.Name = copyStr(sym.getName()); + symInfo.Scope = copyStr(sym.getScope()); symInfo.CodegenName = copyStr(sym.getCodegenName()); Symbols.push_back(std::move(symInfo)); } @@ -392,6 +403,102 @@ OS << "}\n"; } +static StringRef getSymbolKindForYAML(SymbolKind K) { + switch (K) { + case SymbolKind::Unknown: return ""; + case SymbolKind::Module: return "Module"; + case SymbolKind::Namespace: return "Namespace"; + case SymbolKind::NamespaceAlias: return "NamespaceAlias"; + case SymbolKind::Macro: return "Macro"; + case SymbolKind::Enum: return "Enum"; + case SymbolKind::Struct: return "Struct"; + case SymbolKind::Class: return "Class"; + case SymbolKind::Protocol: return "Protocol"; + case SymbolKind::Extension: return "Extension"; + case SymbolKind::Union: return "Union"; + case SymbolKind::TypeAlias: return "TypeAlias"; + case SymbolKind::Function: return "Function"; + case SymbolKind::Variable: return "Variable"; + case SymbolKind::Field: return "Field"; + case SymbolKind::EnumConstant: return "EnumConstant"; + case SymbolKind::InstanceMethod: return "InstanceMethod"; + case SymbolKind::ClassMethod: return "ClassMethod"; + case SymbolKind::StaticMethod: return "StaticMethod"; + case SymbolKind::InstanceProperty: return "InstanceProperty"; + case SymbolKind::ClassProperty: return "ClassProperty"; + case SymbolKind::StaticProperty: return "StaticProperty"; + case SymbolKind::Constructor: return "Constructor"; + case SymbolKind::Destructor: return "Destructor"; + case SymbolKind::ConversionFunction: return "ConversionFunction"; + case SymbolKind::Parameter: return "Parameter"; + case SymbolKind::Using: return "Using"; + } + llvm_unreachable("invalid symbol kind"); +} + +void Aggregator::dumpYAML(raw_ostream &OS) { + std::set files; + for (unsigned i = 0, e = Units.size(); i != e; ++i) { + UnitInfo &unit = *Units[i]; + for (unsigned si = 0, se = unit.Sources.size(); si != se; ++si) { + UnitSourceInfo &source = unit.Sources[si]; + if (source.FilePath >= FilePaths.size()) + continue; + if (files.find(source.FilePath) == files.end()) + files.insert(source.FilePath); + else + continue; + + StringRef filename = FilePaths[source.FilePath]; + if (filename.empty()) + continue; + + for (unsigned ri = 0, re = source.AssociatedRecords.size(); ri != re; + ++ri) { + size_t RecordNumber = source.AssociatedRecords[ri]; + if (RecordNumber >= Records.size()) + continue; + RecordInfo &recInfo = *Records[source.AssociatedRecords[ri]]; + for (unsigned oi = 0, oe = recInfo.Occurrences.size(); oi != oe; ++oi) { + SymbolOccurrenceInfo &occurInfo = recInfo.Occurrences[oi]; + if (occurInfo.Symbol >= Symbols.size()) + continue; + StringRef role; + if (occurInfo.Roles & (unsigned)SymbolRole::Declaration) + role = "CanonicalDeclaration:\n"; + else if (occurInfo.Roles & (unsigned)SymbolRole::Definition) + role = "Definition:\n"; + else + continue; + SymbolInfo &symInfo = Symbols[occurInfo.Symbol]; + OS << "---\n"; + OS << "ID: " + << toHex( + toStringRef(SHA1::hash(arrayRefFromStringRef(symInfo.USR)))) + << "\n"; + OS << "Name : '" << symInfo.Name << "'\n"; + OS << "Scope: '" << symInfo.Scope << "'\n"; + OS << "SymInfo:\n"; + OS.indent(2) << "Kind: " << getSymbolKindForYAML(symInfo.Kind) + << "\n"; + OS.indent(2) << "Lang: Cpp\n"; + OS << role; + + OS.indent(2) << "FileURI: file://" << (filename[0] != '/' ? "/" : "") + << filename << "\n"; + OS.indent(2) << "Start:\n"; + OS.indent(4) << "Line: " << occurInfo.Line << "\n"; + OS.indent(4) << "Column: " << occurInfo.Column << "\n"; + OS.indent(2) << "End:\n"; + OS.indent(4) << "Line: " << occurInfo.EndLine << "\n"; + OS.indent(4) << "Column: " << occurInfo.EndColumn << "\n"; + OS << "...\n"; + } + } + } + } +} + bool index::aggregateDataAsJSON(StringRef StorePath, raw_ostream &OS) { std::string error; auto dataStore = IndexStore(StorePath, error); @@ -410,3 +517,22 @@ aggregator->dumpJSON(OS); return false; } + +bool index::aggregateDataAsYAML(StringRef StorePath, raw_ostream &OS) { + std::string error; + auto dataStore = IndexStore(StorePath, error); + if (!dataStore) { + errs() << "error opening store path '" << StorePath << "': " << error + << '\n'; + return true; + } + + // Explicitely avoid doing any memory cleanup for aggregator since the process + // is going to exit when we are done. + Aggregator *aggregator = new Aggregator(std::move(dataStore)); + bool err = aggregator->process(); + if (err) + return true; + aggregator->dumpYAML(OS); + return false; +} Index: tools/c-index-test/core_main.cpp =================================================================== --- tools/c-index-test/core_main.cpp +++ tools/c-index-test/core_main.cpp @@ -59,6 +59,7 @@ PrintUnit, PrintStoreFormatVersion, AggregateAsJSON, + AggregateAsYAML, WatchDir, }; @@ -80,6 +81,8 @@ "print-store-format-version", "Print store format version"), clEnumValN(ActionType::AggregateAsJSON, "aggregate-json", "Aggregate index data in JSON format"), + clEnumValN(ActionType::AggregateAsYAML, "aggregate-yaml", + "Aggregate index data in YAML format"), clEnumValN(ActionType::WatchDir, "watch-dir", "Watch directory for file events")), cl::cat(IndexTestCoreCategory)); @@ -1063,7 +1066,8 @@ outs() << indexstore::IndexStore::formatVersion() << '\n'; } - if (options::Action == ActionType::AggregateAsJSON) { + if (options::Action == ActionType::AggregateAsJSON || + options::Action == ActionType::AggregateAsYAML) { if (options::InputFiles.empty()) { errs() << "error: missing input data store directory\n"; return 1; @@ -1077,7 +1081,10 @@ errs() << "failed to open output file: " << EC.message() << '\n'; return 1; } - return aggregateDataAsJSON(storePath, OS); + if (options::Action == ActionType::AggregateAsJSON) + return aggregateDataAsJSON(storePath, OS); + else + return aggregateDataAsYAML(storePath, OS); } if (options::Action == ActionType::WatchDir) {