Index: clangd/index/FileIndex.h =================================================================== --- clangd/index/FileIndex.h +++ clangd/index/FileIndex.h @@ -41,16 +41,21 @@ public: /// \brief Updates all symbols in a file. If \p Slab is nullptr, symbols for /// \p Path will be removed. - void update(PathRef Path, std::unique_ptr Slab); + void update(PathRef Path, std::unique_ptr Slab, + std::unique_ptr Occurrences = nullptr); // The shared_ptr keeps the symbols alive std::shared_ptr> allSymbols(); + std::vector> allOccurrenceSlabs() const; + private: mutable std::mutex Mutex; /// \brief Stores the latest snapshots for all active files. llvm::StringMap> FileToSlabs; + /// \breif Stores the latest occurrence slabs for all active files. + llvm::StringMap> FileToOccurrenceSlabs; }; /// \brief This manages symbls from files and an in-memory index on all symbols. @@ -92,7 +97,7 @@ /// If URISchemes is empty, the default schemes in SymbolCollector will be used. /// If \p TopLevelDecls is set, only these decls are indexed. Otherwise, all top /// level decls obtained from \p AST are indexed. -SymbolSlab +std::pair indexAST(ASTContext &AST, std::shared_ptr PP, llvm::Optional> TopLevelDecls = llvm::None, llvm::ArrayRef URISchemes = {}); Index: clangd/index/FileIndex.cpp =================================================================== --- clangd/index/FileIndex.cpp +++ clangd/index/FileIndex.cpp @@ -16,9 +16,10 @@ namespace clang { namespace clangd { -SymbolSlab indexAST(ASTContext &AST, std::shared_ptr PP, - llvm::Optional> TopLevelDecls, - llvm::ArrayRef URISchemes) { +std::pair +indexAST(ASTContext &AST, std::shared_ptr PP, + llvm::Optional> TopLevelDecls, + llvm::ArrayRef URISchemes) { SymbolCollector::Options CollectorOpts; // FIXME(ioeric): we might also want to collect include headers. We would need // to make sure all includes are canonicalized (with CanonicalIncludes), which @@ -31,8 +32,6 @@ CollectorOpts.URISchemes = URISchemes; CollectorOpts.Origin = SymbolOrigin::Dynamic; - SymbolCollector Collector(std::move(CollectorOpts)); - Collector.setPreprocessor(PP); index::IndexingOptions IndexOpts; // We only need declarations, because we don't count references. IndexOpts.SystemSymbolFilter = @@ -46,20 +45,41 @@ DeclsToIndex.assign(AST.getTranslationUnitDecl()->decls().begin(), AST.getTranslationUnitDecl()->decls().end()); + if (TopLevelDecls) { // index main AST, set occurrence flag. + CollectorOpts.OccurrenceFilter = SymbolOccurrenceKind::Declaration | + SymbolOccurrenceKind::Definition | + SymbolOccurrenceKind::Reference; + } + + SymbolCollector Collector(std::move(CollectorOpts)); + Collector.setPreprocessor(PP); index::indexTopLevelDecls(AST, DeclsToIndex, Collector, IndexOpts); - return Collector.takeSymbols(); + vlog("index for AST: "); + auto Syms = Collector.takeSymbols(); + auto Occurrences = Collector.takeOccurrences(); + vlog(" symbol slab: {0} symbols, {1} bytes", Syms.size(), Syms.bytes()); + vlog(" occurrence slab: {0} symbols, {1} bytes", Occurrences.size(), + Occurrences.bytes()); + return {std::move(Syms), std::move(Occurrences)}; } FileIndex::FileIndex(std::vector URISchemes) : URISchemes(std::move(URISchemes)) {} -void FileSymbols::update(PathRef Path, std::unique_ptr Slab) { +void FileSymbols::update(PathRef Path, std::unique_ptr Slab, + std::unique_ptr Occurrences) { std::lock_guard Lock(Mutex); - if (!Slab) + if (!Slab) { FileToSlabs.erase(Path); - else + } else { FileToSlabs[Path] = std::move(Slab); + } + if (!Occurrences) { + FileToOccurrenceSlabs.erase(Path); + } else { + FileToOccurrenceSlabs[Path] = std::move(Occurrences); + } } std::shared_ptr> FileSymbols::allSymbols() { @@ -85,19 +105,32 @@ return {std::move(Snap), Pointers}; } +std::vector> +FileSymbols::allOccurrenceSlabs() const{ + std::vector> Slabs; + std::lock_guard Lock(Mutex); + + for (const auto &FileAndSlab : FileToOccurrenceSlabs) + Slabs.push_back(FileAndSlab.second); + return Slabs; +} + void FileIndex::update(PathRef Path, ASTContext *AST, std::shared_ptr PP, llvm::Optional> TopLevelDecls) { if (!AST) { - FSymbols.update(Path, nullptr); + FSymbols.update(Path, nullptr, nullptr); } else { assert(PP); auto Slab = llvm::make_unique(); - *Slab = indexAST(*AST, PP, TopLevelDecls, URISchemes); - FSymbols.update(Path, std::move(Slab)); + auto IndexResults = indexAST(*AST, PP, TopLevelDecls, URISchemes); + *Slab = std::move(IndexResults.first); + auto OccurrenceSlab = llvm::make_unique(); + *OccurrenceSlab = std::move(IndexResults.second); + FSymbols.update(Path, std::move(Slab), std::move(OccurrenceSlab)); } auto Symbols = FSymbols.allSymbols(); - Index.build(std::move(Symbols)); + Index.build(std::move(Symbols), FSymbols.allOccurrenceSlabs()); } bool FileIndex::fuzzyFind( @@ -115,7 +148,7 @@ void FileIndex::findOccurrences( const OccurrencesRequest &Req, llvm::function_ref Callback) const { - log("findOccurrences is not implemented."); + Index.findOccurrences(Req, Callback); } } // namespace clangd Index: clangd/index/Index.h =================================================================== --- clangd/index/Index.h +++ clangd/index/Index.h @@ -369,6 +369,15 @@ const_iterator begin() const { return Occurrences.begin(); } const_iterator end() const { return Occurrences.end(); } + size_t bytes() const { + return sizeof(*this) + Arena.getTotalMemory() + + Occurrences.getMemorySize(); + } + + size_t size() const { + return Occurrences.size(); + } + // Adds a symbol occurrence. // This is a deep copy: underlying FileURI will be owned by the slab. void insert(const SymbolID &SymID, const SymbolOccurrence &Occurrence); Index: clangd/index/MemIndex.h =================================================================== --- clangd/index/MemIndex.h +++ clangd/index/MemIndex.h @@ -22,7 +22,9 @@ public: /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain /// accessible as long as `Symbols` is kept alive. - void build(std::shared_ptr> Symbols); + void + build(std::shared_ptr> Symbols, + std::vector> OccurrenceSlabs = {}); /// \brief Build index from a symbol slab. static std::unique_ptr build(SymbolSlab Slab); @@ -44,6 +46,8 @@ // Index is a set of symbols that are deduplicated by symbol IDs. // FIXME: build smarter index structure. llvm::DenseMap Index; + + std::vector> OccurrenceSlabs; mutable std::mutex Mutex; }; Index: clangd/index/MemIndex.cpp =================================================================== --- clangd/index/MemIndex.cpp +++ clangd/index/MemIndex.cpp @@ -15,7 +15,9 @@ namespace clang { namespace clangd { -void MemIndex::build(std::shared_ptr> Syms) { +void MemIndex::build( + std::shared_ptr> Syms, + std::vector> Occurrences) { llvm::DenseMap TempIndex; for (const Symbol *Sym : *Syms) TempIndex[Sym->ID] = Sym; @@ -25,12 +27,13 @@ std::lock_guard Lock(Mutex); Index = std::move(TempIndex); Symbols = std::move(Syms); // Relase old symbols. + OccurrenceSlabs = std::move(Occurrences); } } std::unique_ptr MemIndex::build(SymbolSlab Slab) { auto Idx = llvm::make_unique(); - Idx->build(getSymbolsFromSlab(std::move(Slab))); + Idx->build(getSymbolsFromSlab(std::move(Slab)), {}); return std::move(Idx); } @@ -81,7 +84,15 @@ void MemIndex::findOccurrences( const OccurrencesRequest &Req, llvm::function_ref Callback) const { - log("findOccurrences is not implemented."); + for (const auto &Slab : OccurrenceSlabs) { + for (const auto &ID : Req.IDs) { + for (const auto &Occurrence : Slab->find(ID)) { + if (static_cast(Req.Filter & Occurrence.Kind)) { + Callback(Occurrence); + } + } + } + } } std::shared_ptr> Index: clangd/index/Merge.cpp =================================================================== --- clangd/index/Merge.cpp +++ clangd/index/Merge.cpp @@ -81,7 +81,21 @@ void findOccurrences(const OccurrencesRequest &Req, llvm::function_ref Callback) const override { - log("findOccurrences is not implemented."); + // Store all seen files by dynamic index. + llvm::DenseSet SeenFiles; + // Emit all dynamic occurrences, and static occurrences that are not + // in seen files. + // FIXME: If a file has been updated, and there are no occurrences indexed + // in dynamic index, stale results are still returned (from static index). + Dynamic->findOccurrences(Req, [&](const SymbolOccurrence &O) { + SeenFiles.insert(O.Location.FileURI); + Callback(O); + }); + Static->findOccurrences(Req, [&](const SymbolOccurrence& O) { + if (llvm::is_contained(SeenFiles, O.Location.FileURI)) + return; + Callback(O); + }); } private: Index: unittests/clangd/FileIndexTests.cpp =================================================================== --- unittests/clangd/FileIndexTests.cpp +++ unittests/clangd/FileIndexTests.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "Annotations.h" #include "ClangdUnit.h" #include "TestFS.h" #include "TestTU.h" @@ -19,6 +20,19 @@ #include "gtest/gtest.h" using testing::UnorderedElementsAre; +using testing::AllOf; + +MATCHER_P(OccurrenceRange, Range, "") { + return std::tie(arg.Location.Start.Line, + arg.Location.Start.Column, + arg.Location.End.Line, + arg.Location.End.Column) == + std::tie(Range.start.line, Range.start.character, Range.end.line, + Range.end.character); +} +MATCHER_P(FileURI, F, "") { + return arg.Location.FileURI == F; +} namespace clang { namespace clangd { @@ -270,6 +284,46 @@ UnorderedElementsAre("ns_in_header", "ns_in_header::func_in_header")); } +TEST(FileIndexTest, Occurrences) { + const char* HeaderCode = "class Foo {};"; + Annotations MainCode(R"cpp( + void f() { + $foo[[Foo]] foo; + } + )cpp"); + + auto Foo = + findSymbol(TestTU::withHeaderCode(HeaderCode).headerSymbols(), "Foo"); + + OccurrencesRequest Request; + Request.IDs = {Foo.ID}; + Request.Filter = SymbolOccurrenceKind::Declaration | + SymbolOccurrenceKind::Definition | + SymbolOccurrenceKind::Reference; + + FileIndex Index(/*URISchemes*/{"unittest"}); + // Add test.cc + auto AST = + TestTU::withAllCode(HeaderCode, MainCode.code(), "test.cc").build(); + Index.update("test.cc", &AST.getASTContext(), AST.getPreprocessorPtr(), + AST.getLocalTopLevelDecls()); + // Add test2.cc + AST = TestTU::withAllCode(HeaderCode, MainCode.code(), "test2.cc").build(); + Index.update("test2.cc", &AST.getASTContext(), AST.getPreprocessorPtr(), + AST.getLocalTopLevelDecls()); + + std::vector Results; + Index.findOccurrences(Request, [&Results](const SymbolOccurrence& O) { + Results.push_back(O); + }); + + EXPECT_THAT(Results, + UnorderedElementsAre(AllOf(OccurrenceRange(MainCode.range("foo")), + FileURI("unittest:///test.cc")), + AllOf(OccurrenceRange(MainCode.range("foo")), + FileURI("unittest:///test2.cc")))); +} + } // namespace } // namespace clangd } // namespace clang Index: unittests/clangd/IndexTests.cpp =================================================================== --- unittests/clangd/IndexTests.cpp +++ unittests/clangd/IndexTests.cpp @@ -8,20 +8,33 @@ //===----------------------------------------------------------------------===// #include "TestIndex.h" +#include "TestTU.h" +#include "Annotations.h" #include "index/Index.h" #include "index/MemIndex.h" #include "index/Merge.h" +#include "index/FileIndex.h" #include "gmock/gmock.h" #include "gtest/gtest.h" using testing::Pointee; using testing::UnorderedElementsAre; +using testing::AllOf; namespace clang { namespace clangd { namespace { MATCHER_P(Named, N, "") { return arg.Name == N; } +MATCHER_P(OccurrenceRange, Range, "") { + return std::tie(arg.Location.Start.Line, + arg.Location.Start.Column, + arg.Location.End.Line, + arg.Location.End.Column) == + std::tie(Range.start.line, Range.start.character, Range.end.line, + Range.end.character); +} +MATCHER_P(FileURI, F, "") { return arg.Location.FileURI == F; } TEST(SymbolSlab, FindAndIterate) { SymbolSlab::Builder B; @@ -243,6 +256,55 @@ EXPECT_EQ(M.Name, "right"); } +TEST(MergeIndexTest, FindOccurrences) { + FileIndex Dyn({"unittest"}); + FileIndex StaticIndex({"unittest"}); + auto MergedIndex = mergeIndex(&Dyn, &StaticIndex); + + const char* HeaderCode = "class Foo;"; + auto HeaderSymbols = TestTU::withHeaderCode("class Foo;").headerSymbols(); + auto Foo = findSymbol(HeaderSymbols, "Foo"); + + // Build dynamic index. + Annotations Test1Code(R"(class $Foo[[Foo]];)"); + auto AST = + TestTU::withAllCode(HeaderCode, Test1Code.code(), "test.cc").build(); + Dyn.update("test.cc", &AST.getASTContext(), AST.getPreprocessorPtr(), + AST.getLocalTopLevelDecls()); + + // Build static index. + auto StaticAST = + TestTU::withAllCode(HeaderCode, "// static\nclass Foo {};", "test.cc") + .build(); + // Add stale occurrences for test.cc. + StaticIndex.update("test.cc", &StaticAST.getASTContext(), + StaticAST.getPreprocessorPtr(), + StaticAST.getLocalTopLevelDecls()); + + // Add occcurrences for test2.cc + Annotations Test2Code(R"(class $Foo[[Foo]] {};)"); + StaticAST = + TestTU::withAllCode(HeaderCode, Test2Code.code(), "test2.cc").build(); + StaticIndex.update("test2.cc", &StaticAST.getASTContext(), + StaticAST.getPreprocessorPtr(), + StaticAST.getLocalTopLevelDecls()); + + OccurrencesRequest Request; + Request.IDs = {Foo.ID}; + Request.Filter = SymbolOccurrenceKind::Declaration | + SymbolOccurrenceKind::Definition | + SymbolOccurrenceKind::Reference; + std::vector Results; + MergedIndex->findOccurrences( + Request, [&](const SymbolOccurrence &O) { Results.push_back(O); }); + + EXPECT_THAT(Results, + UnorderedElementsAre(AllOf(OccurrenceRange(Test1Code.range("Foo")), + FileURI("unittest:///test.cc")), + AllOf(OccurrenceRange(Test2Code.range("Foo")), + FileURI("unittest:///test2.cc")))); +} + } // namespace } // namespace clangd } // namespace clang Index: unittests/clangd/TestTU.h =================================================================== --- unittests/clangd/TestTU.h +++ unittests/clangd/TestTU.h @@ -38,6 +38,16 @@ return TU; } + static TestTU withAllCode(llvm::StringRef HeaderCode, llvm::StringRef Code, + llvm::StringRef Filename = "") { + TestTU TU; + TU.HeaderCode = HeaderCode; + TU.Code = Code; + if (!Filename.empty()) + TU.Filename = Filename; + return TU; + } + // The code to be compiled. std::string Code; std::string Filename = "TestTU.cpp"; Index: unittests/clangd/TestTU.cpp =================================================================== --- unittests/clangd/TestTU.cpp +++ unittests/clangd/TestTU.cpp @@ -45,7 +45,7 @@ SymbolSlab TestTU::headerSymbols() const { auto AST = build(); - return indexAST(AST.getASTContext(), AST.getPreprocessorPtr()); + return indexAST(AST.getASTContext(), AST.getPreprocessorPtr()).first; } std::unique_ptr TestTU::index() const {