diff --git a/clang-tools-extra/clangd/index/BackgroundRebuild.cpp b/clang-tools-extra/clangd/index/BackgroundRebuild.cpp --- a/clang-tools-extra/clangd/index/BackgroundRebuild.cpp +++ b/clang-tools-extra/clangd/index/BackgroundRebuild.cpp @@ -118,7 +118,8 @@ vlog("BackgroundIndex: building version {0} {1}", BuildVersion, Reason); trace::Span Tracer("RebuildBackgroundIndex"); SPAN_ATTACH(Tracer, "reason", Reason); - NewIndex = Source->buildIndex(IndexType::Heavy, DuplicateHandling::Merge); + NewIndex = Source->buildIndex(IndexType::Heavy, IndexDataKind::All, + DuplicateHandling::Merge); } { std::lock_guard Lock(Mu); diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h --- a/clang-tools-extra/clangd/index/FileIndex.h +++ b/clang-tools-extra/clangd/index/FileIndex.h @@ -84,7 +84,7 @@ /// files, while building the index with DuplicateHandling::Merge option. /// Version is populated with an increasing sequence counter. std::unique_ptr - buildIndex(IndexType, + buildIndex(IndexType, IndexDataKind DataKind = IndexDataKind::None, DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne, size_t *Version = nullptr); diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -261,8 +261,8 @@ } std::unique_ptr -FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle, - size_t *Version) { +FileSymbols::buildIndex(IndexType Type, IndexDataKind DataKind, + DuplicateHandling DuplicateHandle, size_t *Version) { std::vector> SymbolSlabs; std::vector> RefSlabs; std::vector> RelationSlabs; @@ -375,14 +375,14 @@ case IndexType::Light: return std::make_unique( llvm::make_pointee_range(AllSymbols), std::move(AllRefs), - std::move(AllRelations), std::move(Files), + std::move(AllRelations), std::move(Files), DataKind, std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), std::move(RefsStorage), std::move(SymsStorage)), StorageSize); case IndexType::Heavy: return std::make_unique( llvm::make_pointee_range(AllSymbols), std::move(AllRefs), - std::move(AllRelations), std::move(Files), + std::move(AllRelations), std::move(Files), DataKind, std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), std::move(RefsStorage), std::move(SymsStorage)), StorageSize); @@ -428,16 +428,18 @@ // We are using the key received from ShardedIndex, so it should always // exist. assert(IF); + auto FilePath = URI::resolve(Uri, Path); PreambleSymbols.update( - Uri, std::make_unique(std::move(*IF->Symbols)), + FilePath ? *FilePath : Uri, + std::make_unique(std::move(*IF->Symbols)), std::make_unique(), std::make_unique(std::move(*IF->Relations)), /*CountReferences=*/false); } size_t IndexVersion = 0; - auto NewIndex = - PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light, - DuplicateHandling::PickOne, &IndexVersion); + auto NewIndex = PreambleSymbols.buildIndex( + UseDex ? IndexType::Heavy : IndexType::Light, IndexDataKind::Symbols, + DuplicateHandling::PickOne, &IndexVersion); { std::lock_guard Lock(UpdateIndexMu); if (IndexVersion <= PreambleIndexVersion) { @@ -461,8 +463,9 @@ std::make_unique(std::move(std::get<2>(Contents))), /*CountReferences=*/true); size_t IndexVersion = 0; - auto NewIndex = MainFileSymbols.buildIndex( - IndexType::Light, DuplicateHandling::Merge, &IndexVersion); + auto NewIndex = + MainFileSymbols.buildIndex(IndexType::Light, IndexDataKind::All, + DuplicateHandling::Merge, &IndexVersion); { std::lock_guard Lock(UpdateIndexMu); if (IndexVersion <= MainIndexVersion) { diff --git a/clang-tools-extra/clangd/index/Index.h b/clang-tools-extra/clangd/index/Index.h --- a/clang-tools-extra/clangd/index/Index.h +++ b/clang-tools-extra/clangd/index/Index.h @@ -82,6 +82,24 @@ llvm::Optional Limit; }; +enum class IndexDataKind : uint8_t { + None = 0, + Symbols = 1 << 1, + References = 1 << 2, + Relations = 1 << 3, + All = Symbols | References | Relations +}; + +inline constexpr IndexDataKind operator&(IndexDataKind L, IndexDataKind R) { + return static_cast(static_cast(L) & + static_cast(R)); +} + +inline constexpr IndexDataKind operator|(IndexDataKind L, IndexDataKind R) { + return static_cast(static_cast(L) | + static_cast(R)); +} + /// Interface for symbol indexes that can be used for searching or /// matching symbols among a set of symbols based on names or unique IDs. class SymbolIndex { @@ -124,7 +142,7 @@ /// Returns function which checks if the specified file was used to build this /// index or not. The function must only be called while the index is alive. - virtual llvm::unique_function + virtual llvm::unique_function indexedFiles() const = 0; /// Returns estimated size of index (in bytes). @@ -151,7 +169,7 @@ llvm::function_ref) const override; - llvm::unique_function + llvm::unique_function indexedFiles() const override; size_t estimateMemoryUsage() const override; diff --git a/clang-tools-extra/clangd/index/Index.cpp b/clang-tools-extra/clangd/index/Index.cpp --- a/clang-tools-extra/clangd/index/Index.cpp +++ b/clang-tools-extra/clangd/index/Index.cpp @@ -76,7 +76,7 @@ return snapshot()->relations(R, CB); } -llvm::unique_function +llvm::unique_function SwapIndex::indexedFiles() const { return snapshot()->indexedFiles(); } diff --git a/clang-tools-extra/clangd/index/MemIndex.h b/clang-tools-extra/clangd/index/MemIndex.h --- a/clang-tools-extra/clangd/index/MemIndex.h +++ b/clang-tools-extra/clangd/index/MemIndex.h @@ -48,12 +48,14 @@ template MemIndex(SymbolRange &&Symbols, RefRange &&Refs, RelationRange &&Relations, - FileRange &&Files, Payload &&BackingData, size_t BackingDataSize) + FileRange &&Files, IndexDataKind DataKind, Payload &&BackingData, + size_t BackingDataSize) : MemIndex(std::forward(Symbols), std::forward(Refs), std::forward(Relations), std::forward(BackingData), BackingDataSize) { this->Files = std::forward(Files); + this->DataKind = DataKind; } /// Builds an index from slabs. The index takes ownership of the data. @@ -74,7 +76,7 @@ llvm::function_ref Callback) const override; - llvm::unique_function + llvm::unique_function indexedFiles() const override; size_t estimateMemoryUsage() const override; @@ -90,6 +92,8 @@ llvm::DenseMap, std::vector> Relations; // Set of files which were used during this index build. llvm::StringSet<> Files; + /// Kind of data in this index (references, symbols, etc.) + IndexDataKind DataKind; std::shared_ptr KeepAlive; // poor man's move-only std::any // Size of memory retained by KeepAlive. size_t BackingDataSize = 0; diff --git a/clang-tools-extra/clangd/index/MemIndex.cpp b/clang-tools-extra/clangd/index/MemIndex.cpp --- a/clang-tools-extra/clangd/index/MemIndex.cpp +++ b/clang-tools-extra/clangd/index/MemIndex.cpp @@ -109,15 +109,15 @@ } } -llvm::unique_function +llvm::unique_function MemIndex::indexedFiles() const { return [this](llvm::StringRef FileURI) { auto Path = URI::resolve(FileURI); if (!Path) { llvm::consumeError(Path.takeError()); - return false; + return IndexDataKind::None; } - return Files.contains(*Path); + return Files.contains(*Path) ? DataKind : IndexDataKind::None; }; } diff --git a/clang-tools-extra/clangd/index/Merge.h b/clang-tools-extra/clangd/index/Merge.h --- a/clang-tools-extra/clangd/index/Merge.h +++ b/clang-tools-extra/clangd/index/Merge.h @@ -41,7 +41,7 @@ void relations(const RelationsRequest &, llvm::function_ref) const override; - llvm::unique_function + llvm::unique_function indexedFiles() const override; size_t estimateMemoryUsage() const override { return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage(); diff --git a/clang-tools-extra/clangd/index/Merge.cpp b/clang-tools-extra/clangd/index/Merge.cpp --- a/clang-tools-extra/clangd/index/Merge.cpp +++ b/clang-tools-extra/clangd/index/Merge.cpp @@ -48,8 +48,9 @@ More |= Static->fuzzyFind(Req, [&](const Symbol &S) { // We expect the definition to see the canonical declaration, so it seems // to be enough to check only the definition if it exists. - if (DynamicContainsFile(S.Definition ? S.Definition.FileURI - : S.CanonicalDeclaration.FileURI)) + if ((DynamicContainsFile(S.Definition ? S.Definition.FileURI + : S.CanonicalDeclaration.FileURI) & + IndexDataKind::Symbols) != IndexDataKind::None) return; auto DynS = Dyn.find(S.ID); ++StaticCount; @@ -81,8 +82,9 @@ Static->lookup(Req, [&](const Symbol &S) { // We expect the definition to see the canonical declaration, so it seems // to be enough to check only the definition if it exists. - if (DynamicContainsFile(S.Definition ? S.Definition.FileURI - : S.CanonicalDeclaration.FileURI)) + if ((DynamicContainsFile(S.Definition ? S.Definition.FileURI + : S.CanonicalDeclaration.FileURI) & + IndexDataKind::Symbols) != IndexDataKind::None) return; const Symbol *Sym = B.find(S.ID); RemainingIDs.erase(S.ID); @@ -117,7 +119,8 @@ // We return less than Req.Limit if static index returns more refs for dirty // files. bool StaticHadMore = Static->refs(Req, [&](const Ref &O) { - if (DynamicContainsFile(O.Location.FileURI)) + if ((DynamicContainsFile(O.Location.FileURI) & IndexDataKind::References) != + IndexDataKind::None) return; // ignore refs that have been seen from dynamic index. if (Remaining == 0) { More = true; @@ -129,11 +132,11 @@ return More || StaticHadMore; } -llvm::unique_function +llvm::unique_function MergedIndex::indexedFiles() const { return [DynamicContainsFile{Dynamic->indexedFiles()}, StaticContainsFile{Static->indexedFiles()}](llvm::StringRef FileURI) { - return DynamicContainsFile(FileURI) || StaticContainsFile(FileURI); + return DynamicContainsFile(FileURI) | StaticContainsFile(FileURI); }; } diff --git a/clang-tools-extra/clangd/index/ProjectAware.cpp b/clang-tools-extra/clangd/index/ProjectAware.cpp --- a/clang-tools-extra/clangd/index/ProjectAware.cpp +++ b/clang-tools-extra/clangd/index/ProjectAware.cpp @@ -54,7 +54,7 @@ llvm::function_ref Callback) const override; - llvm::unique_function + llvm::unique_function indexedFiles() const override; ProjectAwareIndex(IndexFactory Gen) : Gen(std::move(Gen)) {} @@ -115,12 +115,12 @@ return Idx->relations(Req, Callback); } -llvm::unique_function +llvm::unique_function ProjectAwareIndex::indexedFiles() const { trace::Span Tracer("ProjectAwareIndex::indexedFiles"); if (auto *Idx = getIndex()) return Idx->indexedFiles(); - return [](llvm::StringRef) { return false; }; + return [](llvm::StringRef) { return IndexDataKind::None; }; } SymbolIndex *ProjectAwareIndex::getIndex() const { diff --git a/clang-tools-extra/clangd/index/dex/Dex.h b/clang-tools-extra/clangd/index/dex/Dex.h --- a/clang-tools-extra/clangd/index/dex/Dex.h +++ b/clang-tools-extra/clangd/index/dex/Dex.h @@ -70,11 +70,13 @@ template Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations, - FileRange &&Files, Payload &&BackingData, size_t BackingDataSize) + FileRange &&Files, IndexDataKind DataKind, Payload &&BackingData, + size_t BackingDataSize) : Dex(std::forward(Symbols), std::forward(Refs), std::forward(Relations), std::forward(BackingData), BackingDataSize) { this->Files = std::forward(Files); + this->DataKind = DataKind; } /// Builds an index from slabs. The index takes ownership of the slab. @@ -94,7 +96,7 @@ llvm::function_ref Callback) const override; - llvm::unique_function + llvm::unique_function indexedFiles() const override; size_t estimateMemoryUsage() const override; @@ -127,6 +129,8 @@ std::shared_ptr KeepAlive; // poor man's move-only std::any // Set of files which were used during this index build. llvm::StringSet<> Files; + /// Kind of data in this index (references, symbols, etc.) + IndexDataKind DataKind; // Size of memory retained by KeepAlive. size_t BackingDataSize = 0; }; diff --git a/clang-tools-extra/clangd/index/dex/Dex.cpp b/clang-tools-extra/clangd/index/dex/Dex.cpp --- a/clang-tools-extra/clangd/index/dex/Dex.cpp +++ b/clang-tools-extra/clangd/index/dex/Dex.cpp @@ -313,14 +313,15 @@ } } -llvm::unique_function Dex::indexedFiles() const { +llvm::unique_function +Dex::indexedFiles() const { return [this](llvm::StringRef FileURI) { auto Path = URI::resolve(FileURI); if (!Path) { llvm::consumeError(Path.takeError()); - return false; + return IndexDataKind::None; } - return Files.contains(*Path); + return Files.contains(*Path) ? DataKind : IndexDataKind::None; }; } diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp --- a/clang-tools-extra/clangd/index/remote/Client.cpp +++ b/clang-tools-extra/clangd/index/remote/Client.cpp @@ -152,13 +152,13 @@ }); } - llvm::unique_function + llvm::unique_function indexedFiles() const override { - // FIXME: For now we always return "false" regardless of whether the file - // was indexed or not. A possible implementation could be based on - // the idea that we do not want to send a request at every + // FIXME: For now we always return IndexDataKind::None regardless of whether + // the file was indexed or not. A possible implementation could be + // based on the idea that we do not want to send a request at every // call of a function returned by IndexClient::indexedFiles(). - return [](llvm::StringRef) { return false; }; + return [](llvm::StringRef) { return IndexDataKind::None; }; } // IndexClient does not take any space since the data is stored on the diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp --- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp +++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp @@ -1390,9 +1390,9 @@ llvm::function_ref) const override {} - llvm::unique_function + llvm::unique_function indexedFiles() const override { - return [](llvm::StringRef) { return false; }; + return [](llvm::StringRef) { return IndexDataKind::None; }; } // This is incorrect, but IndexRequestCollector is not an actual index and it diff --git a/clang-tools-extra/clangd/unittests/DexTests.cpp b/clang-tools-extra/clangd/unittests/DexTests.cpp --- a/clang-tools-extra/clangd/unittests/DexTests.cpp +++ b/clang-tools-extra/clangd/unittests/DexTests.cpp @@ -739,11 +739,11 @@ auto Data = std::make_pair(std::move(Symbols), std::move(Refs)); llvm::StringSet<> Files = {testPath("foo.cc"), testPath("bar.cc")}; Dex I(std::move(Data.first), std::move(Data.second), RelationSlab(), - std::move(Files), std::move(Data), Size); + std::move(Files), IndexDataKind::All, std::move(Data), Size); auto ContainsFile = I.indexedFiles(); - EXPECT_TRUE(ContainsFile("unittest:///foo.cc")); - EXPECT_TRUE(ContainsFile("unittest:///bar.cc")); - EXPECT_FALSE(ContainsFile("unittest:///foobar.cc")); + EXPECT_EQ(ContainsFile("unittest:///foo.cc"), IndexDataKind::All); + EXPECT_EQ(ContainsFile("unittest:///bar.cc"), IndexDataKind::All); + EXPECT_EQ(ContainsFile("unittest:///foobar.cc"), IndexDataKind::None); } TEST(DexTest, PreferredTypesBoosting) { diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp --- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp @@ -138,10 +138,11 @@ FS.update("f1", OneSymboSlab(X1), nullptr, nullptr, false); FS.update("f2", OneSymboSlab(X2), nullptr, nullptr, false); for (auto Type : {IndexType::Light, IndexType::Heavy}) - EXPECT_THAT( - runFuzzyFind(*FS.buildIndex(Type, DuplicateHandling::Merge), "x"), - UnorderedElementsAre( - AllOf(QName("x"), DeclURI("file:///x1"), DefURI("file:///x2")))); + EXPECT_THAT(runFuzzyFind(*FS.buildIndex(Type, IndexDataKind::Symbols, + DuplicateHandling::Merge), + "x"), + UnorderedElementsAre(AllOf(QName("x"), DeclURI("file:///x1"), + DefURI("file:///x2")))); } TEST(FileSymbolsTest, SnapshotAliveAfterRemove) { @@ -499,7 +500,8 @@ FS.update("f1", numSlab(1, 3), nullptr, nullptr, true); FS.update("f2", numSlab(1, 3), nullptr, nullptr, false); EXPECT_THAT( - runFuzzyFind(*FS.buildIndex(IndexType::Light, DuplicateHandling::Merge), + runFuzzyFind(*FS.buildIndex(IndexType::Light, IndexDataKind::Symbols, + DuplicateHandling::Merge), ""), UnorderedElementsAre(AllOf(QName("1"), NumReferences(0u)), AllOf(QName("2"), NumReferences(0u)), @@ -520,12 +522,14 @@ true); FS.update("f3h", numSlab(1, 3), refSlab(SymbolID("3"), "f3.h"), nullptr, false); - EXPECT_THAT( - runFuzzyFind(*FS.buildIndex(IndexType::Light, DuplicateHandling::Merge), - ""), - UnorderedElementsAre(AllOf(QName("1"), NumReferences(1u)), - AllOf(QName("2"), NumReferences(1u)), - AllOf(QName("3"), NumReferences(1u)))); + EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light, + IndexDataKind::Symbols | + IndexDataKind::References, + DuplicateHandling::Merge), + ""), + UnorderedElementsAre(AllOf(QName("1"), NumReferences(1u)), + AllOf(QName("2"), NumReferences(1u)), + AllOf(QName("3"), NumReferences(1u)))); } TEST(FileIndexTest, StalePreambleSymbolsDeleted) { diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp --- a/clang-tools-extra/clangd/unittests/IndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp @@ -231,11 +231,11 @@ auto Data = std::make_pair(std::move(Symbols), std::move(Refs)); llvm::StringSet<> Files = {testPath("foo.cc"), testPath("bar.cc")}; MemIndex I(std::move(Data.first), std::move(Data.second), RelationSlab(), - std::move(Files), std::move(Data), Size); + std::move(Files), IndexDataKind::All, std::move(Data), Size); auto ContainsFile = I.indexedFiles(); - EXPECT_TRUE(ContainsFile("unittest:///foo.cc")); - EXPECT_TRUE(ContainsFile("unittest:///bar.cc")); - EXPECT_FALSE(ContainsFile("unittest:///foobar.cc")); + EXPECT_EQ(ContainsFile("unittest:///foo.cc"), IndexDataKind::All); + EXPECT_EQ(ContainsFile("unittest:///bar.cc"), IndexDataKind::All); + EXPECT_EQ(ContainsFile("unittest:///foobar.cc"), IndexDataKind::None); } TEST(MemIndexTest, TemplateSpecialization) { @@ -508,23 +508,24 @@ auto DynData = std::make_pair(std::move(DynSymbols), std::move(DynRefs)); llvm::StringSet<> DynFiles = {testPath("foo.cc")}; MemIndex DynIndex(std::move(DynData.first), std::move(DynData.second), - RelationSlab(), std::move(DynFiles), std::move(DynData), - DynSize); + RelationSlab(), std::move(DynFiles), IndexDataKind::Symbols, + std::move(DynData), DynSize); SymbolSlab StaticSymbols; RefSlab StaticRefs; auto StaticData = std::make_pair(std::move(StaticSymbols), std::move(StaticRefs)); - llvm::StringSet<> StaticFiles = {testPath("bar.cc")}; - MemIndex StaticIndex(std::move(StaticData.first), - std::move(StaticData.second), RelationSlab(), - std::move(StaticFiles), std::move(StaticData), - StaticSymbols.bytes() + StaticRefs.bytes()); + llvm::StringSet<> StaticFiles = {testPath("foo.cc"), testPath("bar.cc")}; + MemIndex StaticIndex( + std::move(StaticData.first), std::move(StaticData.second), RelationSlab(), + std::move(StaticFiles), IndexDataKind::References, std::move(StaticData), + StaticSymbols.bytes() + StaticRefs.bytes()); MergedIndex Merge(&DynIndex, &StaticIndex); auto ContainsFile = Merge.indexedFiles(); - EXPECT_TRUE(ContainsFile("unittest:///foo.cc")); - EXPECT_TRUE(ContainsFile("unittest:///bar.cc")); - EXPECT_FALSE(ContainsFile("unittest:///foobar.cc")); + EXPECT_EQ(ContainsFile("unittest:///foo.cc"), + IndexDataKind::Symbols | IndexDataKind::References); + EXPECT_EQ(ContainsFile("unittest:///bar.cc"), IndexDataKind::References); + EXPECT_EQ(ContainsFile("unittest:///foobar.cc"), IndexDataKind::None); } TEST(MergeIndexTest, NonDocumentation) { diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp --- a/clang-tools-extra/clangd/unittests/RenameTests.cpp +++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp @@ -1238,9 +1238,9 @@ llvm::function_ref Callback) const override {} - llvm::unique_function + llvm::unique_function indexedFiles() const override { - return [](llvm::StringRef) { return false; }; + return [](llvm::StringRef) { return IndexDataKind::None; }; } size_t estimateMemoryUsage() const override { return 0; } @@ -1292,9 +1292,9 @@ llvm::function_ref) const override {} - llvm::unique_function + llvm::unique_function indexedFiles() const override { - return [](llvm::StringRef) { return false; }; + return [](llvm::StringRef) { return IndexDataKind::None; }; } size_t estimateMemoryUsage() const override { return 0; }