diff --git a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp --- a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp +++ b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp @@ -58,7 +58,8 @@ auto Buffer = llvm::MemoryBuffer::getFile(ShardPath); if (!Buffer) return nullptr; - if (auto I = readIndexFile(Buffer->get()->getBuffer())) + if (auto I = + readIndexFile(Buffer->get()->getBuffer(), SymbolOrigin::Background)) return std::make_unique(std::move(*I)); else elog("Error while reading shard {0}: {1}", ShardIdentifier, diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -27,14 +27,12 @@ #include "clang/AST/ASTContext.h" #include "clang/Index/IndexingAction.h" #include "clang/Index/IndexingOptions.h" -#include "clang/Lex/MacroInfo.h" #include "clang/Lex/Preprocessor.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Error.h" #include #include #include @@ -54,7 +52,8 @@ CollectorOpts.CollectIncludePath = true; CollectorOpts.Includes = &Includes; CollectorOpts.CountReferences = false; - CollectorOpts.Origin = SymbolOrigin::Dynamic; + CollectorOpts.Origin = + IsIndexMainAST ? SymbolOrigin::Open : SymbolOrigin::Preamble; CollectorOpts.CollectMainFileRefs = CollectMainFileRefs; // We want stdlib implementation details in the index only if we've opened the // file in question. This does means xrefs won't work, though. diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h --- a/clang-tools-extra/clangd/index/Serialization.h +++ b/clang-tools-extra/clangd/index/Serialization.h @@ -49,7 +49,7 @@ llvm::Optional Cmd; }; // Parse an index file. The input must be a RIFF or YAML file. -llvm::Expected readIndexFile(llvm::StringRef); +llvm::Expected readIndexFile(llvm::StringRef, SymbolOrigin); // Specifies the contents of an index file to be written. struct IndexFileOut { @@ -79,16 +79,10 @@ std::string toYAML(const Relation &); std::string toYAML(const Ref &); -// Deserialize a single symbol from YAML. -llvm::Expected symbolFromYAML(StringRef YAML, - llvm::UniqueStringSaver *Strings); -llvm::Expected refFromYAML(StringRef YAML, - llvm::UniqueStringSaver *Strings); - // Build an in-memory static index from an index file. // The size should be relatively small, so data can be managed in memory. std::unique_ptr loadIndex(llvm::StringRef Filename, - bool UseDex = true); + SymbolOrigin Origin, bool UseDex = true); } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp --- a/clang-tools-extra/clangd/index/Serialization.cpp +++ b/clang-tools-extra/clangd/index/Serialization.cpp @@ -320,7 +320,6 @@ writeLocation(Sym.CanonicalDeclaration, Strings, OS); writeVar(Sym.References, OS); OS.write(static_cast(Sym.Flags)); - OS.write(static_cast(Sym.Origin)); writeVar(Strings.index(Sym.Signature), OS); writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS); writeVar(Strings.index(Sym.Documentation), OS); @@ -336,7 +335,8 @@ WriteInclude(Include); } -Symbol readSymbol(Reader &Data, llvm::ArrayRef Strings) { +Symbol readSymbol(Reader &Data, llvm::ArrayRef Strings, + SymbolOrigin Origin) { Symbol Sym; Sym.ID = Data.consumeID(); Sym.SymInfo.Kind = static_cast(Data.consume8()); @@ -348,7 +348,7 @@ Sym.CanonicalDeclaration = readLocation(Data, Strings); Sym.References = Data.consumeVar(); Sym.Flags = static_cast(Data.consume8()); - Sym.Origin = static_cast(Data.consume8()); + Sym.Origin = Origin; Sym.Signature = Data.consumeString(Strings); Sym.CompletionSnippetSuffix = Data.consumeString(Strings); Sym.Documentation = Data.consumeString(Strings); @@ -452,9 +452,10 @@ // The current versioning scheme is simple - non-current versions are rejected. // If you make a breaking change, bump this version number to invalidate stored // data. Later we may want to support some backward compatibility. -constexpr static uint32_t Version = 16; +constexpr static uint32_t Version = 17; -llvm::Expected readRIFF(llvm::StringRef Data) { +llvm::Expected readRIFF(llvm::StringRef Data, + SymbolOrigin Origin) { auto RIFF = riff::readFile(Data); if (!RIFF) return RIFF.takeError(); @@ -503,7 +504,7 @@ Reader SymbolReader(Chunks.lookup("symb")); SymbolSlab::Builder Symbols; while (!SymbolReader.eof()) - Symbols.insert(readSymbol(SymbolReader, Strings->Strings)); + Symbols.insert(readSymbol(SymbolReader, Strings->Strings, Origin)); if (SymbolReader.err()) return error("malformed or truncated symbol"); Result.Symbols = std::move(Symbols).build(); @@ -670,7 +671,7 @@ // Defined in YAMLSerialization.cpp. void writeYAML(const IndexFileOut &, llvm::raw_ostream &); -llvm::Expected readYAML(llvm::StringRef); +llvm::Expected readYAML(llvm::StringRef, SymbolOrigin Origin); llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) { switch (O.Format) { @@ -684,11 +685,12 @@ return OS; } -llvm::Expected readIndexFile(llvm::StringRef Data) { +llvm::Expected readIndexFile(llvm::StringRef Data, + SymbolOrigin Origin) { if (Data.startswith("RIFF")) { - return readRIFF(Data); + return readRIFF(Data, Origin); } - if (auto YAMLContents = readYAML(Data)) { + if (auto YAMLContents = readYAML(Data, Origin)) { return std::move(*YAMLContents); } else { return error("Not a RIFF file and failed to parse as YAML: {0}", @@ -697,7 +699,7 @@ } std::unique_ptr loadIndex(llvm::StringRef SymbolFilename, - bool UseDex) { + SymbolOrigin Origin, bool UseDex) { trace::Span OverallTracer("LoadIndex"); auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); if (!Buffer) { @@ -710,7 +712,7 @@ RelationSlab Relations; { trace::Span Tracer("ParseIndex"); - if (auto I = readIndexFile(Buffer->get()->getBuffer())) { + if (auto I = readIndexFile(Buffer->get()->getBuffer(), Origin)) { if (I->Symbols) Symbols = std::move(*I->Symbols); if (I->Refs) diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.h b/clang-tools-extra/clangd/index/SymbolOrigin.h --- a/clang-tools-extra/clangd/index/SymbolOrigin.h +++ b/clang-tools-extra/clangd/index/SymbolOrigin.h @@ -18,27 +18,29 @@ // Describes the source of information about a symbol. // Mainly useful for debugging, e.g. understanding code completion results. // This is a bitfield as information can be combined from several sources. -enum class SymbolOrigin : uint8_t { +enum class SymbolOrigin : uint16_t { Unknown = 0, AST = 1 << 0, // Directly from the AST (indexes should not set this). - Dynamic = 1 << 1, // From the dynamic index of opened files. - Static = 1 << 2, // From the static, externally-built index. + Open = 1 << 1, // From the dynamic index of open files. + Static = 1 << 2, // From a static, externally-built index. Merge = 1 << 3, // A non-trivial index merge was performed. Identifier = 1 << 4, // Raw identifiers in file. Remote = 1 << 5, // Remote index. - // Remaining bits reserved for index implementations. + Preamble = 1 << 6, // From the dynamic index of preambles. + // 7 reserved + Background = 1 << 8, // From the automatic project index. }; inline SymbolOrigin operator|(SymbolOrigin A, SymbolOrigin B) { - return static_cast(static_cast(A) | - static_cast(B)); + return static_cast(static_cast(A) | + static_cast(B)); } inline SymbolOrigin &operator|=(SymbolOrigin &A, SymbolOrigin B) { return A = A | B; } inline SymbolOrigin operator&(SymbolOrigin A, SymbolOrigin B) { - return static_cast(static_cast(A) & - static_cast(B)); + return static_cast(static_cast(A) & + static_cast(B)); } llvm::raw_ostream &operator<<(llvm::raw_ostream &, SymbolOrigin); diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.cpp b/clang-tools-extra/clangd/index/SymbolOrigin.cpp --- a/clang-tools-extra/clangd/index/SymbolOrigin.cpp +++ b/clang-tools-extra/clangd/index/SymbolOrigin.cpp @@ -14,9 +14,9 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) { if (O == SymbolOrigin::Unknown) return OS << "unknown"; - constexpr static char Sigils[] = "ADSMIR67"; + constexpr static char Sigils[] = "AOSMIRP7B9012345"; for (unsigned I = 0; I < sizeof(Sigils); ++I) - if (static_cast(O) & 1u << I) + if (static_cast(O) & 1u << I) OS << Sigils[I]; return OS; } diff --git a/clang-tools-extra/clangd/index/YAMLSerialization.cpp b/clang-tools-extra/clangd/index/YAMLSerialization.cpp --- a/clang-tools-extra/clangd/index/YAMLSerialization.cpp +++ b/clang-tools-extra/clangd/index/YAMLSerialization.cpp @@ -69,7 +69,6 @@ using clang::clangd::Symbol; using clang::clangd::SymbolID; using clang::clangd::SymbolLocation; -using clang::clangd::SymbolOrigin; using clang::index::SymbolInfo; using clang::index::SymbolKind; using clang::index::SymbolLanguage; @@ -108,17 +107,6 @@ uint8_t Flag = 0; }; -struct NormalizedSymbolOrigin { - NormalizedSymbolOrigin(IO &) {} - NormalizedSymbolOrigin(IO &, SymbolOrigin O) { - Origin = static_cast(O); - } - - SymbolOrigin denormalize(IO &) { return static_cast(Origin); } - - uint8_t Origin = 0; -}; - template <> struct MappingTraits { static void mapping(IO &IO, YPosition &Value) { IO.mapRequired("Line", Value.Line); @@ -194,8 +182,6 @@ MappingNormalization NSymbolID(IO, Sym.ID); MappingNormalization NSymbolFlag( IO, Sym.Flags); - MappingNormalization NSymbolOrigin( - IO, Sym.Origin); IO.mapRequired("ID", NSymbolID->HexString); IO.mapRequired("Name", Sym.Name); IO.mapRequired("Scope", Sym.Scope); @@ -204,7 +190,6 @@ SymbolLocation()); IO.mapOptional("Definition", Sym.Definition, SymbolLocation()); IO.mapOptional("References", Sym.References, 0u); - IO.mapOptional("Origin", NSymbolOrigin->Origin); IO.mapOptional("Flags", NSymbolFlag->Flag); IO.mapOptional("Signature", Sym.Signature); IO.mapOptional("TemplateSpecializationArgs", @@ -436,7 +421,8 @@ } } -llvm::Expected readYAML(llvm::StringRef Data) { +llvm::Expected readYAML(llvm::StringRef Data, + SymbolOrigin Origin) { SymbolSlab::Builder Symbols; RefSlab::Builder Refs; RelationSlab::Builder Relations; @@ -453,8 +439,10 @@ if (Yin.error()) return llvm::errorCodeToError(Yin.error()); - if (Variant.Symbol) + if (Variant.Symbol) { + Variant.Symbol->Origin = Origin; Symbols.insert(*Variant.Symbol); + } if (Variant.Refs) for (const auto &Ref : Variant.Refs->second) Refs.insert(Variant.Refs->first, Ref); @@ -528,25 +516,5 @@ return Buf; } -llvm::Expected -symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) { - clangd::Symbol Deserialized; - llvm::yaml::Input YAMLInput(YAML, Strings); - if (YAMLInput.error()) - return error("Unable to deserialize Symbol from YAML: {0}", YAML); - YAMLInput >> Deserialized; - return Deserialized; -} - -llvm::Expected refFromYAML(StringRef YAML, - llvm::UniqueStringSaver *Strings) { - clangd::Ref Deserialized; - llvm::yaml::Input YAMLInput(YAML, Strings); - if (YAMLInput.error()) - return error("Unable to deserialize Symbol from YAML: {0}", YAML); - YAMLInput >> Deserialized; - return Deserialized; -} - } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp --- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp +++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp @@ -334,7 +334,8 @@ } // Auto-detects input format when parsing - auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer()); + auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer(), + SymbolOrigin::Static); if (!IndexIn) { llvm::errs() << llvm::toString(IndexIn.takeError()) << "\n"; return; @@ -374,7 +375,7 @@ return Index.startswith("remote:") ? remote::getClient(Index.drop_front(strlen("remote:")), ProjectRoot) - : loadIndex(Index, /*UseDex=*/true); + : loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true); } bool runCommand(std::string Request, const SymbolIndex &Index) { diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto --- a/clang-tools-extra/clangd/index/remote/Index.proto +++ b/clang-tools-extra/clangd/index/remote/Index.proto @@ -66,7 +66,7 @@ optional string scope = 5; optional SymbolLocation canonical_declaration = 6; optional int32 references = 7; - optional uint32 origin = 8; + reserved 8; optional string signature = 9; optional string template_specialization_args = 10; optional string completion_snippet_suffix = 11; diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp --- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp +++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp @@ -269,7 +269,6 @@ return Declaration.takeError(); *Result.mutable_canonical_declaration() = *Declaration; Result.set_references(From.References); - Result.set_origin(static_cast(From.Origin)); Result.set_signature(From.Signature.str()); Result.set_template_specialization_args( From.TemplateSpecializationArgs.str()); diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp --- a/clang-tools-extra/clangd/index/remote/server/Server.cpp +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -395,7 +395,8 @@ "{0}, new index was modified at {1}. Attempting to reload.", LastStatus.getLastModificationTime(), Status->getLastModificationTime()); LastStatus = *Status; - std::unique_ptr NewIndex = loadIndex(IndexPath); + std::unique_ptr NewIndex = + loadIndex(IndexPath, SymbolOrigin::Static); if (!NewIndex) { elog("Failed to load new index. Old index will be served."); return; diff --git a/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx b/clang-tools-extra/clangd/test/index-serialization/Inputs/sample.idx index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@(std::make_unique()); auto IndexLoadTask = [File = External.Location, PlaceHolder = NewIndex.get()] { - if (auto Idx = loadIndex(File, /*UseDex=*/true)) + if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true)) PlaceHolder->reset(std::move(Idx)); }; if (Tasks) { diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp --- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp @@ -73,7 +73,8 @@ if (Storage.find(ShardIdentifier) == Storage.end()) { return nullptr; } - auto IndexFile = readIndexFile(Storage[ShardIdentifier]); + auto IndexFile = + readIndexFile(Storage[ShardIdentifier], SymbolOrigin::Background); if (!IndexFile) { ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':' << IndexFile.takeError(); diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp --- a/clang-tools-extra/clangd/unittests/IndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp @@ -392,7 +392,7 @@ L.Signature = "()"; // present in left only R.CompletionSnippetSuffix = "{$1:0}"; // present in right only R.Documentation = "--doc--"; - L.Origin = SymbolOrigin::Dynamic; + L.Origin = SymbolOrigin::Preamble; R.Origin = SymbolOrigin::Static; R.Type = "expectedType"; @@ -404,8 +404,8 @@ EXPECT_EQ(M.CompletionSnippetSuffix, "{$1:0}"); EXPECT_EQ(M.Documentation, "--doc--"); EXPECT_EQ(M.Type, "expectedType"); - EXPECT_EQ(M.Origin, - SymbolOrigin::Dynamic | SymbolOrigin::Static | SymbolOrigin::Merge); + EXPECT_EQ(M.Origin, SymbolOrigin::Preamble | SymbolOrigin::Static | + SymbolOrigin::Merge); } TEST(MergeTest, PreferSymbolWithDefn) { diff --git a/clang-tools-extra/clangd/unittests/SerializationTests.cpp b/clang-tools-extra/clangd/unittests/SerializationTests.cpp --- a/clang-tools-extra/clangd/unittests/SerializationTests.cpp +++ b/clang-tools-extra/clangd/unittests/SerializationTests.cpp @@ -49,7 +49,6 @@ End: Line: 1 Column: 1 -Origin: 128 Flags: 129 Documentation: 'Foo doc' ReturnType: 'int' @@ -121,6 +120,10 @@ return (arg.IncludeHeader == IncludeHeader) && (arg.References == References); } +auto readIndexFile(llvm::StringRef Text) { + return readIndexFile(Text, SymbolOrigin::Static); +} + TEST(SerializationTest, NoCrashOnEmptyYAML) { EXPECT_TRUE(bool(readIndexFile(""))); } @@ -143,7 +146,7 @@ EXPECT_EQ(Sym1.Documentation, "Foo doc"); EXPECT_EQ(Sym1.ReturnType, "int"); EXPECT_EQ(StringRef(Sym1.CanonicalDeclaration.FileURI), "file:///path/foo.h"); - EXPECT_EQ(Sym1.Origin, static_cast(1 << 7)); + EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static); EXPECT_EQ(static_cast(Sym1.Flags), 129); EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion); EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);