Index: clangd/CMakeLists.txt =================================================================== --- clangd/CMakeLists.txt +++ clangd/CMakeLists.txt @@ -21,6 +21,7 @@ DraftStore.cpp FindSymbols.cpp FileDistance.cpp + FS.cpp FuzzyMatch.cpp GlobalCompilationDatabase.cpp Headers.cpp Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -181,8 +181,6 @@ if (isCancelled()) return CB(llvm::make_error()); - auto PreambleData = IP->Preamble; - llvm::Optional SpecFuzzyFind; if (CodeCompleteOpts.Index && CodeCompleteOpts.SpeculativeIndexRequest) { SpecFuzzyFind.emplace(); @@ -195,10 +193,8 @@ // FIXME(ibiryukov): even if Preamble is non-null, we may want to check // both the old and the new version in case only one of them matches. CodeCompleteResult Result = clangd::codeComplete( - File, IP->Command, PreambleData ? &PreambleData->Preamble : nullptr, - PreambleData ? PreambleData->Includes : IncludeStructure(), - IP->Contents, Pos, FS, PCHs, CodeCompleteOpts, - SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr); + File, IP->Command, IP->Preamble, IP->Contents, Pos, FS, PCHs, + CodeCompleteOpts, SpecFuzzyFind ? SpecFuzzyFind.getPointer() : nullptr); { clang::clangd::trace::Span Tracer("Completion results callback"); CB(std::move(Result)); @@ -231,9 +227,8 @@ return CB(IP.takeError()); auto PreambleData = IP->Preamble; - CB(clangd::signatureHelp(File, IP->Command, - PreambleData ? &PreambleData->Preamble : nullptr, - IP->Contents, Pos, FS, PCHs, Index)); + CB(clangd::signatureHelp(File, IP->Command, PreambleData, IP->Contents, Pos, + FS, PCHs, Index)); }; // Unlike code completion, we wait for an up-to-date preamble here. Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -11,6 +11,7 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDUNIT_H #include "Diagnostics.h" +#include "FS.h" #include "Function.h" #include "Headers.h" #include "Path.h" @@ -45,7 +46,8 @@ // Stores Preamble and associated data. struct PreambleData { PreambleData(PrecompiledPreamble Preamble, std::vector Diags, - IncludeStructure Includes); + IncludeStructure Includes, + std::unique_ptr StatCache); tooling::CompileCommand CompileCommand; PrecompiledPreamble Preamble; @@ -53,6 +55,9 @@ // Processes like code completions and go-to-definitions will need #include // information, and their compile action skips preamble range. IncludeStructure Includes; + // Cache of FS operations performed when building the preamble. + // When reusing a preamble, this cache can be consumed to save IO. + std::unique_ptr StatCache; }; /// Information required to run clang, e.g. to parse AST or do code completion. Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -246,9 +246,10 @@ } PreambleData::PreambleData(PrecompiledPreamble Preamble, - std::vector Diags, IncludeStructure Includes) + std::vector Diags, IncludeStructure Includes, + std::unique_ptr StatCache) : Preamble(std::move(Preamble)), Diags(std::move(Diags)), - Includes(std::move(Includes)) {} + Includes(std::move(Includes)), StatCache(std::move(StatCache)) {} ParsedAST::ParsedAST(std::shared_ptr Preamble, std::unique_ptr Clang, @@ -334,9 +335,12 @@ // We proceed anyway, our lit-tests rely on results for non-existing working // dirs. } + + auto StatCache = llvm::make_unique(); auto BuiltPreamble = PrecompiledPreamble::Build( - CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Inputs.FS, PCHs, - StoreInMemory, SerializedDeclsCollector); + CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, + StatCache->getProducingFS(Inputs.FS), PCHs, StoreInMemory, + SerializedDeclsCollector); // When building the AST for the main file, we do want the function // bodies. @@ -347,7 +351,7 @@ FileName); return std::make_shared( std::move(*BuiltPreamble), PreambleDiagnostics.take(), - SerializedDeclsCollector.takeIncludes()); + SerializedDeclsCollector.takeIncludes(), std::move(StatCache)); } else { elog("Could not build a preamble for file {0}", FileName); return nullptr; @@ -361,15 +365,19 @@ trace::Span Tracer("BuildAST"); SPAN_ATTACH(Tracer, "File", FileName); - if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) { + auto VFS = Inputs.FS; + if (Preamble && Preamble->StatCache) + VFS = Preamble->StatCache->getConsumingFS(std::move(VFS)); + if (VFS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) { log("Couldn't set working directory when building the preamble."); // We proceed anyway, our lit-tests rely on results for non-existing working // dirs. } - return ParsedAST::build( - llvm::make_unique(*Invocation), Preamble, - llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), PCHs, Inputs.FS); + return ParsedAST::build(llvm::make_unique(*Invocation), + Preamble, + llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), + PCHs, std::move(VFS)); } SourceLocation clangd::getBeginningOfIdentifier(ParsedAST &Unit, Index: clangd/CodeComplete.h =================================================================== --- clangd/CodeComplete.h +++ clangd/CodeComplete.h @@ -16,6 +16,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H +#include "ClangdUnit.h" #include "Headers.h" #include "Logger.h" #include "Path.h" @@ -101,6 +102,13 @@ /// Whether to generate snippets for function arguments on code-completion. /// Needs snippets to be enabled as well. bool EnableFunctionArgSnippets = true; + + /// Whether to include index symbols that are not defined in the scopes + /// visible from the code completion point. This applies in contexts without + /// explicit scope qualifiers. + /// + /// Such completions can insert scope qualifiers. + bool AllScopes = false; }; // Semi-structured representation of a code-complete suggestion for our C++ API. @@ -214,8 +222,7 @@ /// destroyed when the async request finishes. CodeCompleteResult codeComplete(PathRef FileName, const tooling::CompileCommand &Command, - PrecompiledPreamble const *Preamble, - const IncludeStructure &PreambleInclusions, + const PreambleData *Preamble, StringRef Contents, Position Pos, IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, @@ -225,8 +232,8 @@ /// Get signature help at a specified \p Pos in \p FileName. SignatureHelp signatureHelp(PathRef FileName, const tooling::CompileCommand &Command, - PrecompiledPreamble const *Preamble, - StringRef Contents, Position Pos, + const PreambleData *Preamble, StringRef Contents, + Position Pos, IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, const SymbolIndex *Index); Index: clangd/CodeComplete.cpp =================================================================== --- clangd/CodeComplete.cpp +++ clangd/CodeComplete.cpp @@ -20,6 +20,7 @@ #include "CodeComplete.h" #include "AST.h" +#include "ClangdUnit.h" #include "CodeCompletionStrings.h" #include "Compiler.h" #include "Diagnostics.h" @@ -44,6 +45,7 @@ #include "clang/Sema/CodeCompleteConsumer.h" #include "clang/Sema/Sema.h" #include "clang/Tooling/Core/Replacement.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Error.h" @@ -330,6 +332,7 @@ struct CodeCompletionBuilder { CodeCompletionBuilder(ASTContext &ASTCtx, const CompletionCandidate &C, CodeCompletionString *SemaCCS, + llvm::ArrayRef QueryScopes, const IncludeInserter &Includes, StringRef FileName, CodeCompletionContext::Kind ContextKind, const CodeCompleteOptions &Opts) @@ -374,6 +377,18 @@ Completion.Kind = toCompletionItemKind(C.IndexResult->SymInfo.Kind); if (Completion.Name.empty()) Completion.Name = C.IndexResult->Name; + // If the completion was visible to Sema, no qualifier is needed. This + // avoids unneeded qualifiers in cases like with `using ns::X`. + if (Completion.RequiredQualifier.empty() && !C.SemaResult) { + StringRef ShortestQualifier = C.IndexResult->Scope; + for (StringRef Scope : QueryScopes) { + StringRef Qualifier = C.IndexResult->Scope; + if (Qualifier.consume_front(Scope) && + Qualifier.size() < ShortestQualifier.size()) + ShortestQualifier = Qualifier; + } + Completion.RequiredQualifier = ShortestQualifier; + } Completion.Deprecated |= (C.IndexResult->Flags & Symbol::Deprecated); } @@ -604,9 +619,11 @@ } }; -// Get all scopes that will be queried in indexes. -std::vector getQueryScopes(CodeCompletionContext &CCContext, - const SourceManager &SM) { +// Get all scopes that will be queried in indexes and whether symbols from +// any scope is allowed. +std::pair, bool> +getQueryScopes(CodeCompletionContext &CCContext, const SourceManager &SM, + const CodeCompleteOptions &Opts) { auto GetAllAccessibleScopes = [](CodeCompletionContext &CCContext) { SpecifiedScope Info; for (auto *Context : CCContext.getVisitedContexts()) { @@ -627,13 +644,15 @@ // FIXME: Capture scopes and use for scoring, for example, // "using namespace std; namespace foo {v^}" => // foo::value > std::vector > boost::variant - return GetAllAccessibleScopes(CCContext).scopesForIndexQuery(); + auto Scopes = GetAllAccessibleScopes(CCContext).scopesForIndexQuery(); + // Allow AllScopes completion only for there is no explicit scope qualifier. + return {Scopes, Opts.AllScopes}; } // Qualified completion ("std::vec^"), we have two cases depending on whether // the qualifier can be resolved by Sema. if ((*SS)->isValid()) { // Resolved qualifier. - return GetAllAccessibleScopes(CCContext).scopesForIndexQuery(); + return {GetAllAccessibleScopes(CCContext).scopesForIndexQuery(), false}; } // Unresolved qualifier. @@ -651,7 +670,7 @@ if (!Info.UnresolvedQualifier->empty()) *Info.UnresolvedQualifier += "::"; - return Info.scopesForIndexQuery(); + return {Info.scopesForIndexQuery(), false}; } // Should we perform index-based completion in a context of the specified kind? @@ -1035,7 +1054,7 @@ struct SemaCompleteInput { PathRef FileName; const tooling::CompileCommand &Command; - PrecompiledPreamble const *Preamble; + const PreambleData *Preamble; StringRef Contents; Position Pos; IntrusiveRefCntPtr VFS; @@ -1059,12 +1078,15 @@ // working dirs. } + IntrusiveRefCntPtr VFS = Input.VFS; + if (Input.Preamble && Input.Preamble->StatCache) + VFS = Input.Preamble->StatCache->getConsumingFS(std::move(VFS)); IgnoreDiagnostics DummyDiagsConsumer; auto CI = createInvocationFromCommandLine( ArgStrs, CompilerInstance::createDiagnostics(new DiagnosticOptions, &DummyDiagsConsumer, false), - Input.VFS); + VFS); if (!CI) { elog("Couldn't create CompilerInvocation"); return false; @@ -1103,8 +1125,10 @@ // NOTE: we must call BeginSourceFile after prepareCompilerInstance. Otherwise // the remapped buffers do not get freed. auto Clang = prepareCompilerInstance( - std::move(CI), CompletingInPreamble ? nullptr : Input.Preamble, - std::move(ContentsBuffer), std::move(Input.PCHs), std::move(Input.VFS), + std::move(CI), + (Input.Preamble && !CompletingInPreamble) ? &Input.Preamble->Preamble + : nullptr, + std::move(ContentsBuffer), std::move(Input.PCHs), std::move(VFS), DummyDiagsConsumer); Clang->getPreprocessorOpts().SingleFileParseMode = CompletingInPreamble; Clang->setCodeCompletionConsumer(Consumer.release()); @@ -1262,8 +1286,10 @@ CompletionRecorder *Recorder = nullptr; int NSema = 0, NIndex = 0, NBoth = 0; // Counters for logging. bool Incomplete = false; // Would more be available with a higher limit? - llvm::Optional Filter; // Initialized once Sema runs. - std::vector QueryScopes; // Initialized once Sema runs. + llvm::Optional Filter; // Initialized once Sema runs. + std::vector QueryScopes; // Initialized once Sema runs. + // Whether to query symbols from any scope. Initialized once Sema runs. + bool AllScopes = false; // Include-insertion and proximity scoring rely on the include structure. // This is available after Sema has run. llvm::Optional Inserter; // Available during runWithSema. @@ -1339,9 +1365,9 @@ Inserter.reset(); // Make sure this doesn't out-live Clang. SPAN_ATTACH(Tracer, "sema_completion_kind", getCompletionKindString(Recorder->CCContext.getKind())); - log("Code complete: sema context {0}, query scopes [{1}]", + log("Code complete: sema context {0}, query scopes [{1}] (AnyScope={2})", getCompletionKindString(Recorder->CCContext.getKind()), - llvm::join(QueryScopes.begin(), QueryScopes.end(), ",")); + llvm::join(QueryScopes.begin(), QueryScopes.end(), ","), AllScopes); }); Recorder = RecorderOwner.get(); @@ -1387,8 +1413,8 @@ } Filter = FuzzyMatcher( Recorder->CCSema->getPreprocessor().getCodeCompletionFilter()); - QueryScopes = getQueryScopes(Recorder->CCContext, - Recorder->CCSema->getSourceManager()); + std::tie(QueryScopes, AllScopes) = getQueryScopes( + Recorder->CCContext, Recorder->CCSema->getSourceManager(), Opts); // Sema provides the needed context to query the index. // FIXME: in addition to querying for extra/overlapping symbols, we should // explicitly request symbols corresponding to Sema results. @@ -1428,6 +1454,7 @@ Req.Query = Filter->pattern(); Req.RestrictForCodeCompletion = true; Req.Scopes = QueryScopes; + Req.AnyScope = AllScopes; // FIXME: we should send multiple weighted paths here. Req.ProximityPaths.push_back(FileName); vlog("Code complete: fuzzyFind({0:2})", toJSON(Req)); @@ -1538,6 +1565,8 @@ Relevance.Context = Recorder->CCContext.getKind(); Relevance.Query = SymbolRelevanceSignals::CodeComplete; Relevance.FileProximityMatch = FileProximity.getPointer(); + // FIXME: incorparate scope proximity into relevance score. + auto &First = Bundle.front(); if (auto FuzzyScore = fuzzyScore(First)) Relevance.NameMatch = *FuzzyScore; @@ -1587,8 +1616,8 @@ : nullptr; if (!Builder) Builder.emplace(Recorder->CCSema->getASTContext(), Item, SemaCCS, - *Inserter, FileName, Recorder->CCContext.getKind(), - Opts); + QueryScopes, *Inserter, FileName, + Recorder->CCContext.getKind(), Opts); else Builder->add(Item, SemaCCS); } @@ -1623,19 +1652,20 @@ CodeCompleteResult codeComplete(PathRef FileName, const tooling::CompileCommand &Command, - PrecompiledPreamble const *Preamble, - const IncludeStructure &PreambleInclusions, StringRef Contents, - Position Pos, IntrusiveRefCntPtr VFS, + const PreambleData *Preamble, StringRef Contents, Position Pos, + IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, CodeCompleteOptions Opts, SpeculativeFuzzyFind *SpecFuzzyFind) { - return CodeCompleteFlow(FileName, PreambleInclusions, SpecFuzzyFind, Opts) + return CodeCompleteFlow(FileName, + Preamble ? Preamble->Includes : IncludeStructure(), + SpecFuzzyFind, Opts) .run({FileName, Command, Preamble, Contents, Pos, VFS, PCHs}); } SignatureHelp signatureHelp(PathRef FileName, const tooling::CompileCommand &Command, - PrecompiledPreamble const *Preamble, - StringRef Contents, Position Pos, + const PreambleData *Preamble, StringRef Contents, + Position Pos, IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, const SymbolIndex *Index) { Index: clangd/FS.h =================================================================== --- /dev/null +++ clangd/FS.h @@ -0,0 +1,65 @@ +//===--- FS.h - File system related utils ------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_FS_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_FS_H + +#include "clang/Basic/VirtualFileSystem.h" +#include "llvm/ADT/Optional.h" + +namespace clang { +namespace clangd { + +/// Records status information for files open()ed or stat()ed during preamble +/// build, so we can avoid stat()s on the underlying FS when reusing the +/// preamble. For example, code completion can re-stat files when getting FileID +/// for source locations stored in preamble (e.g. checking whether a location is +/// in the main file). +/// +/// The cache is keyed by absolute path of file name in cached status, as this +/// is what preamble stores. +/// +/// The cache is not thread-safe when updates happen, so the use pattern should +/// be: +/// - One FS writes to the cache from one thread (or several but strictly +/// sequenced), e.g. when building preamble. +/// - Sequence point (no writes after this point, no reads before). +/// - Several FSs can read from the cache, e.g. code completions. +/// +/// Note that the cache is only valid when reusing preamble. +class PreambleFileStatusCache { +public: + void update(const vfs::FileSystem &FS, vfs::Status S); + /// \p Path is a path stored in preamble. + llvm::Optional lookup(llvm::StringRef Path) const; + + /// Returns a VFS that collects file status. + /// Only cache stats for files that exist because + /// 1) we only care about existing files when reusing preamble, unlike + /// building preamble. + /// 2) we use the file name in the Status as the cache key. + /// + /// Note that the returned VFS should not outlive the cache. + IntrusiveRefCntPtr + getProducingFS(IntrusiveRefCntPtr FS); + + /// Returns a VFS that uses the cache collected. + /// + /// Note that the returned VFS should not outlive the cache. + IntrusiveRefCntPtr + getConsumingFS(IntrusiveRefCntPtr FS) const; + +private: + llvm::StringMap StatCache; +}; + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_FS_H Index: clangd/FS.cpp =================================================================== --- /dev/null +++ clangd/FS.cpp @@ -0,0 +1,92 @@ +//===--- FS.cpp - File system related utils ----------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FS.h" +#include "clang/Basic/VirtualFileSystem.h" +#include "llvm/ADT/None.h" + +namespace clang { +namespace clangd { + +void PreambleFileStatusCache::update(const vfs::FileSystem &FS, vfs::Status S) { + SmallString<32> PathStore(S.getName()); + if (auto Err = FS.makeAbsolute(PathStore)) + return; + // Stores the latest status in cache as it can change in a preamble build. + StatCache.insert({PathStore, std::move(S)}); +} + +llvm::Optional +PreambleFileStatusCache::lookup(llvm::StringRef File) const { + auto I = StatCache.find(File); + if (I != StatCache.end()) + return I->getValue(); + return llvm::None; +} + +IntrusiveRefCntPtr PreambleFileStatusCache::getProducingFS( + IntrusiveRefCntPtr FS) { + // This invalidates old status in cache if files are re-`open()`ed or + // re-`stat()`ed in case file status has changed during preamble build. + class CollectFS : public vfs::ProxyFileSystem { + public: + CollectFS(IntrusiveRefCntPtr FS, + PreambleFileStatusCache &StatCache) + : ProxyFileSystem(std::move(FS)), StatCache(StatCache) {} + + llvm::ErrorOr> + openFileForRead(const Twine &Path) override { + auto File = getUnderlyingFS().openFileForRead(Path); + if (!File || !*File) + return File; + // Eagerly stat opened file, as the followup `status` call on the file + // doesn't necessarily go through this FS. This puts some extra work on + // preamble build, but it should be worth it as preamble can be reused + // many times (e.g. code completion) and the repeated status call is + // likely to be cached in the underlying file system anyway. + if (auto S = File->get()->status()) + StatCache.update(getUnderlyingFS(), std::move(*S)); + return File; + } + + llvm::ErrorOr status(const Twine &Path) override { + auto S = getUnderlyingFS().status(Path); + if (S) + StatCache.update(getUnderlyingFS(), *S); + return S; + } + + private: + PreambleFileStatusCache &StatCache; + }; + return IntrusiveRefCntPtr(new CollectFS(std::move(FS), *this)); +} + +IntrusiveRefCntPtr PreambleFileStatusCache::getConsumingFS( + IntrusiveRefCntPtr FS) const { + class CacheVFS : public vfs::ProxyFileSystem { + public: + CacheVFS(IntrusiveRefCntPtr FS, + const PreambleFileStatusCache &StatCache) + : ProxyFileSystem(std::move(FS)), StatCache(StatCache) {} + + llvm::ErrorOr status(const Twine &Path) override { + if (auto S = StatCache.lookup(Path.str())) + return *S; + return getUnderlyingFS().status(Path); + } + + private: + const PreambleFileStatusCache &StatCache; + }; + return IntrusiveRefCntPtr(new CacheVFS(std::move(FS), *this)); +} + +} // namespace clangd +} // namespace clang Index: clangd/index/Index.h =================================================================== --- clangd/index/Index.h +++ clangd/index/Index.h @@ -428,7 +428,13 @@ /// namespace xyz::abc. /// /// The global scope is "", a top level scope is "foo::", etc. + /// FIXME: drop the special case for empty list, which is the same as + /// `AnyScope = true`. + /// FIXME: support scope proximity. std::vector Scopes; + /// If set to true, allow symbols from any scope. Scopes explicitly listed + /// above will be ranked higher. + bool AnyScope = false; /// \brief The number of top candidates to return. The index may choose to /// return more than this, e.g. if it doesn't know which candidates are best. llvm::Optional Limit; Index: clangd/index/MemIndex.cpp =================================================================== --- clangd/index/MemIndex.cpp +++ clangd/index/MemIndex.cpp @@ -11,6 +11,7 @@ #include "FuzzyMatch.h" #include "Logger.h" #include "Quality.h" +#include "Trace.h" namespace clang { namespace clangd { @@ -28,6 +29,7 @@ llvm::function_ref Callback) const { assert(!StringRef(Req.Query).contains("::") && "There must be no :: in query."); + trace::Span Tracer("MemIndex fuzzyFind"); TopN> Top( Req.Limit ? *Req.Limit : std::numeric_limits::max()); @@ -37,7 +39,8 @@ const Symbol *Sym = Pair.second; // Exact match against all possible scopes. - if (!Req.Scopes.empty() && !llvm::is_contained(Req.Scopes, Sym->Scope)) + if (!Req.AnyScope && !Req.Scopes.empty() && + !llvm::is_contained(Req.Scopes, Sym->Scope)) continue; if (Req.RestrictForCodeCompletion && !(Sym->Flags & Symbol::IndexedForCodeCompletion)) @@ -47,13 +50,16 @@ if (Top.push({*Score * quality(*Sym), Sym})) More = true; // An element with smallest score was discarded. } - for (const auto &Item : std::move(Top).items()) + auto Results = std::move(Top).items(); + SPAN_ATTACH(Tracer, "results", static_cast(Results.size())); + for (const auto &Item : Results) Callback(*Item.second); return More; } void MemIndex::lookup(const LookupRequest &Req, llvm::function_ref Callback) const { + trace::Span Tracer("MemIndex lookup"); for (const auto &ID : Req.IDs) { auto I = Index.find(ID); if (I != Index.end()) @@ -63,6 +69,7 @@ void MemIndex::refs(const RefsRequest &Req, llvm::function_ref Callback) const { + trace::Span Tracer("MemIndex refs"); for (const auto &ReqID : Req.IDs) { auto SymRefs = Refs.find(ReqID); if (SymRefs == Refs.end()) Index: clangd/index/Merge.cpp =================================================================== --- clangd/index/Merge.cpp +++ clangd/index/Merge.cpp @@ -9,6 +9,7 @@ #include "Merge.h" #include "Logger.h" +#include "Trace.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/raw_ostream.h" @@ -38,19 +39,31 @@ // a) if it's not in the dynamic slab, yield it directly // b) if it's in the dynamic slab, merge it and yield the result // 3) now yield all the dynamic symbols we haven't processed. + trace::Span Tracer("MergedIndex fuzzyFind"); bool More = false; // We'll be incomplete if either source was. SymbolSlab::Builder DynB; - More |= Dynamic->fuzzyFind(Req, [&](const Symbol &S) { DynB.insert(S); }); + unsigned DynamicCount = 0; + unsigned StaticCount = 0; + unsigned MergedCount = 0; + More |= Dynamic->fuzzyFind(Req, [&](const Symbol &S) { + ++DynamicCount; + DynB.insert(S); + }); SymbolSlab Dyn = std::move(DynB).build(); DenseSet SeenDynamicSymbols; More |= Static->fuzzyFind(Req, [&](const Symbol &S) { auto DynS = Dyn.find(S.ID); + ++StaticCount; if (DynS == Dyn.end()) return Callback(S); + ++MergedCount; SeenDynamicSymbols.insert(S.ID); Callback(mergeSymbol(*DynS, S)); }); + SPAN_ATTACH(Tracer, "dynamic", DynamicCount); + SPAN_ATTACH(Tracer, "static", StaticCount); + SPAN_ATTACH(Tracer, "merged", MergedCount); for (const Symbol &S : Dyn) if (!SeenDynamicSymbols.count(S.ID)) Callback(S); @@ -60,6 +73,7 @@ void lookup(const LookupRequest &Req, llvm::function_ref Callback) const override { + trace::Span Tracer("MergedIndex lookup"); SymbolSlab::Builder B; Dynamic->lookup(Req, [&](const Symbol &S) { B.insert(S); }); @@ -80,6 +94,7 @@ void refs(const RefsRequest &Req, llvm::function_ref Callback) const override { + trace::Span Tracer("MergedIndex refs"); // We don't want duplicated refs from the static/dynamic indexes, // and we can't reliably duplicate them because offsets may differ slightly. // We consider the dynamic index authoritative and report all its refs, Index: clangd/index/dex/Dex.cpp =================================================================== --- clangd/index/dex/Dex.cpp +++ clangd/index/dex/Dex.cpp @@ -12,6 +12,9 @@ #include "FuzzyMatch.h" #include "Logger.h" #include "Quality.h" +#include "Trace.h" +#include "index/Index.h" +#include "index/dex/Iterator.h" #include "llvm/ADT/StringSet.h" #include #include @@ -139,6 +142,8 @@ llvm::function_ref Callback) const { assert(!StringRef(Req.Query).contains("::") && "There must be no :: in query."); + // FIXME: attach the query tree to the trace span. + trace::Span Tracer("Dex fuzzyFind"); FuzzyMatcher Filter(Req.Query); bool More = false; @@ -163,6 +168,10 @@ if (It != InvertedIndex.end()) ScopeIterators.push_back(It->second.iterator()); } + if (Req.AnyScope) + ScopeIterators.push_back(createBoost(createTrue(Symbols.size()), + ScopeIterators.empty() ? 1.0 : 0.2)); + // Add OR iterator for scopes if there are any Scope Iterators. if (!ScopeIterators.empty()) TopLevelChildren.push_back(createOr(move(ScopeIterators))); @@ -228,6 +237,7 @@ void Dex::lookup(const LookupRequest &Req, llvm::function_ref Callback) const { + trace::Span Tracer("Dex lookup"); for (const auto &ID : Req.IDs) { auto I = LookupTable.find(ID); if (I != LookupTable.end()) @@ -237,6 +247,7 @@ void Dex::refs(const RefsRequest &Req, llvm::function_ref Callback) const { + trace::Span Tracer("Dex refs"); log("refs is not implemented."); } Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -136,6 +136,15 @@ "enabled separatedly."), llvm::cl::init(true), llvm::cl::Hidden); +static llvm::cl::opt AllScopesCompletion( + "all-scopes-completion", + llvm::cl::desc( + "If set to true, code completion will include index symbols that are " + "not defined in the scopes (e.g. " + "namespaces) visible from the code completion point. Such completions " + "can insert scope qualifiers."), + llvm::cl::init(false), llvm::cl::Hidden); + static llvm::cl::opt ShowOrigins("debug-origin", llvm::cl::desc("Show origins of completion items"), @@ -304,6 +313,7 @@ } CCOpts.SpeculativeIndexRequest = Opts.StaticIndex; CCOpts.EnableFunctionArgSnippets = EnableFunctionArgSnippets; + CCOpts.AllScopes = AllScopesCompletion; // Initialize and run ClangdLSPServer. ClangdLSPServer LSPServer( Index: unittests/clangd/CMakeLists.txt =================================================================== --- unittests/clangd/CMakeLists.txt +++ unittests/clangd/CMakeLists.txt @@ -21,6 +21,7 @@ FileDistanceTests.cpp FileIndexTests.cpp FindSymbolsTests.cpp + FSTests.cpp FuzzyMatchTests.cpp GlobalCompilationDatabaseTests.cpp HeadersTests.cpp Index: unittests/clangd/ClangdTests.cpp =================================================================== --- unittests/clangd/ClangdTests.cpp +++ unittests/clangd/ClangdTests.cpp @@ -963,6 +963,71 @@ Field(&CodeCompletion::Name, "baz"))); } +// Check that running code completion doesn't stat() a bunch of files from the +// preamble again. (They should be using the preamble's stat-cache) +TEST(ClangdTests, PreambleVFSStatCache) { + class ListenStatsFSProvider : public FileSystemProvider { + public: + ListenStatsFSProvider(llvm::StringMap &CountStats) + : CountStats(CountStats) {} + + IntrusiveRefCntPtr getFileSystem() override { + class ListenStatVFS : public vfs::ProxyFileSystem { + public: + ListenStatVFS(IntrusiveRefCntPtr FS, + llvm::StringMap &CountStats) + : ProxyFileSystem(std::move(FS)), CountStats(CountStats) {} + + llvm::ErrorOr> + openFileForRead(const Twine &Path) override { + ++CountStats[llvm::sys::path::filename(Path.str())]; + return ProxyFileSystem::openFileForRead(Path); + } + llvm::ErrorOr status(const Twine &Path) override { + ++CountStats[llvm::sys::path::filename(Path.str())]; + return ProxyFileSystem::status(Path); + } + + private: + llvm::StringMap &CountStats; + }; + + return IntrusiveRefCntPtr( + new ListenStatVFS(buildTestFS(Files), CountStats)); + } + + // If relative paths are used, they are resolved with testPath(). + llvm::StringMap Files; + llvm::StringMap &CountStats; + }; + + llvm::StringMap CountStats; + ListenStatsFSProvider FS(CountStats); + ErrorCheckingDiagConsumer DiagConsumer; + MockCompilationDatabase CDB; + ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + + auto SourcePath = testPath("foo.cpp"); + auto HeaderPath = testPath("foo.h"); + FS.Files[HeaderPath] = "struct TestSym {};"; + Annotations Code(R"cpp( + #include "foo.h" + + int main() { + TestSy^ + })cpp"); + + runAddDocument(Server, SourcePath, Code.code()); + + EXPECT_EQ(CountStats["foo.h"], 1u); + auto Completions = cantFail(runCodeComplete(Server, SourcePath, Code.point(), + clangd::CodeCompleteOptions())) + .Completions; + EXPECT_EQ(CountStats["foo.h"], 1u); + EXPECT_THAT(Completions, + ElementsAre(Field(&CodeCompletion::Name, "TestSym"))); +} + } // namespace } // namespace clangd } // namespace clang Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -2093,6 +2093,57 @@ Has("bar.h\"", CompletionItemKind::File))); } +TEST(CompletionTest, NoAllScopesCompletionWhenQualified) { + clangd::CodeCompleteOptions Opts = {}; + Opts.AllScopes = true; + + auto Results = completions( + R"cpp( + void f() { na::Clangd^ } + )cpp", + {cls("na::ClangdA"), cls("nx::ClangdX"), cls("Clangd3")}, Opts); + EXPECT_THAT(Results.Completions, + UnorderedElementsAre( + AllOf(Qualifier(""), Scope("na::"), Named("ClangdA")))); +} + +TEST(CompletionTest, AllScopesCompletion) { + clangd::CodeCompleteOptions Opts = {}; + Opts.AllScopes = true; + + auto Results = completions( + R"cpp( + namespace na { + void f() { Clangd^ } + } + )cpp", + {cls("nx::Clangd1"), cls("ny::Clangd2"), cls("Clangd3"), + cls("na::nb::Clangd4")}, + Opts); + EXPECT_THAT( + Results.Completions, + UnorderedElementsAre(AllOf(Qualifier("nx::"), Named("Clangd1")), + AllOf(Qualifier("ny::"), Named("Clangd2")), + AllOf(Qualifier(""), Scope(""), Named("Clangd3")), + AllOf(Qualifier("nb::"), Named("Clangd4")))); +} + +TEST(CompletionTest, NoQualifierIfShadowed) { + clangd::CodeCompleteOptions Opts = {}; + Opts.AllScopes = true; + + auto Results = completions(R"cpp( + namespace nx { class Clangd1 {}; } + using nx::Clangd1; + void f() { Clangd^ } + )cpp", + {cls("nx::Clangd1"), cls("nx::Clangd2")}, Opts); + // Although Clangd1 is from another namespace, Sema tells us it's in-scope and + // needs no qualifier. + EXPECT_THAT(Results.Completions, + UnorderedElementsAre(AllOf(Qualifier(""), Named("Clangd1")), + AllOf(Qualifier("nx::"), Named("Clangd2")))); +} } // namespace } // namespace clangd Index: unittests/clangd/DexTests.cpp =================================================================== --- unittests/clangd/DexTests.cpp +++ unittests/clangd/DexTests.cpp @@ -523,6 +523,17 @@ EXPECT_THAT(match(*I, Req), UnorderedElementsAre("a::y1")); } +TEST(DexTest, WildcardScope) { + auto I = + Dex::build(generateSymbols({"a::y1", "a::b::y2", "c::y3"}), URISchemes); + FuzzyFindRequest Req; + Req.Query = "y"; + Req.Scopes = {"a::"}; + Req.AnyScope = true; + EXPECT_THAT(match(*I, Req), + UnorderedElementsAre("a::y1", "a::b::y2", "c::y3")); +} + TEST(DexTest, IgnoreCases) { auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), URISchemes); FuzzyFindRequest Req; Index: unittests/clangd/FSTests.cpp =================================================================== --- /dev/null +++ unittests/clangd/FSTests.cpp @@ -0,0 +1,46 @@ +//===-- FSTests.cpp - File system related tests -----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FS.h" +#include "TestFS.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace clang { +namespace clangd { +namespace { + +TEST(FSTests, PreambleStatusCache) { + llvm::StringMap Files; + Files["x"] = ""; + Files["y"] = ""; + auto FS = buildTestFS(Files); + FS->setCurrentWorkingDirectory(testRoot()); + + PreambleFileStatusCache StatCache; + auto ProduceFS = StatCache.getProducingFS(FS); + EXPECT_TRUE(ProduceFS->openFileForRead("x")); + EXPECT_TRUE(ProduceFS->status("y")); + + EXPECT_TRUE(StatCache.lookup(testPath("x")).hasValue()); + EXPECT_TRUE(StatCache.lookup(testPath("y")).hasValue()); + + vfs::Status S("fake", llvm::sys::fs::UniqueID(0, 0), + std::chrono::system_clock::now(), 0, 0, 1024, + llvm::sys::fs::file_type::regular_file, llvm::sys::fs::all_all); + StatCache.update(*FS, S); + auto ConsumeFS = StatCache.getConsumingFS(FS); + auto Cached = ConsumeFS->status(testPath("fake")); + EXPECT_TRUE(Cached); + EXPECT_EQ(Cached->getName(), S.getName()); +} + +} // namespace +} // namespace clangd +} // namespace clang Index: unittests/clangd/TestFS.cpp =================================================================== --- unittests/clangd/TestFS.cpp +++ unittests/clangd/TestFS.cpp @@ -23,6 +23,7 @@ llvm::StringMap const &Timestamps) { IntrusiveRefCntPtr MemFS( new vfs::InMemoryFileSystem); + MemFS->setCurrentWorkingDirectory(testRoot()); for (auto &FileAndContents : Files) { StringRef File = FileAndContents.first(); MemFS->addFile(