diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -119,6 +119,7 @@ index/Ref.cpp index/Relation.cpp index/Serialization.cpp + index/StdLib.cpp index/Symbol.cpp index/SymbolCollector.cpp index/SymbolID.cpp diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -14,7 +14,6 @@ #include "FindSymbols.h" #include "Format.h" #include "HeaderSourceSwitch.h" -#include "Headers.h" #include "InlayHints.h" #include "ParsedAST.h" #include "Preamble.h" @@ -27,15 +26,14 @@ #include "index/CanonicalIncludes.h" #include "index/FileIndex.h" #include "index/Merge.h" +#include "index/StdLib.h" #include "refactor/Rename.h" #include "refactor/Tweak.h" #include "support/Logger.h" -#include "support/Markup.h" #include "support/MemoryTree.h" #include "support/ThreadsafeFS.h" #include "support/Trace.h" #include "clang/Format/Format.h" -#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Lex/Preprocessor.h" #include "clang/Tooling/CompilationDatabase.h" @@ -43,14 +41,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" -#include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -67,16 +60,39 @@ // Update the FileIndex with new ASTs and plumb the diagnostics responses. struct UpdateIndexCallbacks : public ParsingCallbacks { UpdateIndexCallbacks(FileIndex *FIndex, - ClangdServer::Callbacks *ServerCallbacks) - : FIndex(FIndex), ServerCallbacks(ServerCallbacks) {} + ClangdServer::Callbacks *ServerCallbacks, + const ThreadsafeFS &TFS, bool Sync) + : FIndex(FIndex), ServerCallbacks(ServerCallbacks), TFS(TFS) { + if (!Sync) + Tasks.emplace(); + } - void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx, + void onPreambleAST(PathRef Path, llvm::StringRef Version, + const CompilerInvocation &CI, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &CanonIncludes) override { + // If this preamble uses a standard library we haven't seen yet, index it. + if (auto Loc = Stdlib.add(*CI.getLangOpts(), PP->getHeaderSearchInfo())) + indexStdlib(CI, std::move(*Loc)); + if (FIndex) FIndex->updatePreamble(Path, Version, Ctx, std::move(PP), CanonIncludes); } + void indexStdlib(const CompilerInvocation &CI, StdLibLocation Loc) { + auto Task = [this, LO(*CI.getLangOpts()), Loc(std::move(Loc)), + CI(std::make_unique(CI))]() mutable { + IndexFileIn IF; + IF.Symbols = indexStandardLibrary(std::move(CI), Loc, TFS); + if (Stdlib.isBest(LO)) + FIndex->updatePreamble(std::move(IF)); + }; + if (Tasks) + Tasks->runAsync("IndexStdlib", std::move(Task)); + else + Task(); + } + void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) override { if (FIndex) FIndex->updateMain(Path, AST); @@ -111,6 +127,9 @@ private: FileIndex *FIndex; ClangdServer::Callbacks *ServerCallbacks; + const ThreadsafeFS &TFS; + StdLibSet Stdlib; + llvm::Optional Tasks; }; class DraftStoreFS : public ThreadsafeFS { @@ -163,9 +182,10 @@ // Pass a callback into `WorkScheduler` to extract symbols from a newly // parsed file and rebuild the file index synchronously each time an AST // is parsed. - WorkScheduler.emplace( - CDB, TUScheduler::Options(Opts), - std::make_unique(DynamicIdx.get(), Callbacks)); + WorkScheduler.emplace(CDB, TUScheduler::Options(Opts), + std::make_unique( + DynamicIdx.get(), Callbacks, TFS, + /*Sync=*/Opts.AsyncThreadsCount == 0)); // Adds an index to the stack, at higher priority than existing indexes. auto AddIndex = [&](SymbolIndex *Idx) { if (this->Index != nullptr) { @@ -895,7 +915,7 @@ // It's safe to pass in the TU, as dumpAST() does not // deserialize the preamble. auto Node = DynTypedNode::create( - *Inputs->AST.getASTContext().getTranslationUnitDecl()); + *Inputs->AST.getASTContext().getTranslationUnitDecl()); return CB(dumpAST(Node, Inputs->AST.getTokens(), Inputs->AST.getASTContext())); } diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h --- a/clang-tools-extra/clangd/Config.h +++ b/clang-tools-extra/clangd/Config.h @@ -79,11 +79,12 @@ /// forward-slashes. std::string MountPoint; }; - /// Controls background-index behavior. + /// Controls index behavior. struct { - /// Whether this TU should be indexed. + /// Whether this TU should be background-indexed. BackgroundPolicy Background = BackgroundPolicy::Build; ExternalIndexSpec External; + bool StandardLibrary = false; } Index; enum UnusedIncludesPolicy { Strict, None }; diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp --- a/clang-tools-extra/clangd/ConfigCompile.cpp +++ b/clang-tools-extra/clangd/ConfigCompile.cpp @@ -321,6 +321,11 @@ } if (F.External) compile(std::move(**F.External), F.External->Range); + if (F.StandardLibrary) + Out.Apply.push_back( + [Val(**F.StandardLibrary)](const Params &, Config &C) { + C.Index.StandardLibrary = Val; + }); } void compile(Fragment::IndexBlock::ExternalBlock &&External, diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h --- a/clang-tools-extra/clangd/ConfigFragment.h +++ b/clang-tools-extra/clangd/ConfigFragment.h @@ -191,6 +191,9 @@ llvm::Optional> MountPoint; }; llvm::Optional> External; + // Whether the standard library visible from this file should be indexed. + // This makes all standard library symbols available, included or not. + llvm::Optional> StandardLibrary; }; IndexBlock Index; diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp --- a/clang-tools-extra/clangd/ConfigYAML.cpp +++ b/clang-tools-extra/clangd/ConfigYAML.cpp @@ -167,6 +167,10 @@ F.External.emplace(std::move(External)); F.External->Range = N.getSourceRange(); }); + Dict.handle("StandardLibrary", [&](Node &N) { + if (auto StandardLibrary = boolValue(N, "StandardLibrary")) + F.StandardLibrary = *StandardLibrary; + }); Dict.parse(N); } @@ -194,12 +198,8 @@ void parse(Fragment::CompletionBlock &F, Node &N) { DictParser Dict("Completion", this); Dict.handle("AllScopes", [&](Node &N) { - if (auto Value = scalarValue(N, "AllScopes")) { - if (auto AllScopes = llvm::yaml::parseBool(**Value)) - F.AllScopes = *AllScopes; - else - warning("AllScopes should be a boolean", N); - } + if (auto AllScopes = boolValue(N, "AllScopes")) + F.AllScopes = *AllScopes; }); Dict.parse(N); } @@ -334,6 +334,16 @@ return Result; } + llvm::Optional> boolValue(Node &N, llvm::StringRef Desc) { + if (auto Scalar = scalarValue(N, Desc)) { + if (auto StandardLibrary = llvm::yaml::parseBool(**Scalar)) + return Located(*StandardLibrary, Scalar->Range); + else + warning(Desc + " should be a boolean", N); + } + return llvm::None; + } + // Report a "hard" error, reflecting a config file that can never be valid. void error(const llvm::Twine &Msg, llvm::SMRange Range) { HadError = true; diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h --- a/clang-tools-extra/clangd/TUScheduler.h +++ b/clang-tools-extra/clangd/TUScheduler.h @@ -134,7 +134,7 @@ /// contains only AST nodes from the #include directives at the start of the /// file. AST node in the current file should be observed on onMainAST call. virtual void onPreambleAST(PathRef Path, llvm::StringRef Version, - ASTContext &Ctx, + const CompilerInvocation &CI, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &) {} diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -977,11 +977,10 @@ LatestBuild = clang::clangd::buildPreamble( FileName, *Req.CI, Inputs, StoreInMemory, - [this, Version(Inputs.Version)](ASTContext &Ctx, - std::shared_ptr PP, - const CanonicalIncludes &CanonIncludes) { - Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP), - CanonIncludes); + [&](ASTContext &Ctx, std::shared_ptr PP, + const CanonicalIncludes &CanonIncludes) { + Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx, + std::move(PP), CanonIncludes); }); if (LatestBuild && isReliable(LatestBuild->CompileCommand)) HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders()); diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h --- a/clang-tools-extra/clangd/index/FileIndex.h +++ b/clang-tools-extra/clangd/index/FileIndex.h @@ -117,6 +117,7 @@ void updatePreamble(PathRef Path, llvm::StringRef Version, ASTContext &AST, std::shared_ptr PP, const CanonicalIncludes &Includes); + void updatePreamble(IndexFileIn); /// Update symbols and references from main file \p Path with /// `indexMainDecls`. diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -423,13 +423,7 @@ MainFileSymbols(IndexContents::All), MainFileIndex(std::make_unique()) {} -void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version, - ASTContext &AST, - std::shared_ptr PP, - const CanonicalIncludes &Includes) { - IndexFileIn IF; - std::tie(IF.Symbols, std::ignore, IF.Relations) = - indexHeaderSymbols(Version, AST, std::move(PP), Includes); +void FileIndex::updatePreamble(IndexFileIn IF) { FileShardedIndex ShardedIndex(std::move(IF)); for (auto Uri : ShardedIndex.getAllSources()) { auto IF = ShardedIndex.getShard(Uri); @@ -460,6 +454,16 @@ } } +void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version, + ASTContext &AST, + std::shared_ptr PP, + const CanonicalIncludes &Includes) { + IndexFileIn IF; + std::tie(IF.Symbols, std::ignore, IF.Relations) = + indexHeaderSymbols(Version, AST, std::move(PP), Includes); + updatePreamble(std::move(IF)); +} + void FileIndex::updateMain(PathRef Path, ParsedAST &AST) { auto Contents = indexMainDecls(AST); MainFileSymbols.update( diff --git a/clang-tools-extra/clangd/index/StdLib.h b/clang-tools-extra/clangd/index/StdLib.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.h @@ -0,0 +1,107 @@ +//===--- StdLib.h - Index the C and C++ standard library ---------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Eagerly indexing the standard library gives a much friendlier "warm start" +// with working code completion in a standalone file or small project. +// +// We act as if we saw a file which included the whole standard library: +// #include +// #include +// #include +// ... +// We index this TU and feed the result into the dynamic index. +// +// This happens within the context of some particular open file, and we reuse +// its CompilerInvocation. Matching its include path, LangOpts etc ensures that +// we see the standard library and configuration that matches the project. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H + +#include "index/Symbol.h" +#include "support/ThreadsafeFS.h" +#include "llvm/ADT/StringRef.h" +#include + +namespace clang { +class CompilerInvocation; +class LangOptions; +class HeaderSearch; +namespace clangd { + +// The filesystem location where a standard library was found. +// +// This is the directory containing or . +// It's used to ensure we only index files that are in the standard library. +struct StdLibLocation { + llvm::SmallVector Paths; +}; + +// Tracks the state of standard library indexing within a particular index. +// +// In general, we don't want to index the standard library multiple times. +// In most cases, this class just acts as a flag to ensure we only do it once. +// +// However, if we first open a C++11 file, and then a C++20 file, we *do* +// want the index to be upgraded to include the extra symbols. +// Similarly, the C and C++ standard library can coexist. +class StdLibSet { + std::atomic Best[2] = {{-1}, {-1}}; + +public: + // Determines if we should index the standard library in a configuration. + // + // This is true if: + // - standard library indexing is enabled for the file + // - the language version is higher than any previous add() for the language + // - the standard library headers exist on the search path + // Returns the location where the standard library was found. + // + // This function is threadsafe. + llvm::Optional add(const LangOptions &, const HeaderSearch &); + + // Indicates whether we a built index should be used. + // It should not be used if a newer version has subsequently been added. + // + // Intended pattern is: + // if (add()) { + // symbols = indexStandardLibrary(); + // if (isBest()) + // index.update(symbols); + // } + // + // This is still technically racy: we could return true here, then another + // thread could add->index->update a better library before we can update. + // We'd then overwrite it with the older version. + // However, it's very unlikely: indexing takes a long time. + bool isBest(const LangOptions &) const; +}; + +// Index a standard library and return the discovered symbols. +// +// The compiler invocation should describe the file whose config we're reusing. +// We overwrite its virtual buffer with a lot of #include statements. +SymbolSlab indexStandardLibrary(std::unique_ptr Invocation, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS); + +// Variant that allows the umbrella header source to be specified. +// Exposed for testing. +SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources, + std::unique_ptr CI, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS); + +// Generate header containing #includes for all standard library headers. +// Exposed for testing. +llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &); + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H diff --git a/clang-tools-extra/clangd/index/StdLib.cpp b/clang-tools-extra/clangd/index/StdLib.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.cpp @@ -0,0 +1,357 @@ +//===-- StdLib.cpp ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "StdLib.h" +#include +#include +#include +#include + +#include "Compiler.h" +#include "Config.h" +#include "SymbolCollector.h" +#include "index/IndexAction.h" +#include "support/Logger.h" +#include "support/ThreadsafeFS.h" +#include "support/Trace.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" + +namespace clang { +namespace clangd { +namespace { + +enum Lang { C, CXX }; + +Lang langFromOpts(const LangOptions &LO) { return LO.CPlusPlus ? CXX : C; } +llvm::StringLiteral mandatoryHeader(Lang L) { + switch (L) { + case C: + return "stdio.h"; + case CXX: + return "vector"; + } + return L == CXX ? llvm::StringLiteral("vector") : "stdio.h"; +} + +LangStandard::Kind standardFromOpts(const LangOptions &LO) { + if (LO.CPlusPlus) { + return !LO.CPlusPlus11 ? LangStandard::lang_cxx98 + : !LO.CPlusPlus14 ? LangStandard::lang_cxx11 + : !LO.CPlusPlus17 ? LangStandard::lang_cxx14 + : !LO.CPlusPlus20 ? LangStandard::lang_cxx17 + : !LO.CPlusPlus2b ? LangStandard::lang_cxx20 + : LangStandard::lang_cxx2b; + } + return !LO.C11 ? LangStandard::lang_c99 + // C17 has no new features, so treat C14 as C17. + : !LO.C2x ? LangStandard::lang_c17 + : LangStandard::lang_c2x; +} + +std::string buildUmbrella(llvm::StringLiteral Mandatory, + std::vector Headers) { + std::string Result; + llvm::raw_string_ostream OS(Result); + + // We __has_include guard all our #includes to avoid errors when using older + // stdlib version that don't have headers for the newest language standards. + // But make sure we get *some* error if things are totally broken. + OS << llvm::formatv( + "#if !__has_include(<{0}>)\n" + "#error Mandatory header <{0}> not found in standard library!\n" + "#endif\n", + Mandatory); + + llvm::sort(Headers.begin(), Headers.end()); + auto Last = std::unique(Headers.begin(), Headers.end()); + for (auto Header = Headers.begin(); Header != Last; ++Header) { + OS << llvm::formatv("#if __has_include({0})\n" + "#include {0}\n" + "#endif\n", + *Header); + } + OS.flush(); + return Result; +} + +} // namespace + +llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &LO) { + // The umbrella header is the same for all versions of each language. + // Headers that are unsupported in old lang versions are usually guarded by + // #if. Some headers may be not present in old stdlib versions, the umbrella + // header guards with __has_include for this purpose. + Lang L = langFromOpts(LO); + switch (L) { + case CXX: + static std::string *UmbrellaCXX = + new std::string(buildUmbrella(mandatoryHeader(L), { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "StdSymbolMap.inc" +#undef SYMBOL + })); + return *UmbrellaCXX; + case C: + static std::string *UmbrellaC = + new std::string(buildUmbrella(mandatoryHeader(L), { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "CSymbolMap.inc" +#undef SYMBOL + })); + return *UmbrellaC; + } +} + +namespace { + +// Including the standard library leaks unwanted transitively included symbols. +// +// We want to drop these, they're a bit tricky to identify: +// - we don't want to limit to symbols our our list, as our list has only +// top-level symbols (and there may be legitimate stdlib extensions). +// - we can't limit to only symbols defined in known stdlib headers, as stdlib +// internal structure is murky +// - we can't strictly require symbols to come from a particular path, e.g. +// libstdc++ is mostly under /usr/include/c++/10/... +// but std::ctype_base is under /usr/include//c++/10/... +// We require the symbol to come from a header that is *either* from +// the standard library path (as identified by the location of ), or +// another header that defines a symbol from our stdlib list. +static SymbolSlab filter(SymbolSlab Slab, const StdLibLocation &Loc) { + SymbolSlab::Builder Result; + + static auto &StandardHeaders = *[] { + auto Set = new llvm::DenseSet(); + for (llvm::StringRef Name : { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "CSymbolMap.inc" +#include "StdSymbolMap.inc" +#undef SYMBOL + }) + Set->insert(Name); + return Set; + }(); + + // Form prefixes like file:///usr/include/c++/10/ + // These can be trivially prefix-compared with URIs in the indexed symbols. + llvm::SmallVector StdLibURIPrefixes; + for (const auto &Path : Loc.Paths) { + StdLibURIPrefixes.push_back(URI::create(Path).toString()); + if (StdLibURIPrefixes.back().back() != '/') + StdLibURIPrefixes.back().push_back('/'); + } + // For each header URI, is it *either* prefixed by StdLibURIPrefixes *or* + // owner of a symbol whose insertable header is in StandardHeaders? + // Pointer key because strings in a SymbolSlab are interned. + llvm::DenseMap GoodHeader; + for (const Symbol &S : Slab) { + if (!S.IncludeHeaders.empty() && + StandardHeaders.contains(S.IncludeHeaders.front().IncludeHeader)) { + GoodHeader[S.CanonicalDeclaration.FileURI] = true; + GoodHeader[S.Definition.FileURI] = true; + continue; + } + for (const char *URI : + {S.CanonicalDeclaration.FileURI, S.Definition.FileURI}) { + auto R = GoodHeader.try_emplace(URI, false); + if (R.second) { + R.first->second = llvm::any_of( + StdLibURIPrefixes, + [&, URIStr(llvm::StringRef(URI))](const std::string &Prefix) { + return URIStr.startswith(Prefix); + }); + } + } + } + for (const auto &Good : GoodHeader) + if (Good.second) + dlog("Stdlib header: {0}", Good.first); + // Empty URIs aren't considered good. (Definition can be blank). + auto IsGoodHeader = [&](const char *C) { return *C && GoodHeader.lookup(C); }; + + for (const Symbol &S : Slab) { + if (!(IsGoodHeader(S.CanonicalDeclaration.FileURI) || + IsGoodHeader(S.Definition.FileURI))) { + dlog("Ignoring wrong-header symbol {0}{1} in {2}", S.Scope, S.Name, + S.CanonicalDeclaration.FileURI); + continue; + } + Result.insert(S); + } + + return std::move(Result).build(); +} + +} // namespace + +SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources, + std::unique_ptr CI, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS) { + if (CI->getFrontendOpts().Inputs.size() != 1 || + !CI->getPreprocessorOpts().ImplicitPCHInclude.empty()) { + elog("Indexing standard library failed: bad CompilerInvocation"); + assert(false && "indexing stdlib with a dubious CompilerInvocation!"); + return SymbolSlab(); + } + const FrontendInputFile &Input = CI->getFrontendOpts().Inputs.front(); + trace::Span Tracer("StandardLibraryIndex"); + LangStandard::Kind LangStd = standardFromOpts(*CI->getLangOpts()); + log("Indexing {0} standard library in the context of {1}", + LangStandard::getLangStandardForKind(LangStd).getName(), Input.getFile()); + + SymbolSlab Symbols; + IgnoreDiagnostics IgnoreDiags; + CI->getPreprocessorOpts().clearRemappedFiles(); + auto Clang = prepareCompilerInstance( + std::move(CI), /*Preamble=*/nullptr, + llvm::MemoryBuffer::getMemBuffer(HeaderSources, Input.getFile()), + TFS.view(/*CWD=*/llvm::None), IgnoreDiags); + if (!Clang) { + elog("Standard Library Index: Couldn't build compiler instance"); + return Symbols; + } + + SymbolCollector::Options IndexOpts; + IndexOpts.Origin = SymbolOrigin::StdLib; + IndexOpts.CollectMainFileSymbols = false; + IndexOpts.CollectMainFileRefs = false; + IndexOpts.CollectMacro = true; + IndexOpts.StoreAllDocumentation = true; + // Sadly we can't use IndexOpts.FileFilter to restrict indexing scope. + // Files from outside the location may define true std symbols anyway. + // We end up "blessing" such headers, and can only do that by indexing + // everything first. + + // Refs, relations, containers in the stdlib mostly aren't useful. + auto Action = createStaticIndexingAction( + IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr, + nullptr, nullptr); + + if (!Action->BeginSourceFile(*Clang, Input)) { + elog("Standard Library Index: BeginSourceFile() failed"); + return Symbols; + } + + if (llvm::Error Err = Action->Execute()) { + elog("Standard Library Index: Execute failed: {0}", std::move(Err)); + return Symbols; + } + + Action->EndSourceFile(); + bool HadErrors = Clang->hasDiagnostics() && + Clang->getDiagnostics().hasUncompilableErrorOccurred(); + if (HadErrors) { + log("Errors when generating the standard library index, index may be " + "incomplete"); + } + + unsigned SymbolsBeforeFilter = Symbols.size(); + Symbols = filter(std::move(Symbols), Loc); + log("Indexed {0} standard library: ({1} symbols, {2} filtered)", + LangStandard::getLangStandardForKind(LangStd).getName(), Symbols.size(), + SymbolsBeforeFilter - Symbols.size()); + SPAN_ATTACH(Tracer, "symbols", int(Symbols.size())); + return Symbols; +} + +SymbolSlab indexStandardLibrary(std::unique_ptr Invocation, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS) { + return indexStandardLibrary( + getStdlibUmbrellaHeader(*Invocation->getLangOpts()), + std::move(Invocation), Loc, TFS); +} + +bool StdLibSet::isBest(const LangOptions &LO) const { + return standardFromOpts(LO) >= + Best[langFromOpts(LO)].load(std::memory_order_acquire); +} + +llvm::Optional StdLibSet::add(const LangOptions &LO, + const HeaderSearch &HS) { + Lang L = langFromOpts(LO); + int OldVersion = Best[L].load(std::memory_order_acquire); + int NewVersion = standardFromOpts(LO); + dlog("Index stdlib? {0}", + LangStandard::getLangStandardForKind(standardFromOpts(LO)).getName()); + + if (NewVersion <= OldVersion) { + dlog("No: have {0}, {1}>={2}", + LangStandard::getLangStandardForKind( + static_cast(NewVersion)) + .getName(), + OldVersion, NewVersion); + return llvm::None; + } + + if (!Config::current().Index.StandardLibrary) { + dlog("No: disabled in config"); + return llvm::None; + } + + // We'd like to index a standard library here if there is one. + // Check for the existence of on the search path. + // We could cache this, but we only get here repeatedly when there's no + // stdlib, and even then only once per preamble build. + llvm::StringLiteral ProbeHeader = mandatoryHeader(L); + llvm::SmallString<256> Path; // Scratch space. + llvm::SmallVector SearchPaths; + auto RecordHeaderPath = [&](llvm::StringRef HeaderPath) { + llvm::StringRef DirPath = llvm::sys::path::parent_path(HeaderPath); + if (!HS.getFileMgr().getVirtualFileSystem().getRealPath(DirPath, Path)) + SearchPaths.emplace_back(Path); + }; + for (const auto &DL : + llvm::make_range(HS.search_dir_begin(), HS.search_dir_end())) { + switch (DL.getLookupType()) { + case DirectoryLookup::LT_NormalDir: { + Path = DL.getDir()->getName(); + llvm::sys::path::append(Path, ProbeHeader); + llvm::vfs::Status Stat; + if (!HS.getFileMgr().getNoncachedStatValue(Path, Stat) && + Stat.isRegularFile()) + RecordHeaderPath(Path); + break; + } + case DirectoryLookup::LT_Framework: + // stdlib can't be a framework (framework includes bust have a slash) + continue; + case DirectoryLookup::LT_HeaderMap: + llvm::StringRef Target = + DL.getHeaderMap()->lookupFilename(ProbeHeader, Path); + if (!Target.empty()) + RecordHeaderPath(Target); + break; + } + } + if (SearchPaths.empty()) { + dlog("No: didn't find <{0}>)", ProbeHeader); + return llvm::None; + } + dlog("Found standard library in {0}", llvm::join(SearchPaths, ", ")); + + while (!Best[L].compare_exchange_weak(OldVersion, NewVersion, + std::memory_order_acq_rel)) + if (OldVersion >= NewVersion) { + dlog("No: lost the race"); + return llvm::None; // Another thread won the race while we were checking. + } + + dlog("Yes, index stdlib!"); + return StdLibLocation{std::move(SearchPaths)}; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.h b/clang-tools-extra/clangd/index/SymbolOrigin.h --- a/clang-tools-extra/clangd/index/SymbolOrigin.h +++ b/clang-tools-extra/clangd/index/SymbolOrigin.h @@ -26,6 +26,7 @@ Merge = 1 << 3, // A non-trivial index merge was performed. Identifier = 1 << 4, // Raw identifiers in file. Remote = 1 << 5, // Remote index. + StdLib = 1 << 6, // Standard library index. // Remaining bits reserved for index implementations. }; diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.cpp b/clang-tools-extra/clangd/index/SymbolOrigin.cpp --- a/clang-tools-extra/clangd/index/SymbolOrigin.cpp +++ b/clang-tools-extra/clangd/index/SymbolOrigin.cpp @@ -14,7 +14,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) { if (O == SymbolOrigin::Unknown) return OS << "unknown"; - constexpr static char Sigils[] = "ADSMIR67"; + constexpr static char Sigils[] = "ADSMIRL7"; for (unsigned I = 0; I < sizeof(Sigils); ++I) if (static_cast(O) & 1u << I) OS << Sigils[I]; diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -80,6 +80,7 @@ SemanticSelectionTests.cpp SerializationTests.cpp SourceCodeTests.cpp + StdLibTests.cpp SymbolCollectorTests.cpp SymbolInfoTests.cpp SyncAPI.cpp diff --git a/clang-tools-extra/clangd/unittests/StdLibTests.cpp b/clang-tools-extra/clangd/unittests/StdLibTests.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/StdLibTests.cpp @@ -0,0 +1,125 @@ +//===-- StdLibTests.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Compiler.h" +#include "Config.h" +#include "TestFS.h" +#include "index/StdLib.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceManager.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include + +using namespace testing; + +namespace clang { +namespace clangd { +namespace { + +// Check the generated header sources contains usual standard library headers. +TEST(StdLibTests, getStdlibUmbrellaHeader) { + LangOptions LO; + LO.CPlusPlus = true; + + auto CXX = getStdlibUmbrellaHeader(LO); + EXPECT_THAT(CXX, HasSubstr("#include ")); + EXPECT_THAT(CXX, HasSubstr("#include ")); + EXPECT_THAT(CXX, Not(HasSubstr("#include "))); + + LO.CPlusPlus = false; + auto C = getStdlibUmbrellaHeader(LO); + EXPECT_THAT(C, Not(HasSubstr("#include "))); + EXPECT_THAT(C, Not(HasSubstr("#include "))); + EXPECT_THAT(C, HasSubstr("#include ")); +} + +MATCHER_P(Named, Name, "") { return arg.Name == Name; } + +// Build an index, and check if it contains the right symbols. +TEST(StdLibTests, indexStandardLibrary) { + MockFS FS; + FS.Files["std/foo.h"] = R"cpp( + #include + #if __cplusplus >= 201703L + int foo17(); + #elif __cplusplus >= 201402L + int foo14(); + #else + bool foo98(); + #endif + )cpp"; + FS.Files["nonstd/platform_stuff.h"] = "int magic = 42;"; + + ParseInputs OriginalInputs; + OriginalInputs.TFS = &FS; + OriginalInputs.CompileCommand.Filename = testPath("main.cc"); + OriginalInputs.CompileCommand.CommandLine = {"clang++", testPath("main.cc"), + "-isystemstd/", + "-isystemnonstd/", "-std=c++14"}; + OriginalInputs.CompileCommand.Directory = testRoot(); + IgnoreDiagnostics Diags; + auto CI = buildCompilerInvocation(OriginalInputs, Diags); + ASSERT_TRUE(CI); + + StdLibLocation Loc; + Loc.Paths.push_back(testPath("std/")); + + auto Symbols = + indexStandardLibrary("#include ", std::move(CI), Loc, FS); + EXPECT_THAT(Symbols, ElementsAre(Named("foo14"))); +} + +TEST(StdLibTests, StdLibSet) { + StdLibSet Set; + MockFS FS; + FS.Files["std/_"] = ""; + FS.Files["libc/_"] = ""; + + auto Add = [&](const LangOptions &LO, + std::vector SearchPath) { + SourceManagerForFile SM("scratch", ""); + SM.get().getFileManager().setVirtualFileSystem(FS.view(llvm::None)); + HeaderSearch HS(/*HSOpts=*/nullptr, SM.get(), SM.get().getDiagnostics(), LO, + /*Target=*/nullptr); + for (auto P : SearchPath) + HS.AddSearchPath( + DirectoryLookup( + cantFail(SM.get().getFileManager().getDirectoryRef(testPath(P))), + SrcMgr::C_System, /*isFramework=*/false), + true); + return Set.add(LO, HS); + }; + + Config Cfg; + Cfg.Index.StandardLibrary = false; + WithContextValue Disabled(Config::Key, std::move(Cfg)); + + LangOptions LO; + LO.CPlusPlus = true; + EXPECT_FALSE(Add(LO, {"std"})) << "No found"; + FS.Files["std/vector"] = "class vector;"; + EXPECT_FALSE(Add(LO, {"std"})) << "Disabled in config"; + + Cfg = Config(); + Cfg.Index.StandardLibrary = true; + WithContextValue Enabled(Config::Key, std::move(Cfg)); + + EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++98"; + EXPECT_FALSE(Add(LO, {"std"})) << "Don't reindex"; + LO.CPlusPlus11 = true; + EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++11"; + LO.CPlusPlus = false; + EXPECT_FALSE(Add(LO, {"libc"})) << "No "; + FS.Files["libc/stdio.h"] = true; + EXPECT_TRUE(Add(LO, {"libc"})) << "Indexing as C"; +} + +} // namespace +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -1121,7 +1121,8 @@ public: BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N) : BlockVersion(BlockVersion), N(N) {} - void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx, + void onPreambleAST(PathRef Path, llvm::StringRef Version, + const CompilerInvocation &, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &) override { if (Version == BlockVersion)