diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -119,6 +119,7 @@ index/Ref.cpp index/Relation.cpp index/Serialization.cpp + index/StdLib.cpp index/Symbol.cpp index/SymbolCollector.cpp index/SymbolID.cpp diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -14,7 +14,6 @@ #include "FindSymbols.h" #include "Format.h" #include "HeaderSourceSwitch.h" -#include "Headers.h" #include "InlayHints.h" #include "ParsedAST.h" #include "Preamble.h" @@ -27,10 +26,10 @@ #include "index/CanonicalIncludes.h" #include "index/FileIndex.h" #include "index/Merge.h" +#include "index/StdLib.h" #include "refactor/Rename.h" #include "refactor/Tweak.h" #include "support/Logger.h" -#include "support/Markup.h" #include "support/MemoryTree.h" #include "support/ThreadsafeFS.h" #include "support/Trace.h" @@ -43,14 +42,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" -#include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -59,6 +53,7 @@ #include #include #include +#include namespace clang { namespace clangd { @@ -67,16 +62,39 @@ // Update the FileIndex with new ASTs and plumb the diagnostics responses. struct UpdateIndexCallbacks : public ParsingCallbacks { UpdateIndexCallbacks(FileIndex *FIndex, - ClangdServer::Callbacks *ServerCallbacks) - : FIndex(FIndex), ServerCallbacks(ServerCallbacks) {} + ClangdServer::Callbacks *ServerCallbacks, + const ThreadsafeFS &TFS, bool Sync) + : FIndex(FIndex), ServerCallbacks(ServerCallbacks), TFS(TFS) { + if (!Sync) + Tasks.emplace(); + } - void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx, + void onPreambleAST(PathRef Path, llvm::StringRef Version, + const CompilerInvocation &CI, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &CanonIncludes) override { + // If this preamble uses a standard library we haven't seen yet, index it. + if (auto Loc = Stdlib.add(*CI.getLangOpts(), PP->getHeaderSearchInfo())) + indexStdlib(CI, std::move(*Loc)); + if (FIndex) FIndex->updatePreamble(Path, Version, Ctx, std::move(PP), CanonIncludes); } + void indexStdlib(const CompilerInvocation &CI, StdLibLocation Loc) { + auto Task = [this, LO(*CI.getLangOpts()), Loc(std::move(Loc)), + CI(std::make_unique(CI))]() mutable { + IndexFileIn IF; + IF.Symbols = indexStandardLibrary(std::move(CI), Loc, TFS); + if (Stdlib.isBest(LO)) + FIndex->updatePreamble(std::move(IF)); + }; + if (Tasks) + Tasks->runAsync("IndexStdlib", std::move(Task)); + else + Task(); + } + void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) override { if (FIndex) FIndex->updateMain(Path, AST); @@ -111,6 +129,9 @@ private: FileIndex *FIndex; ClangdServer::Callbacks *ServerCallbacks; + const ThreadsafeFS &TFS; + StdLibSet Stdlib; + llvm::Optional Tasks; }; class DraftStoreFS : public ThreadsafeFS { @@ -163,9 +184,10 @@ // Pass a callback into `WorkScheduler` to extract symbols from a newly // parsed file and rebuild the file index synchronously each time an AST // is parsed. - WorkScheduler.emplace( - CDB, TUScheduler::Options(Opts), - std::make_unique(DynamicIdx.get(), Callbacks)); + WorkScheduler.emplace(CDB, TUScheduler::Options(Opts), + std::make_unique( + DynamicIdx.get(), Callbacks, TFS, + /*Sync=*/Opts.AsyncThreadsCount == 0)); // Adds an index to the stack, at higher priority than existing indexes. auto AddIndex = [&](SymbolIndex *Idx) { if (this->Index != nullptr) { @@ -895,7 +917,7 @@ // It's safe to pass in the TU, as dumpAST() does not // deserialize the preamble. auto Node = DynTypedNode::create( - *Inputs->AST.getASTContext().getTranslationUnitDecl()); + *Inputs->AST.getASTContext().getTranslationUnitDecl()); return CB(dumpAST(Node, Inputs->AST.getTokens(), Inputs->AST.getASTContext())); } diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h --- a/clang-tools-extra/clangd/Config.h +++ b/clang-tools-extra/clangd/Config.h @@ -79,11 +79,12 @@ /// forward-slashes. std::string MountPoint; }; - /// Controls background-index behavior. + /// Controls index behavior. struct { - /// Whether this TU should be indexed. + /// Whether this TU should be background-indexed. BackgroundPolicy Background = BackgroundPolicy::Build; ExternalIndexSpec External; + bool StandardLibrary = false; } Index; enum UnusedIncludesPolicy { Strict, None }; diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp --- a/clang-tools-extra/clangd/ConfigCompile.cpp +++ b/clang-tools-extra/clangd/ConfigCompile.cpp @@ -321,6 +321,11 @@ } if (F.External) compile(std::move(**F.External), F.External->Range); + if (F.StandardLibrary) + Out.Apply.push_back( + [Val(**F.StandardLibrary)](const Params &, Config &C) { + C.Index.StandardLibrary = Val; + }); } void compile(Fragment::IndexBlock::ExternalBlock &&External, diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h --- a/clang-tools-extra/clangd/ConfigFragment.h +++ b/clang-tools-extra/clangd/ConfigFragment.h @@ -191,6 +191,9 @@ llvm::Optional> MountPoint; }; llvm::Optional> External; + // Whether the standard library visible from this file should be indexed. + // This makes all standard library symbols available, included or not. + llvm::Optional> StandardLibrary; }; IndexBlock Index; diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp --- a/clang-tools-extra/clangd/ConfigYAML.cpp +++ b/clang-tools-extra/clangd/ConfigYAML.cpp @@ -167,6 +167,10 @@ F.External.emplace(std::move(External)); F.External->Range = N.getSourceRange(); }); + Dict.handle("StandardLibrary", [&](Node &N) { + if (auto StandardLibrary = boolValue(N, "StandardLibrary")) + F.StandardLibrary = *StandardLibrary; + }); Dict.parse(N); } @@ -194,12 +198,8 @@ void parse(Fragment::CompletionBlock &F, Node &N) { DictParser Dict("Completion", this); Dict.handle("AllScopes", [&](Node &N) { - if (auto Value = scalarValue(N, "AllScopes")) { - if (auto AllScopes = llvm::yaml::parseBool(**Value)) - F.AllScopes = *AllScopes; - else - warning("AllScopes should be a boolean", N); - } + if (auto AllScopes = boolValue(N, "AllScopes")) + F.AllScopes = *AllScopes; }); Dict.parse(N); } @@ -334,6 +334,16 @@ return Result; } + llvm::Optional> boolValue(Node &N, llvm::StringRef Desc) { + if (auto Scalar = scalarValue(N, Desc)) { + if (auto StandardLibrary = llvm::yaml::parseBool(**Scalar)) + return Located(*StandardLibrary, Scalar->Range); + else + warning(Desc + " should be a boolean", N); + } + return llvm::None; + } + // Report a "hard" error, reflecting a config file that can never be valid. void error(const llvm::Twine &Msg, llvm::SMRange Range) { HadError = true; diff --git a/clang-tools-extra/clangd/FS.cpp b/clang-tools-extra/clangd/FS.cpp --- a/clang-tools-extra/clangd/FS.cpp +++ b/clang-tools-extra/clangd/FS.cpp @@ -117,5 +117,15 @@ return CanonPath.str().str(); } +const llvm::StringLiteral virtualRoot() { +#ifdef _WIN32 + return "C:\\virtual\\"; +#else + // This path must exist on the current file system. The only place we can + // safely assume to exist is "/". + return "/"; +#endif +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h --- a/clang-tools-extra/clangd/TUScheduler.h +++ b/clang-tools-extra/clangd/TUScheduler.h @@ -134,7 +134,7 @@ /// contains only AST nodes from the #include directives at the start of the /// file. AST node in the current file should be observed on onMainAST call. virtual void onPreambleAST(PathRef Path, llvm::StringRef Version, - ASTContext &Ctx, + const CompilerInvocation &CI, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &) {} diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -977,11 +977,10 @@ LatestBuild = clang::clangd::buildPreamble( FileName, *Req.CI, Inputs, StoreInMemory, - [this, Version(Inputs.Version)](ASTContext &Ctx, - std::shared_ptr PP, - const CanonicalIncludes &CanonIncludes) { - Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP), - CanonIncludes); + [&](ASTContext &Ctx, std::shared_ptr PP, + const CanonicalIncludes &CanonIncludes) { + Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx, + std::move(PP), CanonIncludes); }); if (LatestBuild && isReliable(LatestBuild->CompileCommand)) HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders()); diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h --- a/clang-tools-extra/clangd/index/FileIndex.h +++ b/clang-tools-extra/clangd/index/FileIndex.h @@ -118,6 +118,8 @@ std::shared_ptr PP, const CanonicalIncludes &Includes); + void updatePreamble(IndexFileIn); + /// Update symbols and references from main file \p Path with /// `indexMainDecls`. void updateMain(PathRef Path, ParsedAST &AST); diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -423,13 +423,7 @@ MainFileSymbols(IndexContents::All), MainFileIndex(std::make_unique()) {} -void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version, - ASTContext &AST, - std::shared_ptr PP, - const CanonicalIncludes &Includes) { - IndexFileIn IF; - std::tie(IF.Symbols, std::ignore, IF.Relations) = - indexHeaderSymbols(Version, AST, std::move(PP), Includes); +void FileIndex::updatePreamble(IndexFileIn IF) { FileShardedIndex ShardedIndex(std::move(IF)); for (auto Uri : ShardedIndex.getAllSources()) { auto IF = ShardedIndex.getShard(Uri); @@ -460,6 +454,16 @@ } } +void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version, + ASTContext &AST, + std::shared_ptr PP, + const CanonicalIncludes &Includes) { + IndexFileIn IF; + std::tie(IF.Symbols, std::ignore, IF.Relations) = + indexHeaderSymbols(Version, AST, std::move(PP), Includes); + updatePreamble(std::move(IF)); +} + void FileIndex::updateMain(PathRef Path, ParsedAST &AST) { auto Contents = indexMainDecls(AST); MainFileSymbols.update( diff --git a/clang-tools-extra/clangd/index/StdLib.h b/clang-tools-extra/clangd/index/StdLib.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.h @@ -0,0 +1,87 @@ +//===--- StdLib.h ------------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Clangd indexer for the C++ standard library. +// +// The index only contains symbols that are part of the translation unit. So +// if your translation unit does not yet #include , you do not get +// auto completion for std::string. However we expect that many users would +// like to use the the standard library anyway, so we could index that by +// default an offer e.g. code completion without requiring #includes. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H + +#include "index/Symbol.h" +#include "support/ThreadsafeFS.h" +#include "clang/Basic/LangStandard.h" +#include "llvm/ADT/StringRef.h" +#include + +namespace clang { +class CompilerInvocation; +class LangOptions; +class HeaderSearch; +namespace clangd { + +struct StdLibLocation { + llvm::SmallVector Paths; +}; + +class StdLibSet { + std::atomic Best[2] = {{-1}, {-1}}; + +public: + // Determines if we should index the standard library in a configuration. + // + // This is true if: + // - standard library indexing is enabled for the file + // - the language version is higher than any previous add() for the language + // - the standard library headers exist on the search path + // + // This function is threadsafe. + llvm::Optional add(const LangOptions &, const HeaderSearch &); + + // Indicates whether we a built index should be used. + // It should not be used if a newer version has subsequently been added. + // + // Intended pattern is: + // if (add()) { + // symbols = indexStandardLibrary(); + // if (isBest()) + // index.update(symbols); + // } + // + // This is still technically racy: we could return true here, then another + // thread could add->index->update a better library before we can update. + // We'd then overwrite it with the older version. + // However, it's very unlikely: indexing takes a long time. + bool isBest(const LangOptions &) const; +}; + +/// Generate a index of the standard library index for a given variant of +/// the standard library. This index can be included if the translation unit +/// does not (yet) contain any standard library headers. +SymbolSlab indexStandardLibrary(std::unique_ptr Invocation, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS); + +/// Index the given umbrella header file using the standard library from the +/// given file system. +SymbolSlab indexUmbrellaHeaders(llvm::StringRef HeaderSources, + std::unique_ptr CI, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS); + +/// generate header containing #includes for all standard library headers +llvm::StringRef generateUmbrellaHeaders(LangStandard::Kind LibraryVariant); + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H diff --git a/clang-tools-extra/clangd/index/StdLib.cpp b/clang-tools-extra/clangd/index/StdLib.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.cpp @@ -0,0 +1,365 @@ +//===-- StdLib.cpp ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "StdLib.h" +#include +#include +#include +#include + +#include "Compiler.h" +#include "Config.h" +#include "SymbolCollector.h" +#include "index/IndexAction.h" +#include "support/Logger.h" +#include "support/ThreadsafeFS.h" +#include "support/Trace.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" + +namespace clang { +namespace clangd { +namespace { + +enum Lang { C, CXX }; + +llvm::StringLiteral mandatoryHeader(Lang L) { + switch (L) { + case C: + return "stdio.h"; + case CXX: + return "vector"; + } + return L == CXX ? llvm::StringLiteral("vector") : "stdio.h"; +} + +std::string buildUmbrella(llvm::StringLiteral Mandatory, + std::vector Headers) { + std::string Result; + llvm::raw_string_ostream OS(Result); + + // We __has_include guard all our #includes to avoid errors when using older + // stdlib version that don't have headers for the newest language standards. + // But make sure we get *some* error if things are totally broken. + OS << llvm::formatv( + "#if !__has_include(<{0}>)\n" + "#error Mandatory header <{0}> not found in standard library!\n" + "#endif\n", + Mandatory); + + llvm::sort(Headers.begin(), Headers.end()); + auto Last = std::unique(Headers.begin(), Headers.end()); + for (auto Header = Headers.begin(); Header != Last; ++Header) { + OS << llvm::formatv("#if __has_include({0})\n" + "#include {0}\n" + "#endif\n", + *Header); + } + OS.flush(); + return Result; +} + +} // namespace + +// Build umbrella header for the the standard library. +// +// The umbrella header is the same for all versions of each language. +// Headers that are unsupported in old lang versions are usually guarded by #if. +// Some headers may be not present in old stdlib versions, the umbrella header +// guards with __has_include for this purpose. +llvm::StringRef generateUmbrellaHeaders(Lang L) { + switch (L) { + case CXX: + static std::string *UmbrellaCXX = + new std::string(buildUmbrella(mandatoryHeader(L), { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "StdSymbolMap.inc" +#undef SYMBOL + })); + return *UmbrellaCXX; + case C: + static std::string *UmbrellaC = + new std::string(buildUmbrella(mandatoryHeader(L), { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "CSymbolMap.inc" +#undef SYMBOL + })); + return *UmbrellaC; + } +} + +namespace { + +// Including the standard library leaks a lot of symbols/names that are not +// part of its interface, and we want to drop these. There are two main groups: +// +// Implementation details. These tend to have reserved names like __foo. +// We drop symbols with reserved names unless they're on our symbol list. +// +// Transitive includes that are not part of the stdlib (low-level platform deps) +// These are surprisingly tricky to identify: +// - we don't want to limit to symbols our our list, as our list has only +// top-level symbols (and there may be legitimate stdlib extensions). +// - we can't limit to only symbols defined by known stdlib headers, as stdlib +// internal structure is murky +// - we can't strictly require symbols to come from a particular path, e.g. +// libstdc++ is mostly under /usr/include/c++/10/... +// but std::ctype_base is under /usr/include//c++/10/... +// Instead we require the symbol to come from a header that is *either* from +// the standard library path (as identified by the location of ), or +// another header that defines a symbol from our stdlib list. +static SymbolSlab filter(SymbolSlab Slab, const StdLibLocation &Loc, + llvm::StringRef TUPath) { + SymbolSlab::Builder Result; + + static auto &StandardHeaders = *[] { + auto Set = new llvm::DenseSet(); + for (llvm::StringRef Name : { +#define SYMBOL(Name, NameSpace, Header) #Header, +#include "CSymbolMap.inc" +#include "StdSymbolMap.inc" +#undef SYMBOL + }) + Set->insert(Name); + return Set; + }(); + + // Form prefixes like file:///usr/include/c++/10/ + // These can be trivially prefix-compared with URIs in the indexed symbols. + llvm::SmallVector StdLibURIPrefixes; + for (const auto &Path : Loc.Paths) { + StdLibURIPrefixes.push_back(URI::create(Path).toString()); + if (StdLibURIPrefixes.back().back() != '/') + StdLibURIPrefixes.back().push_back('/'); + } + // For each header URI, is it *either* prefixed by StdLibURIPrefixes *or* + // owner of a symbol whose insertable header is in StandardHeaders? + // Pointer key because strings in a SymbolSlab are interned. + llvm::DenseMap GoodHeader; + for (const Symbol &S : Slab) { + if (!S.IncludeHeaders.empty() && + StandardHeaders.contains(S.IncludeHeaders.front().IncludeHeader)) { + GoodHeader[S.CanonicalDeclaration.FileURI] = true; + GoodHeader[S.Definition.FileURI] = true; + continue; + } + for (const char *URI : + {S.CanonicalDeclaration.FileURI, S.Definition.FileURI}) { + auto R = GoodHeader.try_emplace(URI, false); + if (R.second) { + R.first->second = llvm::any_of( + StdLibURIPrefixes, + [&, URIStr(llvm::StringRef(URI))](const std::string &Prefix) { + return URIStr.startswith(Prefix); + }); + } + } + } + for (const auto &Good : GoodHeader) + if (Good.second) + dlog("Stdlib header: {0}", Good.first); + // Empty URIs aren't considered good. (Definition can be blank). + auto IsGoodHeader = [&](const char *C) { return *C && GoodHeader.lookup(C); }; + + for (const Symbol &S : Slab) { + if (!(IsGoodHeader(S.CanonicalDeclaration.FileURI) || + IsGoodHeader(S.Definition.FileURI))) { + dlog("Ignoring wrong-header symbol {0}{1} in {2}", S.Scope, S.Name, + S.CanonicalDeclaration.FileURI); + continue; + } + Result.insert(S); + } + + return std::move(Result).build(); +} + +LangStandard::Kind standardFromOpts(const LangOptions &LO) { + if (LO.CPlusPlus) { + return !LO.CPlusPlus11 ? LangStandard::lang_cxx98 + : !LO.CPlusPlus14 ? LangStandard::lang_cxx11 + : !LO.CPlusPlus17 ? LangStandard::lang_cxx14 + : !LO.CPlusPlus20 ? LangStandard::lang_cxx17 + : !LO.CPlusPlus2b ? LangStandard::lang_cxx20 + : LangStandard::lang_cxx2b; + } + return !LO.C11 ? LangStandard::lang_c99 + // C17 has no new features, so treat C14 as C17. + : !LO.C2x ? LangStandard::lang_c17 + : LangStandard::lang_c2x; +} + +Lang langFromOpts(const LangOptions &LO) { return LO.CPlusPlus ? CXX : C; } + +} // namespace + +SymbolSlab indexUmbrellaHeaders(llvm::StringRef HeaderSources, + std::unique_ptr CI, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS) { + if (CI->getFrontendOpts().Inputs.size() != 1 || + !CI->getPreprocessorOpts().ImplicitPCHInclude.empty()) { + elog("Indexing standard library failed: bad CompilerInvocation"); + assert(false && "indexing stdlib with a dubious CompilerInvocation!"); + return SymbolSlab(); + } + const FrontendInputFile &Input = CI->getFrontendOpts().Inputs.front(); + trace::Span Tracer("StandardLibraryIndex"); + LangStandard::Kind LangStd = standardFromOpts(*CI->getLangOpts()); + log("Indexing {0} standard library in the context of {1}", + LangStandard::getLangStandardForKind(LangStd).getName(), Input.getFile()); + + SymbolSlab Symbols; + IgnoreDiagnostics IgnoreDiags; + CI->getPreprocessorOpts().clearRemappedFiles(); + auto Clang = prepareCompilerInstance( + std::move(CI), /*Preamble=*/nullptr, + llvm::MemoryBuffer::getMemBuffer(HeaderSources, Input.getFile()), + TFS.view(/*CWD=*/llvm::None), IgnoreDiags); + if (!Clang) { + elog("Standard Library Index: Couldn't build compiler instance"); + return Symbols; + } + + SymbolCollector::Options IndexOpts; + IndexOpts.Origin = SymbolOrigin::StdLib; + IndexOpts.CollectMainFileSymbols = false; + IndexOpts.CollectMainFileRefs = false; + IndexOpts.CollectMacro = true; + IndexOpts.StoreAllDocumentation = true; + // Sadly we can't use IndexOpts.FileFilter to restrict indexing scope. + // Files from outside the location may define true std symbols anyway. + // We end up "blessing" such headers, and can only do that by indexing + // everything first. + + // we only care about the symbols, so not storing the other attributes + auto Action = createStaticIndexingAction( + IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr, + nullptr, nullptr); + + if (!Action->BeginSourceFile(*Clang, Input)) { + elog("Standard Library Index: BeginSourceFile() failed"); + return Symbols; + } + + if (llvm::Error Err = Action->Execute()) { + elog("Standard Library Index: Execute failed: {0}", std::move(Err)); + return Symbols; + } + + Action->EndSourceFile(); + bool HadErrors = Clang->hasDiagnostics() && + Clang->getDiagnostics().hasUncompilableErrorOccurred(); + if (HadErrors) { + log("Errors when generating the standard library index, index may be " + "incomplete"); + } + + unsigned SymbolsBeforeFilter = Symbols.size(); + Symbols = filter(std::move(Symbols), Loc, Input.getFile()); + log("Indexed {0} standard library: ({1} symbols, {2} filtered)", + LangStandard::getLangStandardForKind(LangStd).getName(), Symbols.size(), + SymbolsBeforeFilter - Symbols.size()); + SPAN_ATTACH(Tracer, "symbols", int(Symbols.size())); + return Symbols; +} + +SymbolSlab indexStandardLibrary(std::unique_ptr Invocation, + const StdLibLocation &Loc, + const ThreadsafeFS &TFS) { + return indexUmbrellaHeaders( + generateUmbrellaHeaders(langFromOpts(*Invocation->getLangOpts())), + std::move(Invocation), Loc, TFS); +} + +bool StdLibSet::isBest(const LangOptions &LO) const { + return standardFromOpts(LO) >= + Best[langFromOpts(LO)].load(std::memory_order_acquire); +} + +llvm::Optional StdLibSet::add(const LangOptions &LO, + const HeaderSearch &HS) { + Lang L = langFromOpts(LO); + int OldVersion = Best[L].load(std::memory_order_acquire); + int NewVersion = standardFromOpts(LO); + dlog("Index stdlib? {0}", + LangStandard::getLangStandardForKind(standardFromOpts(LO)).getName()); + + if (NewVersion <= OldVersion) { + dlog("No: have {0}, {1}>={2}", + LangStandard::getLangStandardForKind( + static_cast(NewVersion)) + .getName(), + OldVersion, NewVersion); + return llvm::None; + } + + if (!Config::current().Index.StandardLibrary) { + dlog("No: disabled in config"); + return llvm::None; + } + + // We'd like to index a standard library here if there is one. + // Check for the existence of on the search path. + // We could cache this, but we only get here repeatedly when there's no + // stdlib, and even then only once per preamble build. + llvm::StringLiteral ProbeHeader = mandatoryHeader(L); + llvm::SmallString<256> Path; // Scratch space. + llvm::SmallVector SearchPaths; + auto RecordHeaderPath = [&](llvm::StringRef HeaderPath) { + llvm::StringRef DirPath = llvm::sys::path::parent_path(HeaderPath); + if (!HS.getFileMgr().getVirtualFileSystem().getRealPath(DirPath, Path)) + SearchPaths.emplace_back(Path); + }; + for (const auto &DL : + llvm::make_range(HS.search_dir_begin(), HS.search_dir_end())) { + switch (DL.getLookupType()) { + case DirectoryLookup::LT_NormalDir: { + Path = DL.getDir()->getName(); + llvm::sys::path::append(Path, ProbeHeader); + llvm::vfs::Status Stat; + if (!HS.getFileMgr().getNoncachedStatValue(Path, Stat) && + Stat.isRegularFile()) + RecordHeaderPath(Path); + break; + } + case DirectoryLookup::LT_Framework: + // stdlib can't be a framework (framework includes bust have a slash) + continue; + case DirectoryLookup::LT_HeaderMap: + llvm::StringRef Target = + DL.getHeaderMap()->lookupFilename(ProbeHeader, Path); + if (!Target.empty()) + RecordHeaderPath(Target); + break; + } + } + if (SearchPaths.empty()) { + dlog("No: didn't find <{0}>)", ProbeHeader); + return llvm::None; + } + dlog("Found standard library in {0}", llvm::join(SearchPaths, ", ")); + + while (!Best[L].compare_exchange_weak(OldVersion, NewVersion, + std::memory_order_acq_rel)) + if (OldVersion >= NewVersion) { + dlog("No: lost the race"); + return llvm::None; // Another thread won the race while we were checking. + } + + dlog("Yes, index stdlib!"); + return StdLibLocation{std::move(SearchPaths)}; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.h b/clang-tools-extra/clangd/index/SymbolOrigin.h --- a/clang-tools-extra/clangd/index/SymbolOrigin.h +++ b/clang-tools-extra/clangd/index/SymbolOrigin.h @@ -26,6 +26,7 @@ Merge = 1 << 3, // A non-trivial index merge was performed. Identifier = 1 << 4, // Raw identifiers in file. Remote = 1 << 5, // Remote index. + StdLib = 1 << 6, // Standard library index. // Remaining bits reserved for index implementations. }; diff --git a/clang-tools-extra/clangd/index/SymbolOrigin.cpp b/clang-tools-extra/clangd/index/SymbolOrigin.cpp --- a/clang-tools-extra/clangd/index/SymbolOrigin.cpp +++ b/clang-tools-extra/clangd/index/SymbolOrigin.cpp @@ -14,7 +14,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) { if (O == SymbolOrigin::Unknown) return OS << "unknown"; - constexpr static char Sigils[] = "ADSMIR67"; + constexpr static char Sigils[] = "ADSMIRL7"; for (unsigned I = 0; I < sizeof(Sigils); ++I) if (static_cast(O) & 1u << I) OS << Sigils[I]; diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -80,6 +80,7 @@ SemanticSelectionTests.cpp SerializationTests.cpp SourceCodeTests.cpp + StdLibIndexTests.cpp SymbolCollectorTests.cpp SymbolInfoTests.cpp SyncAPI.cpp diff --git a/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp b/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp @@ -0,0 +1,101 @@ +//===-- StdLibIndexTests.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Compiler.h" +#include "Config.h" +#include "TestFS.h" +#include "index/StdLib.h" +#include "clang/Basic/LangStandard.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include + +using namespace testing; + +namespace clang { +namespace clangd { +namespace { + +/// check that the generated header sources contains some usual standard library +/// headers +TEST(StdLibIndexTests, generateUmbrellaHeader) { + auto CXX = generateUmbrellaHeaders(LangStandard::lang_cxx14).str(); + EXPECT_THAT(CXX, HasSubstr("#include ")); + EXPECT_THAT(CXX, HasSubstr("#include ")); + EXPECT_THAT(CXX, Not(HasSubstr("#include "))); + + auto C = generateUmbrellaHeaders(LangStandard::lang_c11).str(); + EXPECT_THAT(C, Not(HasSubstr("#include "))); + EXPECT_THAT(C, Not(HasSubstr("#include "))); + EXPECT_THAT(C, HasSubstr("#include ")); +} + +MATCHER_P(Named, Name, "") { return arg.Name == Name; } + +/// build the index and check if it contains the right symbols +TEST(StdLibIndexTests, buildIndex) { + MockFS FS; + // TODO: maybe find a way to use a local libcxx for testing if that is + // available on the machine + std::string HeaderMock = R"CPP( + #if __cplusplus >= 201703L + int func17(); + #elif __cplusplus >= 201402L + int func14(); + #else + bool func98(); + #endif + int __implementation_details(); + )CPP"; + StdlibIndexSpec Spec; + Spec.LangStd = LangStandard::lang_cxx14; + auto Symbols = indexUmbrellaHeaders(HeaderMock, FS, Spec); + + EXPECT_THAT(Symbols, ElementsAre(Named("func14"))); +} + +CompilerInvocation parse(std::vector Flags) { + ParseInputs Inputs; + Inputs.CompileCommand.Filename = testPath(Flags.back()); + Inputs.CompileCommand.Directory = testRoot(); + Inputs.CompileCommand.CommandLine = std::move(Flags); + MockFS FS; + Inputs.TFS = &FS; + IgnoreDiagnostics IgnoreDiags; + auto Result = buildCompilerInvocation(Inputs, IgnoreDiags); + EXPECT_TRUE(Result); + return *Result; +} + +TEST(StdLibIndexTests, Detect) { + { + Config Cfg; + Cfg.Index.StandardLibrary = false; + WithContextValue WithCfg(Config::Key, std::move(Cfg)); + EXPECT_FALSE(StdlibIndexSpec::detect(parse({"clang++", "test.cc"}))); + } + Config Cfg; + Cfg.Index.StandardLibrary = true; + WithContextValue WithCfg(Config::Key, std::move(Cfg)); + EXPECT_TRUE(StdlibIndexSpec::detect(parse({"clang++", "test.cc"}))); + + auto CXX11 = + StdlibIndexSpec::detect(parse({"clang++", "-std=gnu++11", "test.cc"})); + EXPECT_EQ(CXX11->LangStd, LangStandard::lang_cxx11); + + auto C11 = StdlibIndexSpec::detect(parse({"clang", "-std=c11", "test.c"})); + EXPECT_EQ(C11->LangStd, LangStandard::lang_c17); // No new features in 17. + + auto WithSysroot = StdlibIndexSpec::detect( + parse({"clang++", "-isysroot", testPath("blah"), "test.cc"})); + EXPECT_EQ(testPath("blah"), WithSysroot->Sysroot); +} + +} // namespace +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -1121,7 +1121,8 @@ public: BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N) : BlockVersion(BlockVersion), N(N) {} - void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx, + void onPreambleAST(PathRef Path, llvm::StringRef Version, + const CompilerInvocation &, ASTContext &Ctx, std::shared_ptr PP, const CanonicalIncludes &) override { if (Version == BlockVersion)