diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -118,6 +118,7 @@ index/Ref.cpp index/Relation.cpp index/Serialization.cpp + index/StdLib.cpp index/Symbol.cpp index/SymbolCollector.cpp index/SymbolID.cpp diff --git a/clang-tools-extra/clangd/FS.h b/clang-tools-extra/clangd/FS.h --- a/clang-tools-extra/clangd/FS.h +++ b/clang-tools-extra/clangd/FS.h @@ -74,6 +74,15 @@ /// filtering everything we get over LSP, CDB, etc. Path removeDots(PathRef File); +/// Get a virtual root node for the filesystem depending on the OS +inline const llvm::StringLiteral virtualRoot() { +#ifdef _WIN32 + return "\\"; +#else + return "/"; +#endif +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/index/StdLib.h b/clang-tools-extra/clangd/index/StdLib.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.h @@ -0,0 +1,70 @@ +//===--- StdLib.h ------------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Clangd indexer for the C++ standard library. +// +// The index only contains symbols that are part of the translation unit. So +// if your translation unit does not yet #include , you do not get +// auto completion for std::string. However we expect that many users would +// like to use the the standard library anyway, so we could index that by +// default an offer e.g. code completion without requiring #includes. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H + +#include "index/Index.h" +#include "index/MemIndex.h" +#include "support/ThreadsafeFS.h" +#include "clang/AST/Expr.h" +#include "llvm/ADT/StringRef.h" +#include + +namespace clang { +namespace clangd { + +// Enumeration of supported Standard Library versions. +// FIXME: support muiltiple languages (e.g. C and C++) and versions (e.g. 11, +// 14, 17) of the standard library. Right now hardcoded to one verison. +// FIXME: add feature to detect this version somehow (magically). +enum StandardLibraryVersion { cxx14 = 0 }; + +// external interface for getting a standard library index. +Expected> +indexStandardLibrary(const ThreadsafeFS &TFS, + const StandardLibraryVersion LibraryVersion = + StandardLibraryVersion::cxx14); + +class StandardLibraryIndex { + // Implementation for generating the index. + // TODO: do we really need to make verything public just for unittesting? +public: + /* virtual file name for indexing */ + // TODO: can we make this const static somehow? + const std::string VirtualUmbrellaHeaderFileName; + const ThreadsafeFS &TFS; + const StandardLibraryVersion LibraryVersion; + + StandardLibraryIndex(const ThreadsafeFS &TFS, + StandardLibraryVersion LibraryVersion); + + /* generate the index */ + Expected> + indexHeaders(llvm::StringRef HeaderSources); + + /* generate header containing #includes for all standard library headers */ + llvm::StringRef generateIncludeHeader(); + + /* build a virtual filesystem with the file to be indexed */ + llvm::IntrusiveRefCntPtr + buildFilesystem(std::string HeaderSources); +}; + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H \ No newline at end of file diff --git a/clang-tools-extra/clangd/index/StdLib.cpp b/clang-tools-extra/clangd/index/StdLib.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/index/StdLib.cpp @@ -0,0 +1,132 @@ +//===-- StdLib.cpp ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "StdLib.h" +#include +#include +#include + +#include "Compiler.h" +#include "FS.h" +#include "SymbolCollector.h" +#include "dex/Dex.h" +#include "index/Index.h" +#include "index/IndexAction.h" +#include "index/Ref.h" +#include "index/Serialization.h" +#include "support/Logger.h" +#include "support/Trace.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace clang { +namespace clangd { + +Expected> +indexStandardLibrary(const ThreadsafeFS &TFS, + const StandardLibraryVersion LibraryVersion) { + StandardLibraryIndex SLI(TFS, LibraryVersion); + return SLI.indexHeaders(SLI.generateIncludeHeader()); +} + +StandardLibraryIndex::StandardLibraryIndex( + const ThreadsafeFS &TFS, StandardLibraryVersion LibraryVersion) + : VirtualUmbrellaHeaderFileName(virtualRoot().str() + "UmbrellaHeader.hpp"), + TFS(TFS), LibraryVersion(LibraryVersion) {} + +// build umbrelle header for the the standard library. +// FIXME: consider different library versions, based on `LibraryVersion` +llvm::StringRef StandardLibraryIndex::generateIncludeHeader() { + + // bild the string only once and cache the results in `*Once`. + static std::string Once = [] { + std::vector Headers; + std::string Result; + +#define SYMBOL(Name, NameSpace, Header) Headers.push_back(#Header); +#include "StdSymbolMap.inc" +#undef SYMBOL + + // deduplication of the headers + llvm::sort(Headers.begin(), Headers.end()); + auto Last = std::unique(Headers.begin(), Headers.end()); + for (auto Header = Headers.begin(); Header != Last; ++Header) { + Result += "#include " + Header->str() + "\n"; + } + return Result; + }(); + return llvm::StringRef(Once); +} + +Expected> +StandardLibraryIndex::indexHeaders(llvm::StringRef HeaderSources) { + trace::Span Tracer("StandardLibraryIndex"); + ParseInputs Inputs; + Inputs.TFS = &TFS; + // TODO: can we get a real compile command from somewhere? + Inputs.CompileCommand.Directory = virtualRoot().str(); + Inputs.CompileCommand.Filename = VirtualUmbrellaHeaderFileName; + Inputs.CompileCommand.CommandLine.push_back("clang++"); + Inputs.CompileCommand.CommandLine.push_back(Inputs.CompileCommand.Filename); + + IgnoreDiagnostics IgnoreDiags; + + auto CI = buildCompilerInvocation(Inputs, IgnoreDiags); + if (!CI) + return error("Couldn't build compiler invocation"); + + auto Buffer = llvm::MemoryBuffer::getMemBuffer( + HeaderSources, VirtualUmbrellaHeaderFileName, + /*RequiresNullTerminator=*/false); + assert(Buffer && Buffer->getBufferSize() > 0); + auto Clang = prepareCompilerInstance( + std::move(CI), /*Preamble=*/nullptr, std::move(Buffer), + TFS.view(/*CWD=*/llvm::None), IgnoreDiags); + if (!Clang) + return error("Couldn't build compiler instance"); + + SymbolCollector::Options IndexOpts; + IndexFileIn IndexSlabs; + + // we only care about the symbols, so not storing the other attributes + auto Action = createStaticIndexingAction( + IndexOpts, [&](SymbolSlab S) { IndexSlabs.Symbols = std::move(S); }, + [&](RefSlab R) {}, [&](RelationSlab R) {}, [&](IncludeGraph IG) {}); + + const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front(); + if (!Action->BeginSourceFile(*Clang, Input)) + return error("BeginSourceFile() failed"); + if (llvm::Error Err = Action->Execute()) + return std::move(Err); + + Action->EndSourceFile(); + + assert(IndexSlabs.Symbols && "Symbols must be set."); + + log("Indexed standard library: {0} ({1} symbols)", + Inputs.CompileCommand.Filename, IndexSlabs.Symbols->size()); + + SPAN_ATTACH(Tracer, "symbols", int(IndexSlabs.Symbols->size())); + + bool HadErrors = Clang->hasDiagnostics() && + Clang->getDiagnostics().hasUncompilableErrorOccurred(); + if (HadErrors) { + log("Failed to compile standard library index, index may be incomplete"); + } + + // FIXME: filter Symbols to only include those in our list, rather than + // all the private implementation cruft. (I expect cruft is the + // majority by weight) + auto Index = std::make_unique( + std::move(IndexSlabs.Symbols.getValue()), RefSlab(), RelationSlab()); + + return Index; +} +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -79,6 +79,7 @@ SemanticSelectionTests.cpp SerializationTests.cpp SourceCodeTests.cpp + StdLibIndexTests.cpp SymbolCollectorTests.cpp SymbolInfoTests.cpp SyncAPI.cpp diff --git a/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp b/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/StdLibIndexTests.cpp @@ -0,0 +1,60 @@ +//===-- StdLibIndexTests.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "TestFS.h" +#include "TestIndex.h" +#include "index/StdLib.h" +#include "support/Logger.h" +#include "llvm/Support/Error.h" +#include "llvm/Testing/Support/Error.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace testing; + +namespace clang { +namespace clangd { + +/// check that the generated header sources contains some usual standard library +/// headers +TEST(StdLibIndexTests, generateUmbrellaHeader) { + MockFS FS; + StandardLibraryIndex SLI(FS, StandardLibraryVersion::cxx14); + auto UmbrellaHeader = SLI.generateIncludeHeader(); + + EXPECT_THAT(UmbrellaHeader, HasSubstr("#include ")); + EXPECT_THAT(UmbrellaHeader, HasSubstr("#include ")); + EXPECT_THAT(UmbrellaHeader, HasSubstr("#include ")); +} + +/// build the index and check if it contains the right symbols +TEST(StdLibIndexTests, buildIndex) { + MockFS FS; + StandardLibraryIndex SLI(FS, StandardLibraryVersion::cxx14); + // TODO: maybe find a way to use a local libcxx for testing if that is + // available on the machine + std::string HeaderMock = R"CPP( + int myfunc(int a); + bool otherfunc(int a, int b); + )CPP"; + auto Index = SLI.indexHeaders(HeaderMock); + EXPECT_THAT_EXPECTED(Index, llvm::Succeeded()); + + FuzzyFindRequest Req; + Req.AnyScope = true; + + EXPECT_THAT(match(**Index, Req), + UnorderedElementsAre(llvm::StringRef("myfunc"), + llvm::StringRef("otherfunc"))); +} + +// TODO: add tests for indexStandardLibrary() +// TODO: test with different library versions + +} // namespace clangd +} // namespace clang