diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -8,6 +8,7 @@ #include "ClangdLSPServer.h" #include "Diagnostics.h" +#include "GlobalCompilationDatabase.h" #include "Protocol.h" #include "SourceCode.h" #include "Trace.h" @@ -336,9 +337,13 @@ ErrorCode::InvalidRequest)); if (const auto &Dir = Params.initializationOptions.compilationDatabasePath) CompileCommandsDir = Dir; - if (UseDirBasedCDB) + if (UseDirBasedCDB) { BaseCDB = llvm::make_unique( CompileCommandsDir); + if (!ClangdServerOpts.TrustedCompilerDriverGlob.empty()) + BaseCDB = getSystemIncludeExtractorDatabase( + ClangdServerOpts.TrustedCompilerDriverGlob, std::move(BaseCDB)); + } CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags, ClangdServerOpts.ResourceDir); Server.emplace(*CDB, FSProvider, static_cast(*this), diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -123,6 +123,10 @@ std::chrono::milliseconds(500); bool SuggestMissingIncludes = false; + + /// If set clangd will execute compiler drivers matching the following regex + /// to fetch system include path. + std::string TrustedCompilerDriverGlob; }; // Sensible default options for use in tests. // Features like indexing must be enabled if desired. diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -91,6 +91,14 @@ llvm::Optional CompileCommandsDir; }; +/// Extracts system include search path from trusted drivers and adds them to +/// the compile flags. +/// Returns nullptr when \p TrustedCompilerDriverGlob is empty or \p Base is +/// nullptr. +std::unique_ptr getSystemIncludeExtractorDatabase( + llvm::StringRef TrustedCompilerDriverGlob, + std::unique_ptr Base); + /// Wraps another compilation database, and supports overriding the commands /// using an in-memory mapping. class OverlayCDB : public GlobalCompilationDatabase { diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -8,12 +8,31 @@ #include "GlobalCompilationDatabase.h" #include "Logger.h" +#include "Path.h" +#include "Trace.h" +#include "clang/Driver/Types.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Tooling/ArgumentsAdjusters.h" #include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include namespace clang { namespace clangd { @@ -43,6 +62,157 @@ return CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy); } +std::vector parseDriverOutput(llvm::StringRef Output) { + std::vector SystemIncludes; + constexpr char const *SIS = "#include <...> search starts here:"; + constexpr char const *SIE = "End of search list."; + llvm::SmallVector Lines; + Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + bool FoundStart = false; + for (llvm::StringRef Line : Lines) { + if (Line == SIS) { + FoundStart = true; + continue; + } + if (Line == SIE) + break; + if (!FoundStart) + continue; + + if (!llvm::sys::fs::exists(Line)) + elog("Parsed non-existing system include: {0}", Line); + SystemIncludes.push_back(Line.str()); + vlog("Added: {0}", Line); + } + return SystemIncludes; +} + +std::vector extractSystemIncludes(PathRef Driver, PathRef File) { + trace::Span Tracer("Extract system includes"); + SPAN_ATTACH(Tracer, "driver", Driver); + SPAN_ATTACH(Tracer, "file", File); + + llvm::SmallString<128> OutputPath; + const auto EC = llvm::sys::fs::createTemporaryFile("system-includes", + "clangd", OutputPath); + if (EC) { + elog("Couldn't create temporary file: {0}", EC.message()); + return {}; + } + const llvm::Optional Redirects[] = { + {""}, llvm::StringRef(OutputPath), {""}}; + + const llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.'); + const auto Type = driver::types::lookupTypeForExtension(Ext); + // Should we also preserve flags like "-sysroot", "-nostdinc" ? + const llvm::StringRef Args[] = {"-E", "-x", driver::types::getTypeName(Type), + "-", "-v"}; + + const int RC = + llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None, Redirects); + if (RC) { + elog("Execution failed with return code: {0}", llvm::to_string(RC)); + return {}; + } + + auto BufOrError = llvm::MemoryBuffer::getFile(OutputPath); + if (!BufOrError) { + elog("Failed to read {0}: {1}", OutputPath, + BufOrError.getError().message()); + return {}; + } + + return parseDriverOutput(BufOrError->get()->getBuffer()); +} + +tooling::CompileCommand +addSystemIncludes(tooling::CompileCommand Cmd, + llvm::ArrayRef SystemIncludes) { + for (llvm::StringRef Include : SystemIncludes) { + Cmd.CommandLine.push_back("-isystem"); + Cmd.CommandLine.push_back(Include.str()); + } + return Cmd; +} + +/// Converts a glob containing only ** or * into a regex. +llvm::Regex convertGlobToRegex(llvm::StringRef Glob) { + const llvm::StringRef MetaChars("()^$|*+?.[]\\{}"); + std::string RegText; + llvm::raw_string_ostream RegStream(RegText); + RegStream << '^'; + for (size_t I = 0, E = Glob.size(); I < E; ++I) { + if (Glob[I] == '*') { + if (I + 1 < E && Glob[I + 1] == '*') { + // Double star, accept any sequence. + RegStream << ".*"; + // Also skip the second star. + ++I; + } else { + // Single star, accept any sequence without a slash. + RegStream << "[^/]*"; + } + } else if (MetaChars.find(Glob[I]) != MetaChars.npos) + RegStream << '\\' << Glob[I]; + else + RegStream << Glob[I]; + } + RegStream << '$'; + RegStream.flush(); + + llvm::Regex Result(RegText); + assert(Result.isValid(RegText) && "Glob to regex conversion failed"); + return Result; +} + +/// Extracts system includes from a trusted driver by parsing the output of +/// include search path and appends them to the commands coming from underlying +/// compilation database. +class SystemIncludeExtractorDatabase : public GlobalCompilationDatabase { +public: + SystemIncludeExtractorDatabase( + llvm::StringRef TrustedCompilerDriverGlob, + std::unique_ptr Base) + : TrustedCompilerDriverRegex( + convertGlobToRegex(TrustedCompilerDriverGlob)), + Base(std::move(Base)) { + assert(this->Base && "Base cannot be null!"); + assert(!TrustedCompilerDriverGlob.empty() && + "TrustedCompilerDriverGlob cannot be empty!"); + } + + llvm::Optional + getCompileCommand(PathRef File, ProjectInfo *PI = nullptr) const override { + auto Cmd = Base->getCompileCommand(File, PI); + if (!Cmd) + return llvm::None; + + llvm::StringRef Driver = Cmd->CommandLine.front(); + + llvm::ArrayRef SystemIncludes; + { + std::lock_guard Lock(Mu); + if (!TrustedCompilerDriverRegex.match(Driver)) + return Cmd; + + auto It = DriverToIncludes.try_emplace(Driver); + if (It.second) + It.first->second = extractSystemIncludes(Driver, File); + SystemIncludes = It.first->second; + } + + return addSystemIncludes(*Cmd, SystemIncludes); + } + +private: + mutable std::mutex Mu; + // Caches includes extracted from a driver. + mutable llvm::StringMap> DriverToIncludes; + mutable llvm::Regex TrustedCompilerDriverRegex; + + std::unique_ptr Base; +}; + } // namespace static std::string getFallbackClangPath() { @@ -188,5 +358,12 @@ OnCommandChanged.broadcast({File}); } +std::unique_ptr getSystemIncludeExtractorDatabase( + llvm::StringRef TrustedCompilerDriverGlob, + std::unique_ptr Base) { + return llvm::make_unique( + TrustedCompilerDriverGlob, std::move(Base)); +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/test/system-include-extractor.test b/clang-tools-extra/clangd/test/system-include-extractor.test new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/test/system-include-extractor.test @@ -0,0 +1,41 @@ +# RUN: rm -rf %t.dir && mkdir -p %t.dir + +# RUN: echo '#!/bin/bash' >> %t.dir/my_driver.sh +# RUN: echo 'echo line to ignore' >> %t.dir/my_driver.sh +# RUN: echo 'echo \#include \<...\> search starts here:' >> %t.dir/my_driver.sh +# RUN: echo 'echo %t.dir/my/dir/' >> %t.dir/my_driver.sh +# RUN: echo 'echo End of search list.' >> %t.dir/my_driver.sh +# RUN: chmod +x %t.dir/my_driver.sh + +# RUN: mkdir -p %t.dir/my/dir +# RUN: touch %t.dir/my/dir/a.h + +# RUN: echo '[{"directory": "%/t.dir", "command": "%/t.dir/my_driver.sh the-file.cpp", "file": "the-file.cpp"}]' > %t.dir/compile_commands.json + +# RUN: sed -e "s|INPUT_DIR|%/t.dir|g" %s > %t.test.1 +# On Windows, we need the URI in didOpen to look like "uri":"file:///C:/..." +# (with the extra slash in the front), so we add it here. +# RUN: sed -e "s|file://\([A-Z]\):/|file:///\1:/|g" %t.test.1 > %t.test + +# RUN: clangd -lit-test -trusted-compiler-driver-glob="**/*.sh" < %t.test | FileCheck -strict-whitespace %t.test +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}} +--- +{ + "jsonrpc":"2.0", + "method":"textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://INPUT_DIR/the-file.cpp", + "languageId":"cpp", + "version":1, + "text":"#include " + } + } +} +# CHECK: "method": "textDocument/publishDiagnostics", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "diagnostics": [], +--- +{"jsonrpc":"2.0","id":10000,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp --- a/clang-tools-extra/clangd/tool/ClangdMain.cpp +++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -268,6 +268,13 @@ "Always used text-based completion")), llvm::cl::init(CodeCompleteOptions().RunParser), llvm::cl::Hidden); +static llvm::cl::opt TrustedCompilerDriverGlob( + "trusted-compiler-driver-glob", + llvm::cl::desc("Tells clangd to extract system includes from drivers " + "maching the glob. Only accepts ** for sequence match and * " + "for non-sequence match."), + llvm::cl::init("")); + namespace { /// \brief Supports a test URI scheme with relaxed constraints for lit tests. @@ -521,6 +528,7 @@ return ClangTidyOptProvider->getOptions(File); }; Opts.SuggestMissingIncludes = SuggestMissingIncludes; + Opts.TrustedCompilerDriverGlob = TrustedCompilerDriverGlob; llvm::Optional OffsetEncodingFromFlag; if (ForceOffsetEncoding != OffsetEncoding::UnsupportedEncoding) OffsetEncodingFromFlag = ForceOffsetEncoding;