diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -36,6 +36,10 @@ CollectMacros.cpp CompileCommands.cpp Compiler.cpp + Config.cpp + ConfigCompile.cpp + ConfigProvider.cpp + ConfigYAML.cpp Diagnostics.cpp DraftStore.cpp ExpectedTypes.cpp diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -11,6 +11,7 @@ #include "../clang-tidy/ClangTidyOptions.h" #include "CodeComplete.h" +#include "ConfigProvider.h" #include "GlobalCompilationDatabase.h" #include "Hover.h" #include "Protocol.h" @@ -23,6 +24,7 @@ #include "refactor/Rename.h" #include "refactor/Tweak.h" #include "support/Cancellation.h" +#include "support/Context.h" #include "support/Function.h" #include "support/ThreadsafeFS.h" #include "clang/Tooling/CompilationDatabase.h" @@ -31,6 +33,7 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/SourceMgr.h" #include #include #include @@ -113,6 +116,9 @@ /// If set, use this index to augment code completion results. SymbolIndex *StaticIndex = nullptr; + /// If set, queried to obtain the configuration to handle each request. + config::ConfigProvider *ConfigProvider = nullptr; + /// If set, enable clang-tidy in clangd and use to it get clang-tidy /// configurations for a particular file. /// Clangd supports only a small subset of ClangTidyOptions, these options @@ -246,6 +252,8 @@ void findReferences(PathRef File, Position Pos, uint32_t Limit, Callback CB); + /// FIXME: Make formatting methods async so they will respect configuration. + /// Run formatting for \p Rng inside \p File with content \p Code. llvm::Expected formatRange(StringRef Code, PathRef File, Range Rng); @@ -330,8 +338,11 @@ ArrayRef Ranges); const ThreadsafeFS &TFS; + const config::ConfigProvider *ConfigProvider; + /// Extend the current context with the configuration for a file. + /// Path may be empty, in which case we want "generic" config. + Context getConfiguredContext(PathRef) const; - Path ResourceDir; // The index used to look up symbols. This could be: // - null (all index functionality is optional) // - the dynamic index owned by ClangdServer (DynamicIdx) diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -8,6 +8,7 @@ #include "ClangdServer.h" #include "CodeComplete.h" +#include "Config.h" #include "FindSymbols.h" #include "Format.h" #include "HeaderSourceSwitch.h" @@ -45,6 +46,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -132,7 +134,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS, const Options &Opts, Callbacks *Callbacks) - : TFS(TFS), + : TFS(TFS), ConfigProvider(Opts.ConfigProvider), DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex(Opts.HeavyweightDynamicSymbolIndex) : nullptr), @@ -147,7 +149,14 @@ // FIXME(ioeric): this can be slow and we may be able to index on less // critical paths. WorkScheduler( - CDB, TUScheduler::Options(Opts), + CDB, + [&, this] { + TUScheduler::Options O(Opts); + O.ContextProvider = [this](PathRef P) { + return getConfiguredContext(P); + }; + return O; + }(), std::make_unique( DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) { // Adds an index to the stack, at higher priority than existing indexes. @@ -170,7 +179,8 @@ [Callbacks](BackgroundQueue::Stats S) { if (Callbacks) Callbacks->onBackgroundIndexProgress(S); - }); + }, + [this](PathRef P) { return getConfiguredContext(P); }); AddIndex(BackgroundIdx.get()); } if (DynamicIdx) @@ -636,7 +646,7 @@ llvm::StringRef Query, int Limit, Callback> CB) { WorkScheduler.run( - "getWorkspaceSymbols", + "getWorkspaceSymbols", /*Path=*/llvm::None, [Query = Query.str(), Limit, CB = std::move(CB), this]() mutable { CB(clangd::getWorkspaceSymbols(Query, Limit, Index, WorkspaceRoot.getValueOr(""))); @@ -726,6 +736,25 @@ return WorkScheduler.fileStats(); } +Context ClangdServer::getConfiguredContext(PathRef Path) const { + if (!ConfigProvider) + return Context::current().clone(); + + trace::Span Tracer("Config"); + SPAN_ATTACH(Tracer, "File", Path.empty() ? "" : Path.str()); + + config::Params Params; + Params.Path = Path.str(); + Config C = + ConfigProvider->getConfig(Params, [](const llvm::SMDiagnostic &Diag) { + log("config {0} at {1}:{2}:{3}: {4}", + Diag.getKind() == llvm::SourceMgr::DK_Error ? "error" : "warning", + Diag.getFilename(), Diag.getLineNo(), Diag.getColumnNo(), + Diag.getMessage()); + }); + return Context::current().derive(Config::Key, std::move(C)); +} + LLVM_NODISCARD bool ClangdServer::blockUntilIdleForTest(llvm::Optional TimeoutSeconds) { return WorkScheduler.blockUntilIdle(timeoutSeconds(TimeoutSeconds)) && diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp --- a/clang-tools-extra/clangd/CompileCommands.cpp +++ b/clang-tools-extra/clangd/CompileCommands.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "CompileCommands.h" +#include "Config.h" #include "support/Logger.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Tooling/ArgumentsAdjusters.h" @@ -182,6 +183,12 @@ } void CommandMangler::adjust(std::vector &Cmd) const { + const auto &Cfg = Config::current(); + llvm::errs() << "mangle edits... " << Cfg.CompileFlags.Edits.size() << "\n"; + // FIXME: can we make unique_function const-correct? + for (auto &Edit : const_cast(Cfg).CompileFlags.Edits) + Edit(Cmd); + // Check whether the flag exists, either as -flag or -flag=* auto Has = [&](llvm::StringRef Flag) { for (llvm::StringRef Arg : Cmd) { diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/Config.h @@ -0,0 +1,68 @@ +//===--- Config.h - User configuration of clangd behavior --------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Various clangd features have configurable behaviour (or can be disabled). +// This file defines "resolved" configuration seen by features within clangd. +// +// Because this structure is shared throughout clangd, it's a potential source +// of layering problems. Config should be expressed in terms of simple +// vocubulary types where possible. +// +// For how this is assembled from loaded fragments, see ConfigProvider.h. +// +//===----------------------------------------------------------------------===// +// +// To add a new configuration option, you must: +// - add its syntactic form to ConfigFragment (in ConfigProvider.h) +// - update ConfigYAML.cpp to parse it and populate ConfigFragment +// - add its semantic form to Config (in Config.h) +// - update ConfigCompile.cpp to map ConfigFragment -> Config +// - make use of the option inside clangd +// - document the new option (config.md in the llvm/clangd-www repository) +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H + +#include "support/Context.h" +#include "llvm/ADT/FunctionExtras.h" +#include +#include + +namespace clang { +namespace clangd { + +/// Settings that express user/project preferences and control clangd behavior. +/// +/// Generally, features should use Config::current() and ClangdServer is +/// responsible for ensuring the correct config is available. +/// (There are exceptions: e.g. background-indexing obtains configs directly). +struct Config { + /// Returns the Config of the current Context, or an empty configuration. + static const Config ¤t(); + /// Context key which can be used to set the current Config. + static clangd::Key Key; + + Config() = default; + Config(const Config &) = delete; + Config &operator=(const Config &) = delete; + Config(Config &&) = default; + Config &operator=(Config &&) = default; + + /// Controls how the compile command for the current file is determined. + struct { + // Sequence of edits to apply to the compile command. + std::vector &)>> Edits; + } CompileFlags; +}; + +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/Config.cpp b/clang-tools-extra/clangd/Config.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/Config.cpp @@ -0,0 +1,25 @@ +//===--- Config.cpp - User configuration of clangd behavior ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "support/Context.h" + +namespace clang { +namespace clangd { + +Key Config::Key; + +const Config &Config::current() { + if (const Config *C = Context::current().get(Key)) + return *C; + static Config Default; + return Default; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigCompile.cpp @@ -0,0 +1,171 @@ +//===--- ConfigCompile.cpp - Translating ConfigFragments into Config ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ConfigFragments are applied to Configs in two steps: +// +// 1. (When the fragment is first loaded) +// FragmentCompiler::compile() traverses the ConfigFragment and creates +// function objects that know how to apply the configuration. +// 2. (Every time a config is required) +// CompiledConfigFragment::apply() executes these functions to create the +// configuration. +// +// Work could be split between these steps in different ways. We try to +// do as much work as possible in the first step. For example, regexes are +// compiled in stage 1 and captured by the apply function. This is because: +// +// - it's more efficient, as the work done in stage 1 must only be done once +// - problems can be reported in stage 1, in stage 2 we must silently recover +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "ConfigProvider.h" +#include "support/Logger.h" +#include "support/Trace.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" + +namespace clang { +namespace clangd { +namespace config { + +// Logic to compile and apply config fragments. +namespace { + +// Wrapper around condition compile() functions to reduce arg-passing. +struct FragmentCompiler { + std::vector> &Conditions; + std::vector> &Apply; + DiagnosticCallback Diagnostic; + llvm::SourceMgr *SourceMgr; + + std::string RegexError = ""; + llvm::Optional compileRegex(const Located &Text) { + std::string Anchored = "^(" + *Text + ")$"; + llvm::Regex Result(Anchored); + if (!Result.isValid(RegexError)) { + diag(Error, "Invalid regex " + Anchored + ": " + RegexError, Text.Range); + return llvm::None; + } + return Result; + } + + void compile(ConfigFragment &&F) { + if (F.Condition) + compile(std::move(*F.Condition)); + compile(std::move(F.CompileFlags)); + } + + void compile(ConfigFragment::ConditionFragment &&F) { + if (F.UnrecognizedCondition) { + Conditions.push_back([&](const Params &) { return false; }); + return; + } + std::string RegexError; + + auto PathMatch = std::make_unique>(); + for (auto &Entry : F.PathMatch) { + if (auto RE = compileRegex(Entry)) + PathMatch->push_back(std::move(*RE)); + } + if (!PathMatch->empty()) { + Conditions.push_back([PathMatch(std::move(PathMatch))](const Params &P) { + if (P.Path.empty()) + return false; + return llvm::all_of(*PathMatch, [&](const llvm::Regex &RE) { + return RE.match(P.Path); + }); + }); + } + } + + void compile(ConfigFragment::CompileFlagsFragment &&F) { + if (!F.Add.empty()) { + std::vector Add; + for (auto &A : F.Add) + Add.push_back(std::move(*A)); + Apply.push_back([Add(std::move(Add))](Config &C) { + C.CompileFlags.Edits.push_back([Add](std::vector &Args) { + Args.insert(Args.end(), Add.begin(), Add.end()); + }); + }); + } + } + + constexpr static auto Error = llvm::SourceMgr::DK_Error; + constexpr static auto Warning = llvm::SourceMgr::DK_Warning; + void diag(llvm::SourceMgr::DiagKind Kind, llvm::StringRef Message, + llvm::SMRange Range) { + if (Range.isValid() && SourceMgr != nullptr) + Diagnostic(SourceMgr->GetMessage(Range.Start, Kind, Message, Range)); + else + Diagnostic(llvm::SMDiagnostic("", Kind, Message)); + } +}; + +} // namespace + +CompiledConfigFragment::CompiledConfigFragment(ConfigFragment F, + DiagnosticCallback D) + : Directory(F.Source.Directory) { + llvm::StringRef SourceFile = ""; + std::pair LineCol = {0, 0}; + if (auto *SM = F.Source.Manager.get()) { + unsigned BufID = SM->getMainFileID(); + LineCol = SM->getLineAndColumn(F.Source.Location, BufID); + SourceFile = SM->getBufferInfo(BufID).Buffer->getBufferIdentifier(); + } + vlog("Compiling ConfigFragment {0}:{1} for {2} -> {3}", SourceFile, + LineCol.first, + F.Source.Directory.empty() ? "" : F.Source.Directory, this); + trace::Span Tracer("ConfigCompile"); + SPAN_ATTACH(Tracer, "ConfigFile", SourceFile); + + FragmentCompiler{Conditions, Apply, D, F.Source.Manager.get()}.compile( + std::move(F)); +} + +// Returns adjusted Params, with Path made relative if needed. +// Storage is uninitialized and may be used if a copy is needed. +static const Params &adjustParams(const Params &P, llvm::StringRef Directory, + Params &Storage) { + if (P.Path.empty() || Directory.empty()) + return P; + if (!llvm::StringRef(P.Path).startswith(Directory)) { + assert(false && "Trying to apply a config to a file it doesn't govern?"); + return P; + } + Storage = P; + Storage.Path.erase(0, Directory.size()); + return Storage; +} + +bool CompiledConfigFragment::apply(const Params &P, Config &C) const { + if (!Conditions.empty()) { + Params Storage; + const Params &Adjusted = adjustParams(P, Directory, Storage); + for (auto &C : Conditions) + if (!C(Adjusted)) { + dlog("ConfigFragment {0}: condition not met", this); + return false; + } + } + dlog("ConfigFragment {0}: applying {1} rules", this, Apply.size()); + for (auto &A : Apply) { + llvm::errs() << "applyX\n"; + A(C); + } + return true; +} + +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ConfigProvider.h b/clang-tools-extra/clangd/ConfigProvider.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigProvider.h @@ -0,0 +1,146 @@ +//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Various clangd features have configurable behaviour (or can be disabled). +// The configuration system allows users to control this: +// - in a user config file, a project config file, via LSP, or via flags +// - specifying different settings for different files +// This file defines the structures used for this, that produce a Config. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_PROVIDER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_PROVIDER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include +#include + +namespace clang { +namespace clangd { +class ThreadsafeFS; +struct Config; +namespace config { + +/// A token written in config along with its optional location in the file. +template struct Located { + Located(T Value, llvm::SMRange Range = {}) + : Range(Range), Value(std::move(Value)) {} + + llvm::SMRange Range; + T &operator->() { return Value; } + const T &operator->() const { return Value; } + T &operator*() { return Value; } + const T &operator*() const { return Value; } + +private: + T Value; +}; + +/// Describes the context used to evaluate configuration fragments. +struct Params { + /// Absolute path to file we're targeting. Unix slashes. + /// Empty if we're not configuring a particular file. + std::string Path; +}; + +/// Used to report problems in parsing or interpreting a config. +/// Errors reflect structurally invalid config that should be user-visible. +/// Warnings reflect e.g. unknown properties that are recoverable. +using DiagnosticCallback = llvm::function_ref; + +/// A chunk of configuration obtained from a config file, LSP, or elsewhere. +struct ConfigFragment { + /// Parses fragments from a YAML file (one from each --- delimited document). + /// Documents that contained fatal errors are omitted from the results. + /// BufferName is used for the SourceMgr and diagnostics. + static std::vector parseYAML(llvm::StringRef YAML, + llvm::StringRef BufferName, + DiagnosticCallback); + + struct SourceInfo { + /// Absolute path of directory this fragment is associated with, if any. + /// Unix slashes, including a trailing slash. + std::string Directory; + /// Retains a buffer of the original source this fragment was parsed from. + /// Locations within Located objects point into this SourceMgr. + /// Shared because multiple fragments are often parsed from one (YAML) file. + /// May be null, then all locations are ignored. + std::shared_ptr Manager; + /// The start of the original source for this fragment. + /// Only valid if SourceManager is set. + llvm::SMLoc Location; + }; + SourceInfo Source; + + struct ConditionFragment { + std::vector> PathMatch; + /// An unrecognized key was found while parsing the condition. + /// The condition will evaluate to false. + bool UnrecognizedCondition; + }; + llvm::Optional Condition; + + struct CompileFlagsFragment { + std::vector> Add; + } CompileFlags; +}; + +/// A chunk of configuration that has been fully analyzed and is ready to apply. +class CompiledConfigFragment { +public: + /// Analyzes and consumes a fragment, possibly yielding more diagnostics. + /// This always produces a usable compiled fragment (errors are recovered). + explicit CompiledConfigFragment(ConfigFragment, DiagnosticCallback); + + /// Updates the configuration to reflect settings from the fragment. + /// Returns true if the condition was met and the fragment was applied. + bool apply(const Params &, Config &) const; + +private: + std::string Directory; + // These are not really mutated, but llvm::unique_function doens't provide + // a const call operator. + mutable std::vector> Conditions; + mutable std::vector> Apply; +}; + +/// A source of configuration fragments. +class ConfigProvider { +public: + virtual ~ConfigProvider() = default; + + /// A provider that includes fragments from all the supplied providers. + static std::unique_ptr + combine(std::vector>); + + /// Provide fragments that may be relevant to the file. + /// The configuration provider is not responsible for testing conditions. + /// Providers are expected to cache compiled fragments, and only + /// reparse/recompile when the source data has changed. + /// When parsing/compiling, the DiagnosticCallback is used to report errors. + /// Usual thread-safety guarantees apply: this function must be threadsafe. + virtual std::vector> + getFragments(const Params &, DiagnosticCallback) const = 0; + + /// Build a config based on this provider. + Config getConfig(const Params &, DiagnosticCallback) const; +}; + +/// A provider that reads config from a YAML file in ~/.config/clangd. +std::unique_ptr createFileConfigProvider(const ThreadsafeFS &); + +} // namespace config +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ConfigProvider.cpp b/clang-tools-extra/clangd/ConfigProvider.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigProvider.cpp @@ -0,0 +1,127 @@ +//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ConfigProvider.h" +#include "Config.h" +#include "support/ThreadsafeFS.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include + +namespace clang { +namespace clangd { +namespace config { +namespace { + +class FileConfigProvider : public ConfigProvider { + const ThreadsafeFS &TFS; + std::string UserConfigFile; + + mutable std::mutex Mu; + struct CachedFragment { + unsigned Size = 0; + llvm::sys::TimePoint<> Mtime = {}; + std::vector> Fragments; + + bool invalidate(const llvm::vfs::Status &Status) { + if (Size == Status.getSize() && Mtime == Status.getLastModificationTime()) + return false; + Size = Status.getSize(); + Mtime = Status.getLastModificationTime(); + Fragments.clear(); + return true; + } + + void reset() { + Size = 0; + Mtime = {}; + Fragments.clear(); + } + }; + mutable CachedFragment UserConfigCache; + + std::vector> + getFragments(llvm::StringRef Path, CachedFragment &Cache, + DiagnosticCallback DC) const { + auto FS = TFS.view(/*CWD=*/llvm::None); + auto Stat = FS->status(Path); + if (!Stat || !Stat->isRegularFile()) { + // Config file doesn't exist. + Cache.reset(); + } else if (Cache.invalidate(*Stat)) { + // Config file has changed. + if (auto Buf = FS->getBufferForFile(Path)) { + for (auto &Fragment : + ConfigFragment::parseYAML(Buf->get()->getBuffer(), Path, DC)) { + Cache.Fragments.push_back(std::make_unique( + std::move(Fragment), DC)); + } + } + } + return Cache.Fragments; + } + +public: + FileConfigProvider(const ThreadsafeFS &TFS) : TFS(TFS) { + llvm::SmallString<64> UserConfig; + // FIXME: use XDG_CONFIG_HOME etc. + if (llvm::sys::path::home_directory(UserConfig)) { + llvm::sys::path::append(UserConfig, ".config", "clangd"); + UserConfigFile = UserConfig.str().str(); + } + } + + std::vector> + getFragments(const Params &, DiagnosticCallback DC) const override { + std::lock_guard Lock(Mu); + return getFragments(UserConfigFile, UserConfigCache, DC); + } +}; + +} // namespace + +std::unique_ptr +createFileConfigProvider(const ThreadsafeFS &TFS) { + return std::make_unique(TFS); +}; + +using ProviderList = std::vector>; + +std::unique_ptr +ConfigProvider::combine(ProviderList Providers) { + struct CombinedConfigProvider : ConfigProvider { + ProviderList Providers; + + std::vector> + getFragments(const Params &P, DiagnosticCallback DC) const override { + std::vector> Result; + for (const auto &Provider : Providers) { + for (auto &Fragment : Provider->getFragments(P, DC)) + Result.push_back(std::move(Fragment)); + } + return Result; + } + }; + auto Result = std::make_unique(); + Result->Providers = std::move(Providers); + return Result; +} + +Config ConfigProvider::getConfig(const Params &P, DiagnosticCallback DC) const { + Config C; + for (const auto &Fragment : getFragments(P, DC)) + Fragment->apply(P, C); + return C; +} + +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigYAML.cpp @@ -0,0 +1,180 @@ +//===--- ConfigYAML.cpp - Loading configuration fragments from YAML files -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ConfigProvider.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/YAMLParser.h" +#include + +namespace clang { +namespace clangd { +namespace config { +namespace { +using llvm::yaml::BlockScalarNode; +using llvm::yaml::MappingNode; +using llvm::yaml::Node; +using llvm::yaml::ScalarNode; +using llvm::yaml::SequenceNode; + +struct Parser { + llvm::SourceMgr &SM; + llvm::SmallString<256> Buf = {}; + + struct DictHandler { + DictHandler(llvm::StringRef Key, std::function Handle) + : Key(Key), Handle(Handle) {} + + llvm::StringRef Key; + std::function Handle; + }; + + bool parseDict(Node &N, llvm::StringRef Desc, + llvm::ArrayRef Handlers, + std::function Unknown = nullptr) { + if (N.getType() != Node::NK_Mapping) + return error(Desc + " should be a dictionary", N); + llvm::SmallSet Seen; + for (auto &KV : llvm::cast(N)) { + auto *K = KV.getKey(); + if (!K) + return false; + auto Key = scalarValue(*K, "Dictionary key"); + if (!Key) + continue; + if (!Seen.insert(**Key).second) { + warning("Duplicate key " + **Key, *K); + continue; + } + auto *Value = KV.getValue(); + if (!Value) + return false; + bool Matched = false; + for (const auto &Handler : Handlers) { + if (Handler.Key == **Key) { + if (!Handler.Handle(*Value)) + return false; + Matched = true; + break; + } + } + if (!Matched) { + warning("Unknown " + Desc + " key " + **Key, *K); + if (Unknown) + Unknown(); + } + } + return true; + } + + llvm::Optional> scalarValue(Node &N, + llvm::StringRef Desc) { + if (auto *S = llvm::dyn_cast(&N)) + return Located(S->getValue(Buf).str(), N.getSourceRange()); + else if (auto *BS = llvm::dyn_cast(&N)) + return Located(S->getValue(Buf).str(), N.getSourceRange()); + warning(Desc + " should be scalar", N); + return llvm::None; + } + + llvm::Optional>> scalarValues(Node &N) { + std::vector> Result; + if (auto *S = llvm::dyn_cast(&N)) { + Result.emplace_back(S->getValue(Buf).str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast(&N)) { + Result.emplace_back(S->getValue().str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast(&N)) { + for (auto &Child : *S) { + if (auto Value = scalarValue(Child, "List item")) + Result.push_back(std::move(*Value)); + } + } else { + warning("Expected scalar or list of scalars", N); + return llvm::None; + } + return Result; + } + + bool error(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg, + N.getSourceRange()); + return false; + } + + void warning(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Warning, Msg, + N.getSourceRange()); + } + + bool parse(ConfigFragment::ConditionFragment &F, Node &N) { + std::vector Keys; + Keys.emplace_back("PathMatch", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.PathMatch = std::move(*Values); + return !N.failed(); + }); + return parseDict(N, "Condition", Keys, + [&] { F.UnrecognizedCondition = true; }); + } + + bool parse(ConfigFragment::CompileFlagsFragment &F, Node &N) { + std::vector Keys; + Keys.emplace_back("Add", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.Add = std::move(*Values); + return !N.failed(); + }); + return parseDict(N, "CompileFlags", Keys); + } + + bool parse(ConfigFragment &F, Node &N) { + std::vector Keys; + Keys.emplace_back("If", [&](Node &N) { + F.Condition.emplace(); + return parse(*F.Condition, N); + }); + Keys.emplace_back("CompileFlags", + [&](Node &N) { return parse(F.CompileFlags, N); }); + return parseDict(N, "Config", Keys); + } +}; + +// Adapt DiagnosticCallback to DiagHandlerTy interface. +void diagHandler(const llvm::SMDiagnostic &Diag, void *Ctx) { + (*reinterpret_cast(Ctx))(Diag); +} + +} // namespace + +std::vector +ConfigFragment::parseYAML(llvm::StringRef YAML, llvm::StringRef BufferName, + DiagnosticCallback Diags) { + auto SM = std::make_shared(); + SM->setDiagHandler(&diagHandler, &Diags); + std::vector Result; + for (auto &Doc : + llvm::yaml::Stream(llvm::MemoryBufferRef(YAML, BufferName), *SM)) { + if (Node *N = Doc.parseBlockNode()) { + ConfigFragment Fragment; + Fragment.Source.Manager = SM; + Fragment.Source.Location = N->getSourceRange().Start; + if (Parser{*SM}.parse(Fragment, *N)) + Result.push_back(std::move(Fragment)); + } + } + return Result; +} + +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h --- a/clang-tools-extra/clangd/TUScheduler.h +++ b/clang-tools-extra/clangd/TUScheduler.h @@ -10,9 +10,11 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H #include "Compiler.h" +#include "ConfigProvider.h" #include "Diagnostics.h" #include "GlobalCompilationDatabase.h" #include "index/CanonicalIncludes.h" +#include "support/Context.h" #include "support/Function.h" #include "support/Path.h" #include "support/Threading.h" @@ -195,6 +197,11 @@ /// Whether to run PreamblePeer asynchronously. /// No-op if AsyncThreadsCount is 0. bool AsyncPreambleBuilds = false; + + /// Used to create a context that wraps each single operation. + /// Typically to inject per-file configuration. + /// If the path is empty, context should be "generic". + std::function ContextProvider; }; TUScheduler(const GlobalCompilationDatabase &CDB, const Options &Opts, @@ -233,7 +240,8 @@ llvm::StringMap getAllFileContents() const; /// Schedule an async task with no dependencies. - void run(llvm::StringRef Name, llvm::unique_function Action); + void run(llvm::StringRef Name, llvm::Optional File, + llvm::unique_function Action); /// Defines how a runWithAST action is implicitly cancelled by other actions. enum ASTActionInvalidation { diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -457,6 +457,7 @@ /// File that ASTWorker is responsible for. const Path FileName; const GlobalCompilationDatabase &CDB; + const std::function &ContextProvider; /// Callback invoked when preamble or main file AST is built. ParsingCallbacks &Callbacks; @@ -569,8 +570,9 @@ bool RunSync, const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks) : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(Opts.UpdateDebounce), - FileName(FileName), CDB(CDB), Callbacks(Callbacks), Barrier(Barrier), - Done(false), Status(FileName, Callbacks), + FileName(FileName), CDB(CDB), ContextProvider(Opts.ContextProvider), + Callbacks(Callbacks), Barrier(Barrier), Done(false), + Status(FileName, Callbacks), PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory, RunSync || !Opts.AsyncPreambleBuilds, Status, *this) { // Set a fallback command because compile command can be accessed before @@ -1055,6 +1057,9 @@ Status.ASTActivity.K = ASTAction::RunningAction; Status.ASTActivity.Name = CurrentRequest->Name; }); + llvm::Optional WithProvidedContext; + if (ContextProvider) + WithProvidedContext.emplace(ContextProvider(FileName)); CurrentRequest->Action(); } @@ -1288,14 +1293,18 @@ return Results; } -void TUScheduler::run(llvm::StringRef Name, +void TUScheduler::run(llvm::StringRef Name, llvm::Optional File, llvm::unique_function Action) { if (!PreambleTasks) return Action(); PreambleTasks->runAsync(Name, [this, Ctx = Context::current().clone(), + File(File.getValueOr("").str()), Action = std::move(Action)]() mutable { std::lock_guard BarrierLock(Barrier); WithContext WC(std::move(Ctx)); + llvm::Optional WithProvidedContext; + if (Opts.ContextProvider) + WithProvidedContext.emplace(Opts.ContextProvider(File)); Action(); }); } @@ -1356,6 +1365,10 @@ WithContext Guard(std::move(Ctx)); trace::Span Tracer(Name); SPAN_ATTACH(Tracer, "file", File); + llvm::Optional WithProvidedContext; + if (Opts.ContextProvider) + WithProvidedContext.emplace( + Opts.ContextProvider(llvm::StringRef(File))); Action(InputsAndPreamble{Contents, Command, Preamble.get()}); }; diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -70,6 +70,7 @@ struct Task { explicit Task(std::function Run) : Run(std::move(Run)) {} + std::string Path; // Used to determine config. May be empty. std::function Run; llvm::ThreadPriority ThreadPri = llvm::ThreadPriority::Background; unsigned QueuePri = 0; // Higher-priority tasks will run first. @@ -86,8 +87,10 @@ unsigned LastIdle = 0; // Number of completed tasks when last empty. }; - BackgroundQueue(std::function OnProgress = nullptr) - : OnProgress(OnProgress) {} + BackgroundQueue(std::function OnProgress = nullptr, + std::function ContextProvider = nullptr) + : ContextProvider(std::move(ContextProvider)), + OnProgress(std::move(OnProgress)) {} // Add tasks to the queue. void push(Task); @@ -113,6 +116,7 @@ private: void notifyProgress() const; // Requires lock Mu + std::function ContextProvider; // config for tasks std::mutex Mu; Stats Stat; std::condition_variable CV; @@ -136,7 +140,8 @@ const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize = 0, // 0 = use all hardware threads - std::function OnProgress = nullptr); + std::function OnProgress = nullptr, + std::function ContextProvider = nullptr); ~BackgroundIndex(); // Blocks while the current task finishes. // Enqueue translation units for indexing. @@ -191,12 +196,11 @@ BackgroundIndexStorage::Factory IndexStorageFactory; // Tries to load shards for the MainFiles and their dependencies. - std::vector - loadProject(std::vector MainFiles); + std::vector loadProject(std::vector MainFiles); BackgroundQueue::Task changedFilesTask(const std::vector &ChangedFiles); - BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd); + BackgroundQueue::Task indexFileTask(std::string Path); // from lowest to highest priority enum QueuePriority { diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -93,12 +93,13 @@ Context BackgroundContext, const ThreadsafeFS &TFS, const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize, - std::function OnProgress) + std::function OnProgress, + std::function ContextProvider) : SwapIndex(std::make_unique()), TFS(TFS), CDB(CDB), BackgroundContext(std::move(BackgroundContext)), Rebuilder(this, &IndexedSymbols, ThreadPoolSize), IndexStorageFactory(std::move(IndexStorageFactory)), - Queue(std::move(OnProgress)), + Queue(std::move(OnProgress), std::move(ContextProvider)), CommandsChanged( CDB.watch([&](const std::vector &ChangedFiles) { enqueue(ChangedFiles); @@ -133,8 +134,8 @@ std::mt19937(std::random_device{}())); std::vector Tasks; Tasks.reserve(NeedsReIndexing.size()); - for (auto &Cmd : NeedsReIndexing) - Tasks.push_back(indexFileTask(std::move(Cmd))); + for (std::string &Path : NeedsReIndexing) + Tasks.push_back(indexFileTask(std::move(Path))); Queue.append(std::move(Tasks)); }); @@ -148,17 +149,17 @@ return Path.drop_back(llvm::sys::path::extension(Path).size()); } -BackgroundQueue::Task -BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) { - BackgroundQueue::Task T([this, Cmd] { - // We can't use llvm::StringRef here since we are going to - // move from Cmd during the call below. - const std::string FileName = Cmd.Filename; - if (auto Error = index(std::move(Cmd))) - elog("Indexing {0} failed: {1}", FileName, std::move(Error)); +BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string Path) { + BackgroundQueue::Task T([this, Path] { + auto Cmd = CDB.getCompileCommand(Path); + if (!Cmd) + return; + if (auto Error = index(std::move(*Cmd))) + elog("Indexing {0} failed: {1}", Path, std::move(Error)); }); + T.Tag = filenameWithoutExtension(Path).str(); + T.Path = std::move(Path); T.QueuePri = IndexFile; - T.Tag = std::string(filenameWithoutExtension(Cmd.Filename)); return T; } @@ -242,6 +243,7 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) { trace::Span Tracer("BackgroundIndex"); SPAN_ATTACH(Tracer, "file", Cmd.Filename); + SPAN_ATTACH(Tracer, "cmd", Cmd.CommandLine); auto AbsolutePath = getAbsolutePath(Cmd); auto FS = TFS.view(Cmd.Directory); @@ -343,10 +345,8 @@ // Restores shards for \p MainFiles from index storage. Then checks staleness of // those shards and returns a list of TUs that needs to be indexed to update // staleness. -std::vector +std::vector BackgroundIndex::loadProject(std::vector MainFiles) { - std::vector NeedsReIndexing; - Rebuilder.startLoading(); // Load shards for all of the mainfiles. const std::vector Result = @@ -399,14 +399,7 @@ TUsToIndex.insert(TUForFile); } - for (PathRef TU : TUsToIndex) { - auto Cmd = CDB.getCompileCommand(TU); - if (!Cmd) - continue; - NeedsReIndexing.emplace_back(std::move(*Cmd)); - } - - return NeedsReIndexing; + return {TUsToIndex.begin(), TUsToIndex.end()}; } } // namespace clangd diff --git a/clang-tools-extra/clangd/index/BackgroundQueue.cpp b/clang-tools-extra/clangd/index/BackgroundQueue.cpp --- a/clang-tools-extra/clangd/index/BackgroundQueue.cpp +++ b/clang-tools-extra/clangd/index/BackgroundQueue.cpp @@ -39,7 +39,12 @@ if (Task->ThreadPri != llvm::ThreadPriority::Default && !PreventStarvation.load()) llvm::set_thread_priority(Task->ThreadPri); - Task->Run(); + { + llvm::Optional ProvidedContext; + if (ContextProvider) + ProvidedContext.emplace(ContextProvider(Task->Path)); + Task->Run(); + } if (Task->ThreadPri != llvm::ThreadPriority::Default) llvm::set_thread_priority(llvm::ThreadPriority::Default); diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp --- a/clang-tools-extra/clangd/tool/ClangdMain.cpp +++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -8,6 +8,8 @@ #include "ClangdLSPServer.h" #include "CodeComplete.h" +#include "Config.h" +#include "ConfigProvider.h" #include "Features.inc" #include "PathMapping.h" #include "Protocol.h" @@ -30,6 +32,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" #include @@ -180,6 +183,13 @@ "combined. Type information shown where possible")), }; +opt ConfigEnabled{ + "config", + cat(Misc), + desc("Enable experimental YAML configuration file support"), + init(false), +}; + opt FallbackStyle{ "fallback-style", cat(Features), @@ -629,6 +639,8 @@ } } + RealThreadsafeFS TFS; + ClangdServer::Options Opts; switch (PCHStorage) { case PCHStorageFlag::Memory: @@ -660,6 +672,12 @@ Opts.BuildRecoveryAST = RecoveryAST; Opts.PreserveRecoveryASTType = RecoveryASTType; + std::unique_ptr ConfigProvider; + if (ConfigEnabled) { + ConfigProvider = config::createFileConfigProvider(TFS); + Opts.ConfigProvider = ConfigProvider.get(); + } + clangd::CodeCompleteOptions CCOpts; CCOpts.IncludeIneligibleResults = IncludeIneligibleResults; CCOpts.Limit = LimitResults; @@ -676,7 +694,6 @@ CCOpts.AllScopes = AllScopesCompletion; CCOpts.RunParser = CodeCompletionParse; - RealThreadsafeFS TFS; // Initialize and run ClangdLSPServer. // Change stdin to binary to not lose \r\n on windows. llvm::sys::ChangeStdinToBinary(); @@ -768,7 +785,6 @@ RenameOpts.AllowCrossFile = CrossFileRename; Opts.AsyncPreambleBuilds = AsyncPreamble; - ClangdLSPServer LSPServer( *TransportLayer, TFS, CCOpts, RenameOpts, CompileCommandsDirPath, /*UseDirBasedCDB=*/CompileArgsFrom == FilesystemCompileArgs, diff --git a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp --- a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp +++ b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp @@ -7,7 +7,9 @@ //===----------------------------------------------------------------------===// #include "CompileCommands.h" +#include "Config.h" #include "TestFS.h" +#include "support/Context.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringExtras.h" @@ -185,6 +187,25 @@ } #endif +TEST(CommandMangler, ConfigEdits) { + auto Mangler = CommandMangler::forTests(); + std::vector Cmd = {"clang++", "foo.cc"}; + { + Config Cfg; + Cfg.CompileFlags.Edits.push_back([](std::vector &Argv) { + for (auto &Arg : Argv) + for (char &C : Arg) + C = llvm::toUpper(C); + }); + Cfg.CompileFlags.Edits.push_back( + [](std::vector &Argv) { Argv.push_back("--hello"); }); + WithContextValue WithConfig(Config::Key, std::move(Cfg)); + Mangler.adjust(Cmd); + } + EXPECT_THAT(Cmd, + ElementsAre("CLANG++", "FOO.CC", "--hello", "-fsyntax-only")); +} + } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -851,15 +851,15 @@ TEST_F(TUSchedulerTests, Run) { TUScheduler S(CDB, optsForTest()); std::atomic Counter(0); - S.run("add 1", [&] { ++Counter; }); - S.run("add 2", [&] { Counter += 2; }); + S.run("add 1", llvm::None, [&] { ++Counter; }); + S.run("add 2", llvm::None, [&] { Counter += 2; }); ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10))); EXPECT_EQ(Counter.load(), 3); Notification TaskRun; Key TestKey; WithContextValue CtxWithKey(TestKey, 10); - S.run("props context", [&] { + S.run("props context", llvm::None, [&] { EXPECT_EQ(Context::current().getExisting(TestKey), 10); TaskRun.notify(); });