[clangd] WIP: config

[sammccall]

Summary

This is a glorious tangled mess that needs to be split, and needs tests...

clang-tools-extra/clangd/CMakeLists.txt

add_clang_library(clangDaemon
   AST.cpp
   ClangdLSPServer.cpp
   ClangdServer.cpp
   CodeComplete.cpp
   CodeCompletionStrings.cpp
   CollectMacros.cpp
   CompileCommands.cpp
   Compiler.cpp
+  Config.cpp
+  ConfigCompile.cpp
+  ConfigProvider.cpp
+  ConfigYAML.cpp
   Diagnostics.cpp
   DraftStore.cpp
   ExpectedTypes.cpp
   FindSymbols.cpp
   FindTarget.cpp
   FileDistance.cpp
   Format.cpp
   FS.cpp

clang-tools-extra/clangd/ClangdServer.h

 //===--- ClangdServer.h - Main clangd server code ----------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDSERVER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDSERVER_H
 
 #include "../clang-tidy/ClangTidyOptions.h"
 #include "CodeComplete.h"
+#include "ConfigProvider.h"
 #include "GlobalCompilationDatabase.h"
 #include "Hover.h"
 #include "Protocol.h"
 #include "SemanticHighlighting.h"
 #include "TUScheduler.h"
 #include "XRefs.h"
 #include "index/Background.h"
 #include "index/FileIndex.h"
 #include "index/Index.h"
 #include "refactor/Rename.h"
 #include "refactor/Tweak.h"
 #include "support/Cancellation.h"
+#include "support/Context.h"
 #include "support/Function.h"
 #include "support/ThreadsafeFS.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/Core/Replacement.h"
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SourceMgr.h"
 #include <functional>
 #include <future>
 #include <string>
 #include <type_traits>
 #include <utility>
 
 namespace clang {
 namespace clangd {
struct Options {
     bool HeavyweightDynamicSymbolIndex = true;
     /// If true, ClangdServer automatically indexes files in the current project
     /// on background threads. The index is stored in the project root.
     bool BackgroundIndex = false;
 
     /// If set, use this index to augment code completion results.
     SymbolIndex *StaticIndex = nullptr;
 
+    /// If set, queried to obtain the configuration to handle each request.
+    config::ConfigProvider *ConfigProvider = nullptr;
+
     /// If set, enable clang-tidy in clangd and use to it get clang-tidy
     /// configurations for a particular file.
     /// Clangd supports only a small subset of ClangTidyOptions, these options
     /// (Checks, CheckOptions) are about which clang-tidy checks will be
     /// enabled.
     ClangTidyOptionsBuilder GetClangTidyOptions;
 
     /// If true, turn on the `-frecovery-ast` clang flag.
public:
   /// Retrieve the symbols within the specified file.
   void documentSymbols(StringRef File,
                        Callback<std::vector<DocumentSymbol>> CB);
 
   /// Retrieve locations for symbol references.
   void findReferences(PathRef File, Position Pos, uint32_t Limit,
                       Callback<ReferencesResult> CB);
 
+  /// FIXME: Make formatting methods async so they will respect configuration.
+
   /// Run formatting for \p Rng inside \p File with content \p Code.
   llvm::Expected<tooling::Replacements> formatRange(StringRef Code,
                                                     PathRef File, Range Rng);
 
   /// Run formatting for the whole \p File with content \p Code.
   llvm::Expected<tooling::Replacements> formatFile(StringRef Code,
                                                    PathRef File);
 

 private:
   /// FIXME: This stats several files to find a .clang-format file. I/O can be
   /// slow. Think of a way to cache this.
   llvm::Expected<tooling::Replacements>
   formatCode(llvm::StringRef Code, PathRef File,
              ArrayRef<tooling::Range> Ranges);
 
   const ThreadsafeFS &TFS;
+  const config::ConfigProvider *ConfigProvider;
+  /// Extend the current context with the configuration for a file.
+  /// Path may be empty, in which case we want "generic" config.
+  Context getConfiguredContext(PathRef) const;
 
-  Path ResourceDir;
   // The index used to look up symbols. This could be:
   //   - null (all index functionality is optional)
   //   - the dynamic index owned by ClangdServer (DynamicIdx)
   //   - the static index passed to the constructor
   //   - a merged view of a static and dynamic index (MergedIndex)
   const SymbolIndex *Index = nullptr;
   // If present, an index of symbols in open files. Read via *Index.
   std::unique_ptr<FileIndex> DynamicIdx;

clang-tools-extra/clangd/ClangdServer.cpp

 //===--- ClangdServer.cpp - Main clangd server code --------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-------------------------------------------------------------------===//
 
 #include "ClangdServer.h"
 #include "CodeComplete.h"
+#include "Config.h"
 #include "FindSymbols.h"
 #include "Format.h"
 #include "HeaderSourceSwitch.h"
 #include "Headers.h"
 #include "ParsedAST.h"
 #include "Preamble.h"
 #include "Protocol.h"
 #include "SemanticHighlighting.h"

 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <future>
 #include <memory>
 #include <mutex>
 #include <type_traits>
 
 namespace clang {
ClangdServer::Options::operator TUScheduler::Options() const {
   Opts.UpdateDebounce = UpdateDebounce;
   Opts.AsyncPreambleBuilds = AsyncPreambleBuilds;
   return Opts;
 }
 
 ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, const Options &Opts,
                            Callbacks *Callbacks)
-    : TFS(TFS),
+    : TFS(TFS), ConfigProvider(Opts.ConfigProvider),
       DynamicIdx(Opts.BuildDynamicSymbolIndex
                      ? new FileIndex(Opts.HeavyweightDynamicSymbolIndex)
                      : nullptr),
       GetClangTidyOptions(Opts.GetClangTidyOptions),
       SuggestMissingIncludes(Opts.SuggestMissingIncludes),
       BuildRecoveryAST(Opts.BuildRecoveryAST),
       PreserveRecoveryASTType(Opts.PreserveRecoveryASTType),
       TweakFilter(Opts.TweakFilter), WorkspaceRoot(Opts.WorkspaceRoot),
       // Pass a callback into `WorkScheduler` to extract symbols from a newly
       // parsed file and rebuild the file index synchronously each time an AST
       // is parsed.
       // FIXME(ioeric): this can be slow and we may be able to index on less
       // critical paths.
       WorkScheduler(
-          CDB, TUScheduler::Options(Opts),
+          CDB,
+          [&, this] {
+            TUScheduler::Options O(Opts);
+            O.ContextProvider = [this](PathRef P) {
+              return getConfiguredContext(P);
+            };
+            return O;
+          }(),
           std::make_unique<UpdateIndexCallbacks>(
               DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) {
   // Adds an index to the stack, at higher priority than existing indexes.
   auto AddIndex = [&](SymbolIndex *Idx) {
     if (this->Index != nullptr) {
       MergedIdx.push_back(std::make_unique<MergedIndex>(Idx, this->Index));
       this->Index = MergedIdx.back().get();
     } else {
       this->Index = Idx;
     }
   };
   if (Opts.StaticIndex)
     AddIndex(Opts.StaticIndex);
   if (Opts.BackgroundIndex) {
     BackgroundIdx = std::make_unique<BackgroundIndex>(
         Context::current().clone(), TFS, CDB,
         BackgroundIndexStorage::createDiskBackedStorageFactory(
             [&CDB](llvm::StringRef File) { return CDB.getProjectInfo(File); }),
         std::max(Opts.AsyncThreadsCount, 1u),
         [Callbacks](BackgroundQueue::Stats S) {
           if (Callbacks)
             Callbacks->onBackgroundIndexProgress(S);
-        });
+        },
+        [this](PathRef P) { return getConfiguredContext(P); });
     AddIndex(BackgroundIdx.get());
   }
   if (DynamicIdx)
     AddIndex(DynamicIdx.get());
 }
 
 void ClangdServer::addDocument(PathRef File, llvm::StringRef Contents,
                                llvm::StringRef Version,
void ClangdServer::onFileEvent(const DidChangeWatchedFilesParams &Params) {
   // FIXME: Do nothing for now. This will be used for indexing and potentially
   // invalidating other caches.
 }
 
 void ClangdServer::workspaceSymbols(
     llvm::StringRef Query, int Limit,
     Callback<std::vector<SymbolInformation>> CB) {
   WorkScheduler.run(
-      "getWorkspaceSymbols",
+      "getWorkspaceSymbols", /*Path=*/llvm::None,
       [Query = Query.str(), Limit, CB = std::move(CB), this]() mutable {
         CB(clangd::getWorkspaceSymbols(Query, Limit, Index,
                                        WorkspaceRoot.getValueOr("")));
       });
 }
 
 void ClangdServer::documentSymbols(llvm::StringRef File,
                                    Callback<std::vector<DocumentSymbol>> CB) {
void ClangdServer::semanticHighlights(
   WorkScheduler.runWithAST("SemanticHighlights", File, std::move(Action),
                            TUScheduler::InvalidateOnUpdate);
 }
 
 llvm::StringMap<TUScheduler::FileStats> ClangdServer::fileStats() const {
   return WorkScheduler.fileStats();
 }
 
+Context ClangdServer::getConfiguredContext(PathRef Path) const {
+  if (!ConfigProvider)
+    return Context::current().clone();
+
+  trace::Span Tracer("Config");
+  SPAN_ATTACH(Tracer, "File", Path.empty() ? "<global>" : Path.str());
+
+  config::Params Params;
+  Params.Path = Path.str();
+  Config C =
+      ConfigProvider->getConfig(Params, [](const llvm::SMDiagnostic &Diag) {
+        log("config {0} at {1}:{2}:{3}: {4}",
+            Diag.getKind() == llvm::SourceMgr::DK_Error ? "error" : "warning",
+            Diag.getFilename(), Diag.getLineNo(), Diag.getColumnNo(),
+            Diag.getMessage());
+      });
+  return Context::current().derive(Config::Key, std::move(C));
+}
+
 LLVM_NODISCARD bool
 ClangdServer::blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds) {
   return WorkScheduler.blockUntilIdle(timeoutSeconds(TimeoutSeconds)) &&
          (!BackgroundIdx ||
           BackgroundIdx->blockUntilIdleForTest(TimeoutSeconds));
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/CompileCommands.cpp

 //===--- CompileCommands.cpp ----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CompileCommands.h"
+#include "Config.h"
 #include "support/Logger.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
CommandMangler CommandMangler::detect() {
   return Result;
 }
 
 CommandMangler CommandMangler::forTests() {
   return CommandMangler();
 }
 
 void CommandMangler::adjust(std::vector<std::string> &Cmd) const {
+  const auto &Cfg = Config::current();
+  llvm::errs() << "mangle edits... " << Cfg.CompileFlags.Edits.size() << "\n";
+  // FIXME: can we make unique_function const-correct?
+  for (auto &Edit : const_cast<Config &>(Cfg).CompileFlags.Edits)
+    Edit(Cmd);
+
   // Check whether the flag exists, either as -flag or -flag=*
   auto Has = [&](llvm::StringRef Flag) {
     for (llvm::StringRef Arg : Cmd) {
       if (Arg.consume_front(Flag) && (Arg.empty() || Arg[0] == '='))
         return true;
     }
     return false;
   };

clang-tools-extra/clangd/Config.h

+//===--- Config.h - User configuration of clangd behavior --------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various clangd features have configurable behaviour (or can be disabled).
+// This file defines "resolved" configuration seen by features within clangd.
+//
+// Because this structure is shared throughout clangd, it's a potential source
+// of layering problems. Config should be expressed in terms of simple
+// vocubulary types where possible.
+//
+// For how this is assembled from loaded fragments, see ConfigProvider.h.
+//
+//===----------------------------------------------------------------------===//
+//
+// To add a new configuration option, you must:
+//  - add its syntactic form to ConfigFragment (in ConfigProvider.h)
+//  - update ConfigYAML.cpp to parse it and populate ConfigFragment
+//  - add its semantic form to Config (in Config.h)
+//  - update ConfigCompile.cpp to map ConfigFragment -> Config
+//  - make use of the option inside clangd
+//  - document the new option (config.md in the llvm/clangd-www repository)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_H
+
+#include "support/Context.h"
+#include "llvm/ADT/FunctionExtras.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+
+/// Settings that express user/project preferences and control clangd behavior.
+///
+/// Generally, features should use Config::current() and ClangdServer is
+/// responsible for ensuring the correct config is available.
+/// (There are exceptions: e.g. background-indexing obtains configs directly).
+struct Config {
+  /// Returns the Config of the current Context, or an empty configuration.
+  static const Config &current();
+  /// Context key which can be used to set the current Config.
+  static clangd::Key<Config> Key;
+
+  Config() = default;
+  Config(const Config &) = delete;
+  Config &operator=(const Config &) = delete;
+  Config(Config &&) = default;
+  Config &operator=(Config &&) = default;
+
+  /// Controls how the compile command for the current file is determined.
+  struct {
+    // Sequence of edits to apply to the compile command.
+    std::vector<llvm::unique_function<void(std::vector<std::string> &)>> Edits;
+  } CompileFlags;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif

clang-tools-extra/clangd/Config.cpp

+//===--- Config.cpp - User configuration of clangd behavior ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "support/Context.h"
+
+namespace clang {
+namespace clangd {
+
+Key<Config> Config::Key;
+
+const Config &Config::current() {
+  if (const Config *C = Context::current().get(Key))
+    return *C;
+  static Config Default;
+  return Default;
+}
+
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/ConfigCompile.cpp

+//===--- ConfigCompile.cpp - Translating ConfigFragments into Config ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ConfigFragments are applied to Configs in two steps:
+//
+// 1. (When the fragment is first loaded)
+//    FragmentCompiler::compile() traverses the ConfigFragment and creates
+//    function objects that know how to apply the configuration.
+// 2. (Every time a config is required)
+//    CompiledConfigFragment::apply() executes these functions to create the
+//    configuration.
+//
+// Work could be split between these steps in different ways. We try to
+// do as much work as possible in the first step. For example, regexes are
+// compiled in stage 1 and captured by the apply function. This is because:
+//
+//  - it's more efficient, as the work done in stage 1 must only be done once
+//  - problems can be reported in stage 1, in stage 2 we must silently recover
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "ConfigProvider.h"
+#include "support/Logger.h"
+#include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+
+namespace clang {
+namespace clangd {
+namespace config {
+
+// Logic to compile and apply config fragments.
+namespace {
+
+// Wrapper around condition compile() functions to reduce arg-passing.
+struct FragmentCompiler {
+  std::vector<llvm::unique_function<bool(const Params &)>> &Conditions;
+  std::vector<llvm::unique_function<void(Config &)>> &Apply;
+  DiagnosticCallback Diagnostic;
+  llvm::SourceMgr *SourceMgr;
+
+  std::string RegexError = "";
+  llvm::Optional<llvm::Regex> compileRegex(const Located<std::string> &Text) {
+    std::string Anchored = "^(" + *Text + ")$";
+    llvm::Regex Result(Anchored);
+    if (!Result.isValid(RegexError)) {
+      diag(Error, "Invalid regex " + Anchored + ": " + RegexError, Text.Range);
+      return llvm::None;
+    }
+    return Result;
+  }
+
+  void compile(ConfigFragment &&F) {
+    if (F.Condition)
+      compile(std::move(*F.Condition));
+    compile(std::move(F.CompileFlags));
+  }
+
+  void compile(ConfigFragment::ConditionFragment &&F) {
+    if (F.UnrecognizedCondition) {
+      Conditions.push_back([&](const Params &) { return false; });
+      return;
+    }
+    std::string RegexError;
+
+    auto PathMatch = std::make_unique<std::vector<llvm::Regex>>();
+    for (auto &Entry : F.PathMatch) {
+      if (auto RE = compileRegex(Entry))
+        PathMatch->push_back(std::move(*RE));
+    }
+    if (!PathMatch->empty()) {
+      Conditions.push_back([PathMatch(std::move(PathMatch))](const Params &P) {
+        if (P.Path.empty())
+          return false;
+        return llvm::all_of(*PathMatch, [&](const llvm::Regex &RE) {
+          return RE.match(P.Path);
+        });
+      });
+    }
+  }
+
+  void compile(ConfigFragment::CompileFlagsFragment &&F) {
+    if (!F.Add.empty()) {
+      std::vector<std::string> Add;
+      for (auto &A : F.Add)
+        Add.push_back(std::move(*A));
+      Apply.push_back([Add(std::move(Add))](Config &C) {
+        C.CompileFlags.Edits.push_back([Add](std::vector<std::string> &Args) {
+          Args.insert(Args.end(), Add.begin(), Add.end());
+        });
+      });
+    }
+  }
+
+  constexpr static auto Error = llvm::SourceMgr::DK_Error;
+  constexpr static auto Warning = llvm::SourceMgr::DK_Warning;
+  void diag(llvm::SourceMgr::DiagKind Kind, llvm::StringRef Message,
+            llvm::SMRange Range) {
+    if (Range.isValid() && SourceMgr != nullptr)
+      Diagnostic(SourceMgr->GetMessage(Range.Start, Kind, Message, Range));
+    else
+      Diagnostic(llvm::SMDiagnostic("", Kind, Message));
+  }
+};
+
+} // namespace
+
+CompiledConfigFragment::CompiledConfigFragment(ConfigFragment F,
+                                               DiagnosticCallback D)
+    : Directory(F.Source.Directory) {
+  llvm::StringRef SourceFile = "<unknown>";
+  std::pair<unsigned, unsigned> LineCol = {0, 0};
+  if (auto *SM = F.Source.Manager.get()) {
+    unsigned BufID = SM->getMainFileID();
+    LineCol = SM->getLineAndColumn(F.Source.Location, BufID);
+    SourceFile = SM->getBufferInfo(BufID).Buffer->getBufferIdentifier();
+  }
+  vlog("Compiling ConfigFragment {0}:{1} for {2} -> {3}", SourceFile,
+       LineCol.first,
+       F.Source.Directory.empty() ? "<global>" : F.Source.Directory, this);
+  trace::Span Tracer("ConfigCompile");
+  SPAN_ATTACH(Tracer, "ConfigFile", SourceFile);
+
+  FragmentCompiler{Conditions, Apply, D, F.Source.Manager.get()}.compile(
+      std::move(F));
+}
+
+// Returns adjusted Params, with Path made relative if needed.
+// Storage is uninitialized and may be used if a copy is needed.
+static const Params &adjustParams(const Params &P, llvm::StringRef Directory,
+                                  Params &Storage) {
+  if (P.Path.empty() || Directory.empty())
+    return P;
+  if (!llvm::StringRef(P.Path).startswith(Directory)) {
+    assert(false && "Trying to apply a config to a file it doesn't govern?");
+    return P;
+  }
+  Storage = P;
+  Storage.Path.erase(0, Directory.size());
+  return Storage;
+}
+
+bool CompiledConfigFragment::apply(const Params &P, Config &C) const {
+  if (!Conditions.empty()) {
+    Params Storage;
+    const Params &Adjusted = adjustParams(P, Directory, Storage);
+    for (auto &C : Conditions)
+      if (!C(Adjusted)) {
+        dlog("ConfigFragment {0}: condition not met", this);
+        return false;
+      }
+  }
+  dlog("ConfigFragment {0}: applying {1} rules", this, Apply.size());
+  for (auto &A : Apply) {
+    llvm::errs() << "applyX\n";
+    A(C);
+  }
+  return true;
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/ConfigProvider.h

+//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various clangd features have configurable behaviour (or can be disabled).
+// The configuration system allows users to control this:
+//  - in a user config file, a project config file, via LSP, or via flags
+//  - specifying different settings for different files
+// This file defines the structures used for this, that produce a Config.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_PROVIDER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_PROVIDER_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+class ThreadsafeFS;
+struct Config;
+namespace config {
+
+/// A token written in config along with its optional location in the file.
+template <typename T> struct Located {
+  Located(T Value, llvm::SMRange Range = {})
+      : Range(Range), Value(std::move(Value)) {}
+
+  llvm::SMRange Range;
+  T &operator->() { return Value; }
+  const T &operator->() const { return Value; }
+  T &operator*() { return Value; }
+  const T &operator*() const { return Value; }
+
+private:
+  T Value;
+};
+
+/// Describes the context used to evaluate configuration fragments.
+struct Params {
+  /// Absolute path to file we're targeting. Unix slashes.
+  /// Empty if we're not configuring a particular file.
+  std::string Path;
+};
+
+/// Used to report problems in parsing or interpreting a config.
+/// Errors reflect structurally invalid config that should be user-visible.
+/// Warnings reflect e.g. unknown properties that are recoverable.
+using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>;
+
+/// A chunk of configuration obtained from a config file, LSP, or elsewhere.
+struct ConfigFragment {
+  /// Parses fragments from a YAML file (one from each --- delimited document).
+  /// Documents that contained fatal errors are omitted from the results.
+  /// BufferName is used for the SourceMgr and diagnostics.
+  static std::vector<ConfigFragment> parseYAML(llvm::StringRef YAML,
+                                               llvm::StringRef BufferName,
+                                               DiagnosticCallback);
+
+  struct SourceInfo {
+    /// Absolute path of directory this fragment is associated with, if any.
+    /// Unix slashes, including a trailing slash.
+    std::string Directory;
+    /// Retains a buffer of the original source this fragment was parsed from.
+    /// Locations within Located<T> objects point into this SourceMgr.
+    /// Shared because multiple fragments are often parsed from one (YAML) file.
+    /// May be null, then all locations are ignored.
+    std::shared_ptr<llvm::SourceMgr> Manager;
+    /// The start of the original source for this fragment.
+    /// Only valid if SourceManager is set.
+    llvm::SMLoc Location;
+  };
+  SourceInfo Source;
+
+  struct ConditionFragment {
+    std::vector<Located<std::string>> PathMatch;
+    /// An unrecognized key was found while parsing the condition.
+    /// The condition will evaluate to false.
+    bool UnrecognizedCondition;
+  };
+  llvm::Optional<ConditionFragment> Condition;
+
+  struct CompileFlagsFragment {
+    std::vector<Located<std::string>> Add;
+  } CompileFlags;
+};
+
+/// A chunk of configuration that has been fully analyzed and is ready to apply.
+class CompiledConfigFragment {
+public:
+  /// Analyzes and consumes a fragment, possibly yielding more diagnostics.
+  /// This always produces a usable compiled fragment (errors are recovered).
+  explicit CompiledConfigFragment(ConfigFragment, DiagnosticCallback);
+
+  /// Updates the configuration to reflect settings from the fragment.
+  /// Returns true if the condition was met and the fragment was applied.
+  bool apply(const Params &, Config &) const;
+
+private:
+  std::string Directory;
+  // These are not really mutated, but llvm::unique_function doens't provide
+  // a const call operator.
+  mutable std::vector<llvm::unique_function<bool(const Params &)>> Conditions;
+  mutable std::vector<llvm::unique_function<void(Config &)>> Apply;
+};
+
+/// A source of configuration fragments.
+class ConfigProvider {
+public:
+  virtual ~ConfigProvider() = default;
+
+  /// A provider that includes fragments from all the supplied providers.
+  static std::unique_ptr<ConfigProvider>
+      combine(std::vector<std::unique_ptr<ConfigProvider>>);
+
+  /// Provide fragments that may be relevant to the file.
+  /// The configuration provider is not responsible for testing conditions.
+  /// Providers are expected to cache compiled fragments, and only
+  /// reparse/recompile when the source data has changed.
+  /// When parsing/compiling, the DiagnosticCallback is used to report errors.
+  /// Usual thread-safety guarantees apply: this function must be threadsafe.
+  virtual std::vector<std::shared_ptr<CompiledConfigFragment>>
+  getFragments(const Params &, DiagnosticCallback) const = 0;
+
+  /// Build a config based on this provider.
+  Config getConfig(const Params &, DiagnosticCallback) const;
+};
+
+/// A provider that reads config from a YAML file in ~/.config/clangd.
+std::unique_ptr<ConfigProvider> createFileConfigProvider(const ThreadsafeFS &);
+
+} // namespace config
+} // namespace clangd
+} // namespace clang
+
+#endif

clang-tools-extra/clangd/ConfigProvider.cpp

+//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConfigProvider.h"
+#include "Config.h"
+#include "support/ThreadsafeFS.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include <mutex>
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+
+class FileConfigProvider : public ConfigProvider {
+  const ThreadsafeFS &TFS;
+  std::string UserConfigFile;
+
+  mutable std::mutex Mu;
+  struct CachedFragment {
+    unsigned Size = 0;
+    llvm::sys::TimePoint<> Mtime = {};
+    std::vector<std::shared_ptr<CompiledConfigFragment>> Fragments;
+
+    bool invalidate(const llvm::vfs::Status &Status) {
+      if (Size == Status.getSize() && Mtime == Status.getLastModificationTime())
+        return false;
+      Size = Status.getSize();
+      Mtime = Status.getLastModificationTime();
+      Fragments.clear();
+      return true;
+    }
+
+    void reset() {
+      Size = 0;
+      Mtime = {};
+      Fragments.clear();
+    }
+  };
+  mutable CachedFragment UserConfigCache;
+
+  std::vector<std::shared_ptr<CompiledConfigFragment>>
+  getFragments(llvm::StringRef Path, CachedFragment &Cache,
+               DiagnosticCallback DC) const {
+    auto FS = TFS.view(/*CWD=*/llvm::None);
+    auto Stat = FS->status(Path);
+    if (!Stat || !Stat->isRegularFile()) {
+      // Config file doesn't exist.
+      Cache.reset();
+    } else if (Cache.invalidate(*Stat)) {
+      // Config file has changed.
+      if (auto Buf = FS->getBufferForFile(Path)) {
+        for (auto &Fragment :
+             ConfigFragment::parseYAML(Buf->get()->getBuffer(), Path, DC)) {
+          Cache.Fragments.push_back(std::make_unique<CompiledConfigFragment>(
+              std::move(Fragment), DC));
+        }
+      }
+    }
+    return Cache.Fragments;
+  }
+
+public:
+  FileConfigProvider(const ThreadsafeFS &TFS) : TFS(TFS) {
+    llvm::SmallString<64> UserConfig;
+    // FIXME: use XDG_CONFIG_HOME etc.
+    if (llvm::sys::path::home_directory(UserConfig)) {
+      llvm::sys::path::append(UserConfig, ".config", "clangd");
+      UserConfigFile = UserConfig.str().str();
+    }
+  }
+
+  std::vector<std::shared_ptr<CompiledConfigFragment>>
+  getFragments(const Params &, DiagnosticCallback DC) const override {
+    std::lock_guard<std::mutex> Lock(Mu);
+    return getFragments(UserConfigFile, UserConfigCache, DC);
+  }
+};
+
+} // namespace
+
+std::unique_ptr<ConfigProvider>
+createFileConfigProvider(const ThreadsafeFS &TFS) {
+  return std::make_unique<FileConfigProvider>(TFS);
+};
+
+using ProviderList = std::vector<std::unique_ptr<ConfigProvider>>;
+
+std::unique_ptr<ConfigProvider>
+ConfigProvider::combine(ProviderList Providers) {
+  struct CombinedConfigProvider : ConfigProvider {
+    ProviderList Providers;
+
+    std::vector<std::shared_ptr<CompiledConfigFragment>>
+    getFragments(const Params &P, DiagnosticCallback DC) const override {
+      std::vector<std::shared_ptr<CompiledConfigFragment>> Result;
+      for (const auto &Provider : Providers) {
+        for (auto &Fragment : Provider->getFragments(P, DC))
+          Result.push_back(std::move(Fragment));
+      }
+      return Result;
+    }
+  };
+  auto Result = std::make_unique<CombinedConfigProvider>();
+  Result->Providers = std::move(Providers);
+  return Result;
+}
+
+Config ConfigProvider::getConfig(const Params &P, DiagnosticCallback DC) const {
+  Config C;
+  for (const auto &Fragment : getFragments(P, DC))
+    Fragment->apply(P, C);
+  return C;
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/ConfigYAML.cpp

+//===--- ConfigYAML.cpp - Loading configuration fragments from YAML files -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConfigProvider.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/SMLoc.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/YAMLParser.h"
+#include <system_error>
+
+namespace clang {
+namespace clangd {
+namespace config {
+namespace {
+using llvm::yaml::BlockScalarNode;
+using llvm::yaml::MappingNode;
+using llvm::yaml::Node;
+using llvm::yaml::ScalarNode;
+using llvm::yaml::SequenceNode;
+
+struct Parser {
+  llvm::SourceMgr &SM;
+  llvm::SmallString<256> Buf = {};
+
+  struct DictHandler {
+    DictHandler(llvm::StringRef Key, std::function<bool(Node &)> Handle)
+        : Key(Key), Handle(Handle) {}
+
+    llvm::StringRef Key;
+    std::function<bool(Node &)> Handle;
+  };
+
+  bool parseDict(Node &N, llvm::StringRef Desc,
+                 llvm::ArrayRef<DictHandler> Handlers,
+                 std::function<void()> Unknown = nullptr) {
+    if (N.getType() != Node::NK_Mapping)
+      return error(Desc + " should be a dictionary", N);
+    llvm::SmallSet<std::string, 8> Seen;
+    for (auto &KV : llvm::cast<MappingNode>(N)) {
+      auto *K = KV.getKey();
+      if (!K)
+        return false;
+      auto Key = scalarValue(*K, "Dictionary key");
+      if (!Key)
+        continue;
+      if (!Seen.insert(**Key).second) {
+        warning("Duplicate key " + **Key, *K);
+        continue;
+      }
+      auto *Value = KV.getValue();
+      if (!Value)
+        return false;
+      bool Matched = false;
+      for (const auto &Handler : Handlers) {
+        if (Handler.Key == **Key) {
+          if (!Handler.Handle(*Value))
+            return false;
+          Matched = true;
+          break;
+        }
+      }
+      if (!Matched) {
+        warning("Unknown " + Desc + " key " + **Key, *K);
+        if (Unknown)
+          Unknown();
+      }
+    }
+    return true;
+  }
+
+  llvm::Optional<Located<std::string>> scalarValue(Node &N,
+                                                   llvm::StringRef Desc) {
+    if (auto *S = llvm::dyn_cast<ScalarNode>(&N))
+      return Located<std::string>(S->getValue(Buf).str(), N.getSourceRange());
+    else if (auto *BS = llvm::dyn_cast<BlockScalarNode>(&N))
+      return Located<std::string>(S->getValue(Buf).str(), N.getSourceRange());
+    warning(Desc + " should be scalar", N);
+    return llvm::None;
+  }
+
+  llvm::Optional<std::vector<Located<std::string>>> scalarValues(Node &N) {
+    std::vector<Located<std::string>> Result;
+    if (auto *S = llvm::dyn_cast<ScalarNode>(&N)) {
+      Result.emplace_back(S->getValue(Buf).str(), N.getSourceRange());
+    } else if (auto *S = llvm::dyn_cast<BlockScalarNode>(&N)) {
+      Result.emplace_back(S->getValue().str(), N.getSourceRange());
+    } else if (auto *S = llvm::dyn_cast<SequenceNode>(&N)) {
+      for (auto &Child : *S) {
+        if (auto Value = scalarValue(Child, "List item"))
+          Result.push_back(std::move(*Value));
+      }
+    } else {
+      warning("Expected scalar or list of scalars", N);
+      return llvm::None;
+    }
+    return Result;
+  }
+
+  bool error(const llvm::Twine &Msg, const Node &N) {
+    SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg,
+                    N.getSourceRange());
+    return false;
+  }
+
+  void warning(const llvm::Twine &Msg, const Node &N) {
+    SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Warning, Msg,
+                    N.getSourceRange());
+  }
+
+  bool parse(ConfigFragment::ConditionFragment &F, Node &N) {
+    std::vector<DictHandler> Keys;
+    Keys.emplace_back("PathMatch", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.PathMatch = std::move(*Values);
+      return !N.failed();
+    });
+    return parseDict(N, "Condition", Keys,
+                     [&] { F.UnrecognizedCondition = true; });
+  }
+
+  bool parse(ConfigFragment::CompileFlagsFragment &F, Node &N) {
+    std::vector<DictHandler> Keys;
+    Keys.emplace_back("Add", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.Add = std::move(*Values);
+      return !N.failed();
+    });
+    return parseDict(N, "CompileFlags", Keys);
+  }
+
+  bool parse(ConfigFragment &F, Node &N) {
+    std::vector<DictHandler> Keys;
+    Keys.emplace_back("If", [&](Node &N) {
+      F.Condition.emplace();
+      return parse(*F.Condition, N);
+    });
+    Keys.emplace_back("CompileFlags",
+                      [&](Node &N) { return parse(F.CompileFlags, N); });
+    return parseDict(N, "Config", Keys);
+  }
+};
+
+// Adapt DiagnosticCallback to DiagHandlerTy interface.
+void diagHandler(const llvm::SMDiagnostic &Diag, void *Ctx) {
+  (*reinterpret_cast<DiagnosticCallback *>(Ctx))(Diag);
+}
+
+} // namespace
+
+std::vector<ConfigFragment>
+ConfigFragment::parseYAML(llvm::StringRef YAML, llvm::StringRef BufferName,
+                          DiagnosticCallback Diags) {
+  auto SM = std::make_shared<llvm::SourceMgr>();
+  SM->setDiagHandler(&diagHandler, &Diags);
+  std::vector<ConfigFragment> Result;
+  for (auto &Doc :
+       llvm::yaml::Stream(llvm::MemoryBufferRef(YAML, BufferName), *SM)) {
+    if (Node *N = Doc.parseBlockNode()) {
+      ConfigFragment Fragment;
+      Fragment.Source.Manager = SM;
+      Fragment.Source.Location = N->getSourceRange().Start;
+      if (Parser{*SM}.parse(Fragment, *N))
+        Result.push_back(std::move(Fragment));
+    }
+  }
+  return Result;
+}
+
+} // namespace config
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/TUScheduler.h

 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
 
 #include "Compiler.h"
+#include "ConfigProvider.h"
 #include "Diagnostics.h"
 #include "GlobalCompilationDatabase.h"
 #include "index/CanonicalIncludes.h"
+#include "support/Context.h"
 #include "support/Function.h"
 #include "support/Path.h"
 #include "support/Threading.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <chrono>
struct Options {
     DebouncePolicy UpdateDebounce;
 
     /// Determines when to keep idle ASTs in memory for future use.
     ASTRetentionPolicy RetentionPolicy;
 
     /// Whether to run PreamblePeer asynchronously.
     /// No-op if AsyncThreadsCount is 0.
     bool AsyncPreambleBuilds = false;
+
+    /// Used to create a context that wraps each single operation.
+    /// Typically to inject per-file configuration.
+    /// If the path is empty, context should be "generic".
+    std::function<Context(PathRef)> ContextProvider;
   };
 
   TUScheduler(const GlobalCompilationDatabase &CDB, const Options &Opts,
               std::unique_ptr<ParsingCallbacks> ASTCallbacks = nullptr);
   ~TUScheduler();
 
   struct FileStats {
     std::size_t UsedBytes = 0;
public:
   /// resources. Pending diagnostics for closed files may not be delivered, even
   /// if requested with WantDiags::Auto or WantDiags::Yes.
   void remove(PathRef File);
 
   /// Returns a snapshot of all file buffer contents, per last update().
   llvm::StringMap<std::string> getAllFileContents() const;
 
   /// Schedule an async task with no dependencies.
-  void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
+  void run(llvm::StringRef Name, llvm::Optional<PathRef> File,
+           llvm::unique_function<void()> Action);
 
   /// Defines how a runWithAST action is implicitly cancelled by other actions.
   enum ASTActionInvalidation {
     /// The request will run unless explicitly cancelled.
     NoInvalidation,
     /// The request will be implicitly cancelled by a subsequent update().
     /// (Only if the request was not yet cancelled).
     /// Useful for requests that are generated by clients, without any explicit

clang-tools-extra/clangd/TUScheduler.cpp

private:
   /// Handles retention of ASTs.
   TUScheduler::ASTCache &IdleASTs;
   const bool RunSync;
   /// Time to wait after an update to see whether another update obsoletes it.
   const DebouncePolicy UpdateDebounce;
   /// File that ASTWorker is responsible for.
   const Path FileName;
   const GlobalCompilationDatabase &CDB;
+  const std::function<Context(PathRef)> &ContextProvider;
   /// Callback invoked when preamble or main file AST is built.
   ParsingCallbacks &Callbacks;
 
   Semaphore &Barrier;
   /// Whether the 'onMainAST' callback ran for the current FileInputs.
   bool RanASTCallback = false;
   /// Guards members used by both TUScheduler and the worker thread.
   mutable std::mutex Mutex;
ASTWorkerHandle ASTWorker::create(PathRef FileName,
   return ASTWorkerHandle(std::move(Worker));
 }
 
 ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
                      TUScheduler::ASTCache &LRUCache, Semaphore &Barrier,
                      bool RunSync, const TUScheduler::Options &Opts,
                      ParsingCallbacks &Callbacks)
     : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(Opts.UpdateDebounce),
-      FileName(FileName), CDB(CDB), Callbacks(Callbacks), Barrier(Barrier),
-      Done(false), Status(FileName, Callbacks),
+      FileName(FileName), CDB(CDB), ContextProvider(Opts.ContextProvider),
+      Callbacks(Callbacks), Barrier(Barrier), Done(false),
+      Status(FileName, Callbacks),
       PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory,
                    RunSync || !Opts.AsyncPreambleBuilds, Status, *this) {
   // Set a fallback command because compile command can be accessed before
   // `Inputs` is initialized. Other fields are only used after initialization
   // from client inputs.
   FileInputs.CompileCommand = CDB.getFallbackCommand(FileName);
 }
 
// only writer is also this thread.
         Lock.lock();
       }
       WithContext Guard(std::move(CurrentRequest->Ctx));
       trace::Span Tracer(CurrentRequest->Name);
       Status.update([&](TUStatus &Status) {
         Status.ASTActivity.K = ASTAction::RunningAction;
         Status.ASTActivity.Name = CurrentRequest->Name;
       });
+      llvm::Optional<WithContext> WithProvidedContext;
+      if (ContextProvider)
+        WithProvidedContext.emplace(ContextProvider(FileName));
       CurrentRequest->Action();
     }
 
     bool IsEmpty = false;
     {
       std::lock_guard<std::mutex> Lock(Mutex);
       CurrentRequest.reset();
       IsEmpty = Requests.empty() && PreambleRequests.empty();

 
 llvm::StringMap<std::string> TUScheduler::getAllFileContents() const {
   llvm::StringMap<std::string> Results;
   for (auto &It : Files)
     Results.try_emplace(It.getKey(), It.getValue()->Contents);
   return Results;
 }
 
-void TUScheduler::run(llvm::StringRef Name,
+void TUScheduler::run(llvm::StringRef Name, llvm::Optional<PathRef> File,
                       llvm::unique_function<void()> Action) {
   if (!PreambleTasks)
     return Action();
   PreambleTasks->runAsync(Name, [this, Ctx = Context::current().clone(),
+                                 File(File.getValueOr("").str()),
                                  Action = std::move(Action)]() mutable {
     std::lock_guard<Semaphore> BarrierLock(Barrier);
     WithContext WC(std::move(Ctx));
+    llvm::Optional<WithContext> WithProvidedContext;
+    if (Opts.ContextProvider)
+      WithProvidedContext.emplace(Opts.ContextProvider(File));
     Action();
   });
 }
 
 void TUScheduler::runWithAST(
     llvm::StringRef Name, PathRef File,
     llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
     TUScheduler::ASTActionInvalidation Invalidation) {
auto Task =
           Worker->waitForFirstPreamble();
         }
         Preamble = Worker->getPossiblyStalePreamble();
 
         std::lock_guard<Semaphore> BarrierLock(Barrier);
         WithContext Guard(std::move(Ctx));
         trace::Span Tracer(Name);
         SPAN_ATTACH(Tracer, "file", File);
+        llvm::Optional<WithContext> WithProvidedContext;
+        if (Opts.ContextProvider)
+          WithProvidedContext.emplace(
+              Opts.ContextProvider(llvm::StringRef(File)));
         Action(InputsAndPreamble{Contents, Command, Preamble.get()});
       };
 
   PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File),
                           std::move(Task));
 }
 
 llvm::StringMap<TUScheduler::FileStats> TUScheduler::fileStats() const {

clang-tools-extra/clangd/index/Background.h

 
 // A priority queue of tasks which can be run on (external) worker threads.
 class BackgroundQueue {
 public:
   /// A work item on the thread pool's queue.
   struct Task {
     explicit Task(std::function<void()> Run) : Run(std::move(Run)) {}
 
+    std::string Path; // Used to determine config. May be empty.
     std::function<void()> Run;
     llvm::ThreadPriority ThreadPri = llvm::ThreadPriority::Background;
     unsigned QueuePri = 0; // Higher-priority tasks will run first.
     std::string Tag;       // Allows priority to be boosted later.
 
     bool operator<(const Task &O) const { return QueuePri < O.QueuePri; }
   };
 
   // Describes the number of tasks processed by the queue.
   struct Stats {
     unsigned Enqueued = 0;  // Total number of tasks ever enqueued.
     unsigned Active = 0;    // Tasks being currently processed by a worker.
     unsigned Completed = 0; // Tasks that have been finished.
     unsigned LastIdle = 0;  // Number of completed tasks when last empty.
   };
 
-  BackgroundQueue(std::function<void(Stats)> OnProgress = nullptr)
-      : OnProgress(OnProgress) {}
+  BackgroundQueue(std::function<void(Stats)> OnProgress = nullptr,
+                  std::function<Context(PathRef)> ContextProvider = nullptr)
+      : ContextProvider(std::move(ContextProvider)),
+        OnProgress(std::move(OnProgress)) {}
 
   // Add tasks to the queue.
   void push(Task);
   void append(std::vector<Task>);
   // Boost priority of current and new tasks with matching Tag, if they are
   // lower priority.
   // Reducing the boost of a tag affects future tasks but not current ones.
   void boost(llvm::StringRef Tag, unsigned NewPriority);
public:
   // Only affects tasks that run after the call.
   static void preventThreadStarvationInTests();
   LLVM_NODISCARD bool
   blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds);
 
 private:
   void notifyProgress() const; // Requires lock Mu
 
+  std::function<Context(PathRef)> ContextProvider; // config for tasks
   std::mutex Mu;
   Stats Stat;
   std::condition_variable CV;
   bool ShouldStop = false;
   std::vector<Task> Queue; // max-heap
   llvm::StringMap<unsigned> Boosts;
   std::function<void(Stats)> OnProgress;
 };
 
 // Builds an in-memory index by by running the static indexer action over
 // all commands in a compilation database. Indexing happens in the background.
 // FIXME: it should also persist its state on disk for fast start.
 // FIXME: it should watch for changes to files on disk.
 class BackgroundIndex : public SwapIndex {
 public:
   /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
   /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
   /// rebuilt for each indexed file.
   BackgroundIndex(
       Context BackgroundContext, const ThreadsafeFS &,
       const GlobalCompilationDatabase &CDB,
       BackgroundIndexStorage::Factory IndexStorageFactory,
       size_t ThreadPoolSize = 0, // 0 = use all hardware threads
-      std::function<void(BackgroundQueue::Stats)> OnProgress = nullptr);
+      std::function<void(BackgroundQueue::Stats)> OnProgress = nullptr,
+      std::function<Context(PathRef)> ContextProvider = nullptr);
   ~BackgroundIndex(); // Blocks while the current task finishes.
 
   // Enqueue translation units for indexing.
   // The indexing happens in a background thread, so the symbols will be
   // available sometime later.
   void enqueue(const std::vector<std::string> &ChangedFiles) {
     Queue.push(changedFilesTask(ChangedFiles));
   }
private:
 
   FileSymbols IndexedSymbols;
   BackgroundIndexRebuilder Rebuilder;
   llvm::StringMap<ShardVersion> ShardVersions; // Key is absolute file path.
   std::mutex ShardVersionsMu;
 
   BackgroundIndexStorage::Factory IndexStorageFactory;
   // Tries to load shards for the MainFiles and their dependencies.
-  std::vector<tooling::CompileCommand>
-  loadProject(std::vector<std::string> MainFiles);
+  std::vector<std::string> loadProject(std::vector<std::string> MainFiles);
 
   BackgroundQueue::Task
   changedFilesTask(const std::vector<std::string> &ChangedFiles);
-  BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd);
+  BackgroundQueue::Task indexFileTask(std::string Path);
 
   // from lowest to highest priority
   enum QueuePriority {
     IndexFile,
     IndexBoostedFile,
     LoadShards,
   };
   BackgroundQueue Queue;
   AsyncTaskRunner ThreadPool;
   GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
 };
 
 } // namespace clangd
 } // namespace clang
 
 #endif

clang-tools-extra/clangd/index/Background.cpp

 }
 
 } // namespace
 
 BackgroundIndex::BackgroundIndex(
     Context BackgroundContext, const ThreadsafeFS &TFS,
     const GlobalCompilationDatabase &CDB,
     BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize,
-    std::function<void(BackgroundQueue::Stats)> OnProgress)
+    std::function<void(BackgroundQueue::Stats)> OnProgress,
+    std::function<Context(PathRef)> ContextProvider)
     : SwapIndex(std::make_unique<MemIndex>()), TFS(TFS), CDB(CDB),
       BackgroundContext(std::move(BackgroundContext)),
       Rebuilder(this, &IndexedSymbols, ThreadPoolSize),
       IndexStorageFactory(std::move(IndexStorageFactory)),
-      Queue(std::move(OnProgress)),
+      Queue(std::move(OnProgress), std::move(ContextProvider)),
       CommandsChanged(
           CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
             enqueue(ChangedFiles);
           })) {
   assert(Rebuilder.TUsBeforeFirstBuild > 0 &&
          "Thread pool size can't be zero.");
   assert(this->IndexStorageFactory && "Storage factory can not be null!");
   for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild; ++I) {
BackgroundQueue::Task T([this, ChangedFiles] {
     SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
 
     auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
     // Run indexing for files that need to be updated.
     std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
                  std::mt19937(std::random_device{}()));
     std::vector<BackgroundQueue::Task> Tasks;
     Tasks.reserve(NeedsReIndexing.size());
-    for (auto &Cmd : NeedsReIndexing)
-      Tasks.push_back(indexFileTask(std::move(Cmd)));
+    for (std::string &Path : NeedsReIndexing)
+      Tasks.push_back(indexFileTask(std::move(Path)));
     Queue.append(std::move(Tasks));
   });
 
   T.QueuePri = LoadShards;
   T.ThreadPri = llvm::ThreadPriority::Default;
   return T;
 }
 
 static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path) {
   Path = llvm::sys::path::filename(Path);
   return Path.drop_back(llvm::sys::path::extension(Path).size());
 }
 
-BackgroundQueue::Task
-BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) {
-  BackgroundQueue::Task T([this, Cmd] {
-    // We can't use llvm::StringRef here since we are going to
-    // move from Cmd during the call below.
-    const std::string FileName = Cmd.Filename;
-    if (auto Error = index(std::move(Cmd)))
-      elog("Indexing {0} failed: {1}", FileName, std::move(Error));
+BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string Path) {
+  BackgroundQueue::Task T([this, Path] {
+    auto Cmd = CDB.getCompileCommand(Path);
+    if (!Cmd)
+      return;
+    if (auto Error = index(std::move(*Cmd)))
+      elog("Indexing {0} failed: {1}", Path, std::move(Error));
   });
+  T.Tag = filenameWithoutExtension(Path).str();
+  T.Path = std::move(Path);
   T.QueuePri = IndexFile;
-  T.Tag = std::string(filenameWithoutExtension(Cmd.Filename));
   return T;
 }
 
 void BackgroundIndex::boostRelated(llvm::StringRef Path) {
   if (isHeaderFile(Path))
     Queue.boost(filenameWithoutExtension(Path), IndexBoostedFile);
 }
 
if (auto Error = IndexStorageFactory(Path)->storeShard(Path, *IF))
           Path == MainFile);
     }
   }
 }
 
 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
   trace::Span Tracer("BackgroundIndex");
   SPAN_ATTACH(Tracer, "file", Cmd.Filename);
+  SPAN_ATTACH(Tracer, "cmd", Cmd.CommandLine);
   auto AbsolutePath = getAbsolutePath(Cmd);
 
   auto FS = TFS.view(Cmd.Directory);
   auto Buf = FS->getBufferForFile(AbsolutePath);
   if (!Buf)
     return llvm::errorCodeToError(Buf.getError());
   auto Hash = digest(Buf->get()->getBuffer());
 
llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
 
   Rebuilder.indexedTU();
   return llvm::Error::success();
 }
 
 // Restores shards for \p MainFiles from index storage. Then checks staleness of
 // those shards and returns a list of TUs that needs to be indexed to update
 // staleness.
-std::vector<tooling::CompileCommand>
+std::vector<std::string>
 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
-  std::vector<tooling::CompileCommand> NeedsReIndexing;
-
   Rebuilder.startLoading();
   // Load shards for all of the mainfiles.
   const std::vector<LoadedShard> Result =
       loadIndexShards(MainFiles, IndexStorageFactory, CDB);
   size_t LoadedShards = 0;
   {
     // Update in-memory state.
     std::lock_guard<std::mutex> Lock(ShardVersionsMu);
for (auto &LS : Result) {
     // FIXME: Currently, we simply schedule indexing on a TU whenever any of
     // its dependencies needs re-indexing. We might do it smarter by figuring
     // out a minimal set of TUs that will cover all the stale dependencies.
     // FIXME: Try looking at other TUs if no compile commands are available
     // for this TU, i.e TU was deleted after we performed indexing.
     TUsToIndex.insert(TUForFile);
   }
 
-  for (PathRef TU : TUsToIndex) {
-    auto Cmd = CDB.getCompileCommand(TU);
-    if (!Cmd)
-      continue;
-    NeedsReIndexing.emplace_back(std::move(*Cmd));
-  }
-
-  return NeedsReIndexing;
+  return {TUsToIndex.begin(), TUsToIndex.end()};
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/index/BackgroundQueue.cpp

llvm::Optional<Task> Task;
       Task = std::move(Queue.back());
       Queue.pop_back();
       notifyProgress();
     }
 
     if (Task->ThreadPri != llvm::ThreadPriority::Default &&
         !PreventStarvation.load())
       llvm::set_thread_priority(Task->ThreadPri);
+    {
+      llvm::Optional<WithContext> ProvidedContext;
+      if (ContextProvider)
+        ProvidedContext.emplace(ContextProvider(Task->Path));
       Task->Run();
+    }
     if (Task->ThreadPri != llvm::ThreadPriority::Default)
       llvm::set_thread_priority(llvm::ThreadPriority::Default);
 
     {
       std::unique_lock<std::mutex> Lock(Mu);
       ++Stat.Completed;
       if (Stat.Active == 1 && Queue.empty()) {
         // We just finished the last item, the queue is going idle.

clang-tools-extra/clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangdLSPServer.h"
 #include "CodeComplete.h"
+#include "Config.h"
+#include "ConfigProvider.h"
 #include "Features.inc"
 #include "PathMapping.h"
 #include "Protocol.h"
 #include "Transport.h"
 #include "index/Background.h"
 #include "index/Serialization.h"
 #include "refactor/Rename.h"
 #include "support/Path.h"
 #include "support/Shutdown.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/Basic/Version.h"
 #include "clang/Format/Format.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/raw_ostream.h"
 #include <chrono>
 #include <cstdlib>
 #include <iostream>
 #include <memory>
 #include <mutex>
 #include <string>
opt<CompletionStyleFlag> CompletionStyle{
     values(clEnumValN(Detailed, "detailed",
                       "One completion item for each semantically distinct "
                       "completion, with full type information"),
            clEnumValN(Bundled, "bundled",
                       "Similar completion items (e.g. function overloads) are "
                       "combined. Type information shown where possible")),
 };
 
+opt<bool> ConfigEnabled{
+    "config",
+    cat(Misc),
+    desc("Enable experimental YAML configuration file support"),
+    init(false),
+};
+
 opt<std::string> FallbackStyle{
     "fallback-style",
     cat(Features),
     desc("clang-format style to apply by default when "
          "no .clang-format file is found"),
     init(clang::format::DefaultFallbackStyle),
 };
 
if (llvm::sys::fs::exists(CompileCommandsDir)) {
       }
     } else {
       llvm::errs()
           << "Path specified by --compile-commands-dir does not exist. The "
              "argument will be ignored.\n";
     }
   }
 
+  RealThreadsafeFS TFS;
+
   ClangdServer::Options Opts;
   switch (PCHStorage) {
   case PCHStorageFlag::Memory:
     Opts.StorePreamblesInMemory = true;
     break;
   case PCHStorageFlag::Disk:
     Opts.StorePreamblesInMemory = false;
     break;
if (EnableIndex && !IndexFile.empty()) {
     if (Sync)
       AsyncIndexLoad.wait();
   }
   Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.BuildRecoveryAST = RecoveryAST;
   Opts.PreserveRecoveryASTType = RecoveryASTType;
 
+  std::unique_ptr<config::ConfigProvider> ConfigProvider;
+  if (ConfigEnabled) {
+    ConfigProvider = config::createFileConfigProvider(TFS);
+    Opts.ConfigProvider = ConfigProvider.get();
+  }
+
   clangd::CodeCompleteOptions CCOpts;
   CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
   CCOpts.Limit = LimitResults;
   if (CompletionStyle.getNumOccurrences())
     CCOpts.BundleOverloads = CompletionStyle != Detailed;
   CCOpts.ShowOrigins = ShowOrigins;
   CCOpts.InsertIncludes = HeaderInsertion;
   if (!HeaderInsertionDecorators) {
     CCOpts.IncludeIndicator.Insert.clear();
     CCOpts.IncludeIndicator.NoInsert.clear();
   }
   CCOpts.SpeculativeIndexRequest = Opts.StaticIndex;
   CCOpts.EnableFunctionArgSnippets = EnableFunctionArgSnippets;
   CCOpts.AllScopes = AllScopesCompletion;
   CCOpts.RunParser = CodeCompletionParse;
 
-  RealThreadsafeFS TFS;
   // Initialize and run ClangdLSPServer.
   // Change stdin to binary to not lose \r\n on windows.
   llvm::sys::ChangeStdinToBinary();
 
   std::unique_ptr<Transport> TransportLayer;
   if (getenv("CLANGD_AS_XPC_SERVICE")) {
 #if CLANGD_BUILD_XPC
     log("Starting LSP over XPC service");
#endif
   if (ForceOffsetEncoding != OffsetEncoding::UnsupportedEncoding)
     OffsetEncodingFromFlag = ForceOffsetEncoding;
 
   clangd::RenameOptions RenameOpts;
   // Shall we allow to customize the file limit?
   RenameOpts.AllowCrossFile = CrossFileRename;
 
   Opts.AsyncPreambleBuilds = AsyncPreamble;
-
   ClangdLSPServer LSPServer(
       *TransportLayer, TFS, CCOpts, RenameOpts, CompileCommandsDirPath,
       /*UseDirBasedCDB=*/CompileArgsFrom == FilesystemCompileArgs,
       OffsetEncodingFromFlag, Opts);
   llvm::set_thread_name("clangd.main");
   int ExitCode = LSPServer.run()
                      ? 0
                      : static_cast<int>(ErrorResultCode::NoShutdownRequest);

clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp

 //===-- CompileCommandsTests.cpp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CompileCommands.h"
+#include "Config.h"
 #include "TestFS.h"
+#include "support/Context.h"
 
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 
 #include "gmock/gmock.h"
TEST(CommandMangler, ClangPathResolve) {
 
   // Symlink not resolved with -no-canonical-prefixes.
   Cmd = {"foo", "-no-canonical-prefixes", "foo.cc"};
   Mangler.adjust(Cmd);
   EXPECT_EQ((TempDir + "/bin/foo").str(), Cmd.front());
 }
 #endif
 
+TEST(CommandMangler, ConfigEdits) {
+  auto Mangler = CommandMangler::forTests();
+  std::vector<std::string> Cmd = {"clang++", "foo.cc"};
+  {
+    Config Cfg;
+    Cfg.CompileFlags.Edits.push_back([](std::vector<std::string> &Argv) {
+      for (auto &Arg : Argv)
+        for (char &C : Arg)
+          C = llvm::toUpper(C);
+    });
+    Cfg.CompileFlags.Edits.push_back(
+        [](std::vector<std::string> &Argv) { Argv.push_back("--hello"); });
+    WithContextValue WithConfig(Config::Key, std::move(Cfg));
+    Mangler.adjust(Cmd);
+  }
+  EXPECT_THAT(Cmd,
+              ElementsAre("CLANG++", "FOO.CC", "--hello", "-fsyntax-only"));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp

updateWithDiags(
       S, FooCpp, Contents, WantDiagnostics::Auto,
       [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
   ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
 }
 
 TEST_F(TUSchedulerTests, Run) {
   TUScheduler S(CDB, optsForTest());
   std::atomic<int> Counter(0);
-  S.run("add 1", [&] { ++Counter; });
-  S.run("add 2", [&] { Counter += 2; });
+  S.run("add 1", llvm::None, [&] { ++Counter; });
+  S.run("add 2", llvm::None, [&] { Counter += 2; });
   ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
   EXPECT_EQ(Counter.load(), 3);
 
   Notification TaskRun;
   Key<int> TestKey;
   WithContextValue CtxWithKey(TestKey, 10);
-  S.run("props context", [&] {
+  S.run("props context", llvm::None, [&] {
     EXPECT_EQ(Context::current().getExisting(TestKey), 10);
     TaskRun.notify();
   });
   TaskRun.wait();
 }
 
 TEST_F(TUSchedulerTests, TUStatus) {
   class CaptureTUStatus : public ClangdServer::Callbacks {