diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -37,6 +37,7 @@ CompileCommands.cpp Compiler.cpp Config.cpp + ConfigCompile.cpp ConfigYAML.cpp Diagnostics.cpp DraftStore.cpp diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigCompile.cpp @@ -0,0 +1,148 @@ +//===--- ConfigCompile.cpp - Translating Fragments into Config ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Fragments are applied to Configs in two steps: +// +// 1. (When the fragment is first loaded) +// FragmentCompiler::compile() traverses the Fragment and creates +// function objects that know how to apply the configuration. +// 2. (Every time a config is required) +// CompiledFragment() executes these functions to populate the Config. +// +// Work could be split between these steps in different ways. We try to +// do as much work as possible in the first step. For example, regexes are +// compiled in stage 1 and captured by the apply function. This is because: +// +// - it's more efficient, as the work done in stage 1 must only be done once +// - problems can be reported in stage 1, in stage 2 we must silently recover +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "ConfigFragment.h" +#include "support/Logger.h" +#include "support/Trace.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" + +namespace clang { +namespace clangd { +namespace config { +namespace { + +struct CompiledFragmentImpl { + std::vector> Conditions; + std::vector> Apply; + bool operator()(const Params &P, Config &C) const { + for (const auto &C : Conditions) { + if (!C(P)) { + dlog("Config fragment {0}: condition not met", this); + return false; + } + } + dlog("Config fragment {0}: applying {1} rules", this, Apply.size()); + for (const auto &A : Apply) + A(C); + return true; + } +}; + +// Wrapper around condition compile() functions to reduce arg-passing. +struct FragmentCompiler { + CompiledFragmentImpl &Out; + DiagnosticCallback Diagnostic; + llvm::SourceMgr *SourceMgr; + + llvm::Optional compileRegex(const Located &Text) { + std::string Anchored = "^(" + *Text + ")$"; + llvm::Regex Result(Anchored); + std::string RegexError; + if (!Result.isValid(RegexError)) { + diag(Error, "Invalid regex " + Anchored + ": " + RegexError, Text.Range); + return llvm::None; + } + return Result; + } + + void compile(Fragment &&F) { + compile(std::move(F.If)); + compile(std::move(F.CompileFlags)); + } + + void compile(Fragment::IfBlock &&F) { + if (F.HasUnrecognizedCondition) + Out.Conditions.push_back([&](const Params &) { return false; }); + + auto PathMatch = std::make_unique>(); + for (auto &Entry : F.PathMatch) { + if (auto RE = compileRegex(Entry)) + PathMatch->push_back(std::move(*RE)); + } + if (!PathMatch->empty()) { + Out.Conditions.push_back( + [PathMatch(std::move(PathMatch))](const Params &P) { + if (P.Path.empty()) + return false; + return llvm::any_of(*PathMatch, [&](const llvm::Regex &RE) { + return RE.match(P.Path); + }); + }); + } + } + + void compile(Fragment::CompileFlagsBlock &&F) { + if (!F.Add.empty()) { + std::vector Add; + for (auto &A : F.Add) + Add.push_back(std::move(*A)); + Out.Apply.push_back([Add(std::move(Add))](Config &C) { + C.CompileFlags.Edits.push_back([Add](std::vector &Args) { + Args.insert(Args.end(), Add.begin(), Add.end()); + }); + }); + } + } + + constexpr static llvm::SourceMgr::DiagKind Error = llvm::SourceMgr::DK_Error; + void diag(llvm::SourceMgr::DiagKind Kind, llvm::StringRef Message, + llvm::SMRange Range) { + if (Range.isValid() && SourceMgr != nullptr) + Diagnostic(SourceMgr->GetMessage(Range.Start, Kind, Message, Range)); + else + Diagnostic(llvm::SMDiagnostic("", Kind, Message)); + } +}; + +} // namespace + +CompiledFragment Fragment::compile(DiagnosticCallback D) && { + llvm::StringRef ConfigFile = ""; + std::pair LineCol = {0, 0}; + if (auto *SM = Source.Manager.get()) { + unsigned BufID = SM->getMainFileID(); + LineCol = SM->getLineAndColumn(Source.Location, BufID); + ConfigFile = SM->getBufferInfo(BufID).Buffer->getBufferIdentifier(); + } + trace::Span Tracer("ConfigCompile"); + SPAN_ATTACH(Tracer, "ConfigFile", ConfigFile); + auto Result = std::make_shared(); + vlog("Config fragment: compiling {0}:{1} -> {2}", ConfigFile, LineCol.first, + Result.get()); + + FragmentCompiler{*Result, D, Source.Manager.get()}.compile(std::move(*this)); + // Return as cheaply-copyable wrapper. + return [Result(std::move(Result))](const Params &P, Config &C) { + return (*Result)(P, C); + }; +} + +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h --- a/clang-tools-extra/clangd/ConfigFragment.h +++ b/clang-tools-extra/clangd/ConfigFragment.h @@ -32,6 +32,7 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H +#include "ConfigProvider.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Error.h" @@ -59,11 +60,6 @@ T Value; }; -/// Used to report problems in parsing or interpreting a config. -/// Errors reflect structurally invalid config that should be user-visible. -/// Warnings reflect e.g. unknown properties that are recoverable. -using DiagnosticCallback = llvm::function_ref; - /// A chunk of configuration obtained from a config file, LSP, or elsewhere. struct Fragment { /// Parses fragments from a YAML file (one from each --- delimited document). @@ -73,6 +69,17 @@ llvm::StringRef BufferName, DiagnosticCallback); + /// Analyzes and consumes this fragment, possibly yielding more diagnostics. + /// This always produces a usable result (errors are recovered). + /// + /// Typically, providers will compile a Fragment once when it's first loaded, + /// caching the result for reuse. + /// Like a compiled program, this is good for performance and also encourages + /// errors to be reported early and only once. + /// + /// The returned function is a cheap-copyable wrapper of refcounted internals. + CompiledFragment compile(DiagnosticCallback) &&; + /// These fields are not part of the user-specified configuration, but /// instead are populated by the parser to describe the configuration source. struct SourceInfo { @@ -97,14 +104,14 @@ /// - if the fragment is global (e.g. user config), the path is absolute /// - paths always use forward-slashes (UNIX-style) /// If no file is being processed, these conditions will not match. - struct ConditionBlock { + struct IfBlock { /// The file being processed must fully match a regular expression. std::vector> PathMatch; /// An unrecognized key was found while parsing the condition. /// The condition will evaluate to false. bool HasUnrecognizedCondition = false; }; - ConditionBlock Condition; + IfBlock If; struct CompileFlagsBlock { std::vector> Add; diff --git a/clang-tools-extra/clangd/ConfigProvider.h b/clang-tools-extra/clangd/ConfigProvider.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigProvider.h @@ -0,0 +1,54 @@ +//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Various clangd features have configurable behaviour (or can be disabled). +// The configuration system allows users to control this: +// - in a user config file, a project config file, via LSP, or via flags +// - specifying different settings for different files +// This file defines the structures used for this, that produce a Config. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include +#include + +namespace clang { +namespace clangd { +struct Config; +namespace config { + +/// Describes the context used to evaluate configuration fragments. +struct Params { + /// Absolute path to a source file we're applying the config to. Unix slashes. + /// Empty if not configuring a particular file. + llvm::StringRef Path; +}; + +/// Used to report problems in parsing or interpreting a config. +/// Errors reflect structurally invalid config that should be user-visible. +/// Warnings reflect e.g. unknown properties that are recoverable. +using DiagnosticCallback = llvm::function_ref; + +/// A chunk of configuration that has been fully analyzed and is ready to apply. +/// Typically this is obtained from a Fragment by calling Fragment::compile(). +/// +/// Calling it updates the configuration to reflect settings from the fragment. +/// Returns true if the condition was met and the settings were used. +using CompiledFragment = std::function; + +} // namespace config +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp --- a/clang-tools-extra/clangd/ConfigYAML.cpp +++ b/clang-tools-extra/clangd/ConfigYAML.cpp @@ -35,15 +35,15 @@ // The private parse() helpers follow the same pattern. bool parse(Fragment &F, Node &N) { DictParser Dict("Config", this); - Dict.handle("If", [&](Node &N) { return parse(F.Condition, N); }); + Dict.handle("If", [&](Node &N) { return parse(F.If, N); }); Dict.handle("CompileFlags", [&](Node &N) { return parse(F.CompileFlags, N); }); return Dict.parse(N); } private: - bool parse(Fragment::ConditionBlock &F, Node &N) { - DictParser Dict("Condition", this); + bool parse(Fragment::IfBlock &F, Node &N) { + DictParser Dict("If", this); Dict.unrecognized( [&](llvm::StringRef) { F.HasUnrecognizedCondition = true; }); Dict.handle("PathMatch", [&](Node &N) { diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -41,6 +41,7 @@ CollectMacrosTests.cpp CompileCommandsTests.cpp CompilerTests.cpp + ConfigCompileTests.cpp ConfigYAMLTests.cpp DexTests.cpp DiagnosticsTests.cpp diff --git a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp @@ -0,0 +1,97 @@ +//===-- ConfigCompileTests.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "ConfigFragment.h" +#include "ConfigTesting.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace clang { +namespace clangd { +namespace config { +namespace { +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::SizeIs; +using ::testing::StartsWith; + +class ConfigCompileTests : public ::testing::Test { +protected: + CapturedDiags Diags; + Config Conf; + Fragment Frag; + Params Parm; + + bool compileAndApply() { + Conf = Config(); + Diags.Diagnostics.clear(); + auto Compiled = std::move(Frag).compile(Diags.callback()); + return Compiled(Parm, Conf); + } +}; + +TEST_F(ConfigCompileTests, Condition) { + // No condition. + Frag = {}; + Frag.CompileFlags.Add.emplace_back("X"); + EXPECT_TRUE(compileAndApply()) << "Empty config"; + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1)); + + // Regex with no file. + Frag = {}; + Frag.If.PathMatch.emplace_back("fo*"); + EXPECT_FALSE(compileAndApply()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(0)); + + // Following tests have a file path set. + Parm.Path = "bar"; + + // Non-matching regex. + Frag = {}; + Frag.If.PathMatch.emplace_back("fo*"); + EXPECT_FALSE(compileAndApply()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + + // Matching regex. + Frag = {}; + Frag.If.PathMatch.emplace_back("fo*"); + Frag.If.PathMatch.emplace_back("ba*r"); + EXPECT_TRUE(compileAndApply()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + + // Invalid regex. + Frag = {}; + Frag.If.PathMatch.emplace_back("**]@theu"); + EXPECT_TRUE(compileAndApply()); + EXPECT_THAT(Diags.Diagnostics, SizeIs(1)); + EXPECT_THAT(Diags.Diagnostics.front().Message, StartsWith("Invalid regex")); + + // Valid regex and unknown key. + Frag = {}; + Frag.If.HasUnrecognizedCondition = true; + Frag.If.PathMatch.emplace_back("ba*r"); + EXPECT_FALSE(compileAndApply()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); +} + +TEST_F(ConfigCompileTests, CompileCommands) { + Frag.CompileFlags.Add.emplace_back("-foo"); + std::vector Argv = {"clang", "a.cc"}; + EXPECT_TRUE(compileAndApply()); + EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1)); + Conf.CompileFlags.Edits.front()(Argv); + EXPECT_THAT(Argv, ElementsAre("clang", "a.cc", "-foo")); +} + +} // namespace +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/ConfigTesting.h b/clang-tools-extra/clangd/unittests/ConfigTesting.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/ConfigTesting.h @@ -0,0 +1,77 @@ +//===-- ConfigTesting.h - Helpers for configuration tests -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H + +#include "Protocol.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/SourceMgr.h" +#include "gmock/gmock.h" +#include + +namespace clang { +namespace clangd { +namespace config { + +// Provides a DiagnosticsCallback that records diganostics. +// Unlike just pushing them into a vector, underlying storage need not survive. +struct CapturedDiags { + std::function callback() { + return [this](const llvm::SMDiagnostic &D) { + Diagnostics.emplace_back(); + Diag &Out = Diagnostics.back(); + Out.Message = D.getMessage().str(); + Out.Kind = D.getKind(); + Out.Pos.line = D.getLineNo() - 1; + Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr? + if (!D.getRanges().empty()) { + const auto &R = D.getRanges().front(); + Out.Range.emplace(); + Out.Range->start.line = Out.Range->end.line = Out.Pos.line; + Out.Range->start.character = R.first; + Out.Range->end.character = R.second; + } + }; + } + struct Diag { + std::string Message; + llvm::SourceMgr::DiagKind Kind; + Position Pos; + llvm::Optional Range; + + friend void PrintTo(const Diag &D, std::ostream *OS) { + *OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ") + << D.Message << "@" << llvm::to_string(D.Pos); + } + }; + std::vector Diagnostics; +}; + +MATCHER_P(DiagMessage, M, "") { return arg.Message == M; } +MATCHER_P(DiagKind, K, "") { return arg.Kind == K; } +MATCHER_P(DiagPos, P, "") { return arg.Pos == P; } +MATCHER_P(DiagRange, R, "") { return arg.Range == R; } + +inline Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) { + auto LineCol = SM.getLineAndColumn(L); + Position P; + P.line = LineCol.first - 1; + P.character = LineCol.second - 1; + return P; +} + +inline Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) { + return Range{toPosition(R.Start, SM), toPosition(R.End, SM)}; +} + +} // namespace config +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp --- a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp +++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp @@ -8,14 +8,13 @@ #include "Annotations.h" #include "ConfigFragment.h" -#include "Matchers.h" +#include "ConfigTesting.h" #include "Protocol.h" #include "llvm/Support/SMLoc.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/SourceMgr.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "gtest/internal/gtest-internal.h" namespace clang { namespace clangd { @@ -36,55 +35,6 @@ return false; } -Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) { - auto LineCol = SM.getLineAndColumn(L); - Position P; - P.line = LineCol.first - 1; - P.character = LineCol.second - 1; - return P; -} - -Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) { - return Range{toPosition(R.Start, SM), toPosition(R.End, SM)}; -} - -struct CapturedDiags { - std::function callback() { - return [this](const llvm::SMDiagnostic &D) { - Diagnostics.emplace_back(); - Diag &Out = Diagnostics.back(); - Out.Message = D.getMessage().str(); - Out.Kind = D.getKind(); - Out.Pos.line = D.getLineNo() - 1; - Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr? - if (!D.getRanges().empty()) { - const auto &R = D.getRanges().front(); - Out.Rng.emplace(); - Out.Rng->start.line = Out.Rng->end.line = Out.Pos.line; - Out.Rng->start.character = R.first; - Out.Rng->end.character = R.second; - } - }; - } - struct Diag { - std::string Message; - llvm::SourceMgr::DiagKind Kind; - Position Pos; - llvm::Optional Rng; - - friend void PrintTo(const Diag &D, std::ostream *OS) { - *OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ") - << D.Message << "@" << llvm::to_string(D.Pos); - } - }; - std::vector Diagnostics; -}; - -MATCHER_P(DiagMessage, M, "") { return arg.Message == M; } -MATCHER_P(DiagKind, K, "") { return arg.Kind == K; } -MATCHER_P(DiagPos, P, "") { return arg.Pos == P; } -MATCHER_P(DiagRange, R, "") { return arg.Rng == R; } - TEST(ParseYAML, SyntacticForms) { CapturedDiags Diags; const char *YAML = R"yaml( @@ -101,8 +51,8 @@ auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback()); EXPECT_THAT(Diags.Diagnostics, IsEmpty()); ASSERT_EQ(Results.size(), 2u); - EXPECT_FALSE(Results.front().Condition.HasUnrecognizedCondition); - EXPECT_THAT(Results.front().Condition.PathMatch, ElementsAre(Val("abc"))); + EXPECT_FALSE(Results.front().If.HasUnrecognizedCondition); + EXPECT_THAT(Results.front().If.PathMatch, ElementsAre(Val("abc"))); EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("foo"), Val("bar"))); @@ -120,7 +70,7 @@ EXPECT_THAT(Diags.Diagnostics, IsEmpty()); ASSERT_EQ(Results.size(), 1u); ASSERT_NE(Results.front().Source.Manager, nullptr); - EXPECT_EQ(toRange(Results.front().Condition.PathMatch.front().Range, + EXPECT_EQ(toRange(Results.front().If.PathMatch.front().Range, *Results.front().Source.Manager), YAML.range()); } @@ -140,7 +90,7 @@ ASSERT_THAT( Diags.Diagnostics, - ElementsAre(AllOf(DiagMessage("Unknown Condition key UnknownCondition"), + ElementsAre(AllOf(DiagMessage("Unknown If key UnknownCondition"), DiagKind(llvm::SourceMgr::DK_Warning), DiagPos(YAML.range().start), DiagRange(YAML.range())), AllOf(DiagMessage("Unexpected token. Expected Key, Flow " @@ -150,7 +100,7 @@ ASSERT_EQ(Results.size(), 2u); EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("first"))); - EXPECT_TRUE(Results.front().Condition.HasUnrecognizedCondition); + EXPECT_TRUE(Results.front().If.HasUnrecognizedCondition); EXPECT_THAT(Results.back().CompileFlags.Add, IsEmpty()); }