WIP: Add a way to define matcher builders in the dynamic registry

[steveire]

This is WIP because there is still some error handling and code completion handling to do. Pushed the patch though to complete the series : https://github.com/steveire/llvm-project/commits/mapAnyOf

[steveire]

The split here is artificial. This can't be tested without the changes adding mapAnyOf support.

clang/include/clang/ASTMatchers/Dynamic/Parser.h

private:
   class CodeTokenizer;
   struct ScopedContextEntry;
   struct TokenInfo;
 
   Parser(CodeTokenizer *Tokenizer, Sema *S,
          const NamedValueMap *NamedValues,
          Diagnostics *Error);
 
-  bool parseBindID(std::string &BindID);
+  bool parseBindID(TokenInfo BindToken, std::string &BindID);
   bool parseExpressionImpl(VariantValue *Value);
+  bool parseMatcherBuilder(MatcherCtor Ctor, const TokenInfo &NameToken,
+                           const TokenInfo &OpenToken, VariantValue *Value);
   bool parseMatcherExpressionImpl(const TokenInfo &NameToken,
+                                  const TokenInfo &OpenToken,
+                                  llvm::Optional<MatcherCtor> Ctor,
                                   VariantValue *Value);
   bool parseIdentifierPrefixImpl(VariantValue *Value);
 
   void addCompletion(const TokenInfo &CompToken,
                      const MatcherCompletion &Completion);
   void addExpressionCompletions();
 
   std::vector<MatcherCompletion>

clang/lib/ASTMatchers/Dynamic/Marshallers.h

 class MatcherDescriptor {
 public:
   virtual ~MatcherDescriptor() = default;
 
   virtual VariantMatcher create(SourceRange NameRange,
                                 ArrayRef<ParserValue> Args,
                                 Diagnostics *Error) const = 0;
 
+  virtual ASTNodeKind nodeMatcherType() const = 0;
+
+  virtual bool isBuilderMatcher() const = 0;
+
+  virtual std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const = 0;
+
   /// Returns whether the matcher is variadic. Variadic matchers can take any
   /// number of arguments, but they must be of the same type.
   virtual bool isVariadic() const = 0;
 
   /// Returns the number of arguments accepted by the matcher if not variadic.
   virtual unsigned getNumArgs() const = 0;
 
   /// Given that the matcher is being converted to type \p ThisKind, append the
FixedArgCountMatcherDescriptor(MarshallerType Marshaller, void (*Func)(),
         ArgKinds(ArgKinds.begin(), ArgKinds.end()) {}
 
   VariantMatcher create(SourceRange NameRange,
                         ArrayRef<ParserValue> Args,
                         Diagnostics *Error) const override {
     return Marshaller(Func, MatcherName, NameRange, Args, Error);
   }
 
+  bool isBuilderMatcher() const override { return false; }
+
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ASTNodeKind(); }
+
   bool isVariadic() const override { return false; }
   unsigned getNumArgs() const override { return ArgKinds.size(); }
 
   void getArgKinds(ASTNodeKind ThisKind, unsigned ArgNo,
                    std::vector<ArgKind> &Kinds) const override {
     Kinds.push_back(ArgKinds[ArgNo]);
   }
 
public:
 
   template <typename ResultT, typename ArgT,
             ResultT (*F)(ArrayRef<const ArgT *>)>
   VariadicFuncMatcherDescriptor(
       ast_matchers::internal::VariadicFunction<ResultT, ArgT, F> Func,
       StringRef MatcherName)
       : Func(&variadicMatcherDescriptor<ResultT, ArgT, F>),
         MatcherName(MatcherName.str()),
-        ArgsKind(ArgTypeTraits<ArgT>::getKind()) {
+        ArgsKind(ArgTypeTraits<ArgT>::getKind()),
+        ArgsNodeKind(ASTNodeKind::getFromNodeKind<ArgT>()) {
     BuildReturnTypeVector<ResultT>::build(RetKinds);
   }
 
   VariantMatcher create(SourceRange NameRange,
                         ArrayRef<ParserValue> Args,
                         Diagnostics *Error) const override {
     return Func(MatcherName, NameRange, Args, Error);
   }
 
+  bool isBuilderMatcher() const override { return false; }
+
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ArgsNodeKind; }
+
   bool isVariadic() const override { return true; }
   unsigned getNumArgs() const override { return 0; }
 
   void getArgKinds(ASTNodeKind ThisKind, unsigned ArgNo,
                    std::vector<ArgKind> &Kinds) const override {
     Kinds.push_back(ArgsKind);
   }
 
   bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity,
                        ASTNodeKind *LeastDerivedKind) const override {
     return isRetKindConvertibleTo(RetKinds, Kind, Specificity,
                                   LeastDerivedKind);
   }
 
 private:
   const RunFunc Func;
   const std::string MatcherName;
   std::vector<ASTNodeKind> RetKinds;
   const ArgKind ArgsKind;
+  const ASTNodeKind ArgsNodeKind;
 };
 
 /// Return CK_Trivial when appropriate for VariadicDynCastAllOfMatchers.
 class DynCastAllOfMatcherDescriptor : public VariadicFuncMatcherDescriptor {
 public:
   template <typename BaseT, typename DerivedT>
   DynCastAllOfMatcherDescriptor(
       ast_matchers::internal::VariadicDynCastAllOfMatcher<BaseT, DerivedT> Func,
if (VariadicFuncMatcherDescriptor::isConvertibleTo(Kind, Specificity,
           *Specificity = 0;
       }
       return true;
     } else {
       return false;
     }
   }
 
+  ASTNodeKind nodeMatcherType() const override { return DerivedKind; }
+
 private:
   const ASTNodeKind DerivedKind;
 };
 
 /// Helper macros to check the arguments on all marshaller functions.
 #define CHECK_ARG_COUNT(count)                                                 \
   if (Args.size() != count) {                                                  \
     Error->addError(NameRange, Error->ET_RegistryWrongArgCount)                \
bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity,
                        ASTNodeKind *LeastDerivedKind) const override {
     for (const auto &O : Overloads) {
       if (O->isConvertibleTo(Kind, Specificity, LeastDerivedKind))
         return true;
     }
     return false;
   }
 
+  bool isBuilderMatcher() const override { return false; }
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ASTNodeKind(); }
+
 private:
   std::vector<std::unique_ptr<MatcherDescriptor>> Overloads;
 };
 
 template <typename ReturnType>
 class RegexMatcherDescriptor : public MatcherDescriptor {
 public:
   RegexMatcherDescriptor(ReturnType (*WithFlags)(StringRef,
if (!ArgTypeTraits<llvm::Regex::RegexFlags>::hasCorrectValue(
       }
       return VariantMatcher();
     }
     return outvalueToVariantMatcher(
         WithFlags(ArgTypeTraits<StringRef>::get(Args[0].Value),
                   ArgTypeTraits<llvm::Regex::RegexFlags>::get(Args[1].Value)));
   }
 
+  bool isBuilderMatcher() const override { return false; }
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ASTNodeKind(); }
+
 private:
   ReturnType (*const WithFlags)(StringRef, llvm::Regex::RegexFlags);
   ReturnType (*const NoFlags)(StringRef);
   const std::vector<ASTNodeKind> RetKinds;
 };
 
 /// Variadic operator marshaller function.
 class VariadicOperatorMatcherDescriptor : public MatcherDescriptor {
if (Specificity)
       *Specificity = 1;
     if (LeastDerivedKind)
       *LeastDerivedKind = Kind;
     return true;
   }
 
   bool isPolymorphic() const override { return true; }
 
+  bool isBuilderMatcher() const override { return false; }
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ASTNodeKind(); }
+
 private:
   const unsigned MinCount;
   const unsigned MaxCount;
   const VarOp Op;
   const StringRef MatcherName;
 };
 
 class MapAnyOfMatcherDescriptor : public MatcherDescriptor {
public:
   bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity,
                        ASTNodeKind *LeastDerivedKind) const override {
     if (Specificity)
       *Specificity = 1;
     if (LeastDerivedKind)
       *LeastDerivedKind = CladeNodeKind;
     return true;
   }
+
+  bool isBuilderMatcher() const override { return false; }
+  std::unique_ptr<MatcherDescriptor>
+  buildMatcherCtor(SourceRange NameRange, ArrayRef<ParserValue> Args,
+                   Diagnostics *Error) const override {
+    return {};
+  }
+
+  ASTNodeKind nodeMatcherType() const override { return ASTNodeKind(); }
 };
 
 /// Helper functions to select the appropriate marshaller functions.
 /// They detect the number of arguments, arguments types and return type.
 
 /// 0-arg overload
 template <typename ReturnType>
 std::unique_ptr<MatcherDescriptor>

clang/lib/ASTMatchers/Dynamic/Parser.cpp

 #include "clang/ASTMatchers/ASTMatchersInternal.h"
 #include "clang/ASTMatchers/Dynamic/Diagnostics.h"
 #include "clang/ASTMatchers/Dynamic/Registry.h"
 #include "clang/Basic/CharInfo.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
+
+#include "Marshallers.h"
+
 #include <algorithm>
 #include <cassert>
 #include <cerrno>
 #include <cstddef>
 #include <cstdlib>
 #include <string>
 #include <utility>
 #include <vector>
enum TokenKind {
     TK_Ident,
     TK_InvalidChar,
     TK_Error,
     TK_CodeCompletion
   };
 
   /// Some known identifiers.
   static const char* const ID_Bind;
+  static const char *const ID_With;
 
   TokenInfo() = default;
 
   StringRef Text;
   TokenKind Kind = TK_Eof;
   SourceRange Range;
   VariantValue Value;
 };
 
 const char* const Parser::TokenInfo::ID_Bind = "bind";
+const char *const Parser::TokenInfo::ID_With = "with";
 
 /// Simple tokenizer for the parser.
 class Parser::CodeTokenizer {
 public:
   explicit CodeTokenizer(StringRef &MatcherCode, Diagnostics *Error)
       : Code(MatcherCode), StartOfLine(MatcherCode), Error(Error) {
     NextToken = getNextToken();
   }
if (const VariantValue NamedValue =
                         : VariantValue()) {
 
       if (Tokenizer->nextTokenKind() != TokenInfo::TK_Period) {
         *Value = NamedValue;
         return true;
       }
 
       std::string BindID;
-      if (!parseBindID(BindID))
+      Tokenizer->consumeNextToken(); // consume the period.
+      const TokenInfo ChainCallToken = Tokenizer->consumeNextToken();
+      if (ChainCallToken.Kind == TokenInfo::TK_CodeCompletion) {
+        addCompletion(ChainCallToken, MatcherCompletion("bind(\"", "bind", 1));
+        addCompletion(ChainCallToken, MatcherCompletion("with(\"", "with", 1));
+        return false;
+      }
+      if (!parseBindID(ChainCallToken, BindID))
         return false;
 
       assert(NamedValue.isMatcher());
       llvm::Optional<DynTypedMatcher> Result =
           NamedValue.getMatcher().getSingleMatcher();
       if (Result.hasValue()) {
         llvm::Optional<DynTypedMatcher> Bound = Result->tryBind(BindID);
         if (Bound.hasValue()) {
if ((Tokenizer->nextTokenKind() == TokenInfo::TK_Comma ||
           << NameToken.Text;
       return false;
     }
     // Otherwise, fallback to the matcher parser.
   }
 
   Tokenizer->SkipNewlines();
 
-  // Parse as a matcher expression.
-  return parseMatcherExpressionImpl(NameToken, Value);
+  assert(NameToken.Kind == TokenInfo::TK_Ident);
+  const TokenInfo OpenToken = Tokenizer->consumeNextToken();
+  if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
+    Error->addError(OpenToken.Range, Error->ET_ParserNoOpenParen)
+        << OpenToken.Text;
+    return false;
   }
 
-bool Parser::parseBindID(std::string &BindID) {
-  // Parse .bind("foo")
-  assert(Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period);
-  Tokenizer->consumeNextToken(); // consume the period.
-  const TokenInfo BindToken = Tokenizer->consumeNextToken();
-  if (BindToken.Kind == TokenInfo::TK_CodeCompletion) {
-    addCompletion(BindToken, MatcherCompletion("bind(\"", "bind", 1));
-    return false;
-  }
+  llvm::Optional<MatcherCtor> Ctor = S->lookupMatcherCtor(NameToken.Text);
+
+  // Parse as a matcher expression.
+  return parseMatcherExpressionImpl(NameToken, OpenToken, Ctor, Value);
 
+bool Parser::parseBindID(TokenInfo BindToken, std::string &BindID) {
+  // Parse .bind("foo")
   const TokenInfo OpenToken = Tokenizer->consumeNextToken();
   const TokenInfo IDToken = Tokenizer->consumeNextTokenIgnoreNewlines();
   const TokenInfo CloseToken = Tokenizer->consumeNextTokenIgnoreNewlines();
 
-  // TODO: We could use different error codes for each/some to be more
-  //       explicit about the syntax error.
-  if (BindToken.Kind != TokenInfo::TK_Ident ||
-      BindToken.Text != TokenInfo::ID_Bind) {
-    Error->addError(BindToken.Range, Error->ET_ParserMalformedBindExpr);
-    return false;
-  }
   if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
     Error->addError(OpenToken.Range, Error->ET_ParserMalformedBindExpr);
     return false;
   }
   if (IDToken.Kind != TokenInfo::TK_Literal || !IDToken.Value.isString()) {
     Error->addError(IDToken.Range, Error->ET_ParserMalformedBindExpr);
     return false;
   }
   if (CloseToken.Kind != TokenInfo::TK_CloseParen) {
     Error->addError(CloseToken.Range, Error->ET_ParserMalformedBindExpr);
     return false;
   }
   BindID = IDToken.Value.getString();
   return true;
 }
 
+bool Parser::parseMatcherBuilder(MatcherCtor Ctor, const TokenInfo &NameToken,
+                                 const TokenInfo &OpenToken,
+                                 VariantValue *Value) {
+  std::vector<ParserValue> Args;
+  TokenInfo EndToken;
+
+  Tokenizer->SkipNewlines();
+
+  {
+    ScopedContextEntry SCE(this, Ctor);
+
+    while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
+      if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
+        // End of args.
+        EndToken = Tokenizer->consumeNextToken();
+        break;
+      }
+      if (!Args.empty()) {
+        // We must find a , token to continue.
+        const TokenInfo CommaToken = Tokenizer->consumeNextToken();
+        if (CommaToken.Kind != TokenInfo::TK_Comma) {
+          Error->addError(CommaToken.Range, Error->ET_ParserNoComma)
+              << CommaToken.Text;
+          return false;
+        }
+      }
+
+      Diagnostics::Context Ctx(Diagnostics::Context::MatcherArg, Error,
+                               NameToken.Text, NameToken.Range,
+                               Args.size() + 1);
+      ParserValue ArgValue;
+      Tokenizer->SkipNewlines();
+      const auto NodeMatcherToken = Tokenizer->consumeNextToken();
+      ArgValue.Text = NodeMatcherToken.Text;
+      ArgValue.Range = NodeMatcherToken.Range;
+
+      auto MappedMatcher = S->lookupMatcherCtor(ArgValue.Text);
+
+      if (!MappedMatcher) {
+        // TODO: error
+      }
+
+      if ((*MappedMatcher)->nodeMatcherType().isNone()) {
+        // TODO: error
+      }
+
+      ArgValue.Value = (*MappedMatcher)->nodeMatcherType();
+
+      Tokenizer->SkipNewlines();
+      Args.push_back(ArgValue);
+
+      SCE.nextArg();
+    }
+  }
+
+  if (EndToken.Kind == TokenInfo::TK_Eof) {
+    Error->addError(OpenToken.Range, Error->ET_ParserNoCloseParen);
+    return false;
+  }
+
+  auto BuiltCtor = Ctor->buildMatcherCtor(NameToken.Range, Args, Error);
+
+  if (!BuiltCtor) {
+    // Too early for completion?
+    return false;
+  }
+
+  std::string BindID;
+  if (Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period) {
+    Tokenizer->consumeNextToken(); // consume the period.
+    const TokenInfo ChainCallToken = Tokenizer->consumeNextToken();
+    if (ChainCallToken.Kind == TokenInfo::TK_CodeCompletion) {
+      addCompletion(ChainCallToken, MatcherCompletion("bind(\"", "bind", 1));
+      addCompletion(ChainCallToken, MatcherCompletion("with(\"", "with", 1));
+      return false;
+    }
+    if (ChainCallToken.Kind != TokenInfo::TK_Ident) {
+      // TODO: Change error diagnostic to also account for with
+      Error->addError(ChainCallToken.Range, Error->ET_ParserMalformedBindExpr);
+      return false;
+    }
+    if (ChainCallToken.Text == TokenInfo::ID_Bind) {
+      if (!parseBindID(ChainCallToken, BindID))
+        return false;
+      Diagnostics::Context Ctx(Diagnostics::Context::ConstructMatcher, Error,
+                               NameToken.Text, NameToken.Range);
+      SourceRange MatcherRange = NameToken.Range;
+      MatcherRange.End = ChainCallToken.Range.End;
+      VariantMatcher Result = S->actOnMatcherExpression(
+          BuiltCtor.get(), MatcherRange, BindID, {}, Error);
+      if (Result.isNull())
+        return false;
+
+      *Value = Result;
+      return true;
+    } else if (ChainCallToken.Text == TokenInfo::ID_With) {
+      Tokenizer->SkipNewlines();
+      const TokenInfo WithOpenToken = Tokenizer->consumeNextToken();
+      return parseMatcherExpressionImpl(NameToken, WithOpenToken,
+                                        BuiltCtor.get(), Value);
+    }
+  }
+
+  Diagnostics::Context Ctx(Diagnostics::Context::ConstructMatcher, Error,
+                           NameToken.Text, NameToken.Range);
+  SourceRange MatcherRange = NameToken.Range;
+  MatcherRange.End = EndToken.Range.End;
+  VariantMatcher Result = S->actOnMatcherExpression(
+      BuiltCtor.get(), MatcherRange, BindID, {}, Error);
+  if (Result.isNull())
+    return false;
+
+  *Value = Result;
+  return true;
+}
+
 /// Parse and validate a matcher expression.
 /// \return \c true on success, in which case \c Value has the matcher parsed.
 ///   If the input is malformed, or some argument has an error, it
 ///   returns \c false.
 bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken,
+                                        const TokenInfo &OpenToken,
+                                        llvm::Optional<MatcherCtor> Ctor,
                                         VariantValue *Value) {
-  assert(NameToken.Kind == TokenInfo::TK_Ident);
-  const TokenInfo OpenToken = Tokenizer->consumeNextToken();
-  if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
-    Error->addError(OpenToken.Range, Error->ET_ParserNoOpenParen)
-        << OpenToken.Text;
-    return false;
-  }
-
-  llvm::Optional<MatcherCtor> Ctor = S->lookupMatcherCtor(NameToken.Text);
-
   if (!Ctor) {
     Error->addError(NameToken.Range, Error->ET_RegistryMatcherNotFound)
         << NameToken.Text;
     // Do not return here. We need to continue to give completion suggestions.
   }
 
+  if (Ctor && *Ctor && (*Ctor)->isBuilderMatcher()) {
+    return parseMatcherBuilder(*Ctor, NameToken, OpenToken, Value);
+  }
+
   std::vector<ParserValue> Args;
   TokenInfo EndToken;
 
   Tokenizer->SkipNewlines();
 
   {
     ScopedContextEntry SCE(this, Ctor ? *Ctor : nullptr);
 
bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken,
 
   if (EndToken.Kind == TokenInfo::TK_Eof) {
     Error->addError(OpenToken.Range, Error->ET_ParserNoCloseParen);
     return false;
   }
 
   std::string BindID;
   if (Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period) {
-    if (!parseBindID(BindID))
+    Tokenizer->consumeNextToken();
+    const TokenInfo ChainCallToken = Tokenizer->consumeNextToken();
+    if (ChainCallToken.Kind == TokenInfo::TK_CodeCompletion) {
+      addCompletion(ChainCallToken, MatcherCompletion("bind(\"", "bind", 1));
+      addCompletion(ChainCallToken, MatcherCompletion("with(\"", "with", 1));
+      return false;
+    }
+
+    if (!parseBindID(ChainCallToken, BindID))
       return false;
   }
 
   if (!Ctor)
     return false;
 
   // Merge the start and end infos.
   Diagnostics::Context Ctx(Diagnostics::Context::ConstructMatcher, Error,