Add a batch query and replace tool based on AST matchers.

[jbangert]

Summary

This requires https://reviews.llvm.org/D29613. It adds a new tool, clang-query-replace that replaces and AST node with the result of evaluating a simple template language.

[klimek]

clang-query/QueryReplace.cpp

51

Shouldn't that also just return the error?

clang-query/QueryReplace.h

36–37

This doesn't seem to come up in the test? (and I don't understand what it's trying to tell me :)

[ioeric]

clang-query/QueryReplace.cpp

2

Please add license header.

38

I'd prefer using llvm::Expected<T> with an llvm::StringError.

59

return error here?

Consider using llvm::Error with llvm::StringError.

73

Missing // namespace ...

clang-query/replace-tool/ClangQueryReplace.cpp

77

If this parses yaml string, shouldn't it be called parseFromYAML instead?

[sbenza]

clang-query/QueryReplace.h

36–37

In the other change the identifier was specified with ${identifier}, not %"identifier".
They should be consistent, no?

[jbangert]

Response to initial code review.

[jbangert]

Thank you for the feedback, addressed and reformatted using llvm style.

clang-query/QueryReplace.h

36–37

I edited the comment for consistency. This indeed has an older syntax.

[jbangert]

Fix command line tool.

clang-query/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   lineeditor
   support
   )
 
 add_clang_library(clangQuery
   Query.cpp
   QueryParser.cpp
+  QueryReplace.cpp
+  QueryReplace.h
+
 
   LINK_LIBS
   clangAST
   clangASTMatchers
   clangBasic
   clangDynamicASTMatchers
   clangFrontend
   )
 
 add_subdirectory(tool)
+add_subdirectory(replace-tool)

clang-query/QueryReplace.h

+//===--- QueryReplace.h - clang-query-replace ----------------------*- C++-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERYREPLACE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERYREPLACE_H
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/Dynamic/Parser.h"
+#include "clang/Tooling/RefactoringCallbacks.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace query {
+/// \brief Describes an AST query and replace operation.
+struct QueryReplaceSpec {
+  static llvm::Expected<QueryReplaceSpec> parseFromYAML(llvm::StringRef Spec);
+  /// \brief clang-query style matcher expression
+  std::string Query;
+
+  /// \brief The node bound to this identifier will be replaced by the
+  /// result of evaluating the template below.
+  std::string FromId;
+
+  /// \brief Replacement template. ${identifier} will be replaced by the text of
+  /// the node bound to the identifier, and $$ by a literal '$'.
+  std::string ToTemplate;
+};
+
+class QueryReplaceTool {
+public:
+  QueryReplaceTool(
+      std::map<std::string, tooling::Replacements> &FileToReplaces);
+  llvm::Error addOperation(QueryReplaceSpec &spec);
+  std::unique_ptr<ASTConsumer> newASTConsumer();
+
+private:
+  struct ReplaceOp {
+    ast_matchers::dynamic::DynTypedMatcher Matcher;
+    std::unique_ptr<tooling::RefactoringCallback> Callback;
+  };
+  std::vector<ReplaceOp> Operations;
+
+  tooling::ASTMatchRefactorer Refactorer;
+};
+} // namespace query
+} // namespace clang
+#endif

clang-query/QueryReplace.cpp

+//===--- QueryReplace.cpp - clang-query-replace --------------------------*- C++
+//-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "QueryReplace.h"
+#include "QueryParser.h"
+#include "QuerySession.h"
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/Dynamic/Parser.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/RefactoringCallbacks.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/YAMLTraits.h"
+
+using clang::query::QueryReplaceSpec;
+using clang::ast_matchers::MatchFinder;
+using clang::ast_type_traits::DynTypedNode;
+using clang::tooling::ReplaceNodeWithTemplate;
+using llvm::Error;
+using llvm::Expected;
+using llvm::StringError;
+using llvm::StringRef;
+
+namespace llvm {
+namespace yaml {
+template <> struct MappingTraits<clang::query::QueryReplaceSpec> {
+  static void mapping(IO &IO, clang::query::QueryReplaceSpec &Op) {
+    IO.mapRequired("Query", Op.Query);
+    IO.mapRequired("FromId", Op.FromId);
+    IO.mapRequired("ToTemplate", Op.ToTemplate);
+  }
+};
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace query {
+
+Expected<QueryReplaceSpec> QueryReplaceSpec::parseFromYAML(StringRef Spec) {
+  llvm::yaml::Input Input(Spec);
+  QueryReplaceSpec Op;
+  Input >> Op;
+  if (Input.error())
+    return llvm::errorCodeToError(Input.error());
+  return Op;
+}
+
+QueryReplaceTool::QueryReplaceTool(
+    std::map<std::string, tooling::Replacements> &FileToReplaces)
+    : Refactorer(FileToReplaces) {}
+
+Error QueryReplaceTool::addOperation(clang::query::QueryReplaceSpec &Spec) {
+  ast_matchers::dynamic::Diagnostics Diag;
+  // FIXME: Support query sessions (for interactive use).
+  auto Matcher = ast_matchers::dynamic::Parser::parseMatcherExpression(
+      Spec.Query, nullptr, nullptr, &Diag);
+  if (!Matcher) {
+    return llvm::make_error<StringError>(
+        StringRef("Cannot parse Query:") + Diag.toStringFull(),
+        std::make_error_code(std::errc::bad_message));
+  }
+  auto Operation =
+      ReplaceNodeWithTemplate::create(Spec.FromId, Spec.ToTemplate);
+  if (!Operation)
+    return Operation.takeError();
+  Operations.emplace_back(ReplaceOp{Matcher.getValue(), std::move(*Operation)});
+
+  Refactorer.addDynamicMatcher(Operations.back().Matcher,
+                               Operations.back().Callback.get());
+  return Error::success();
+}
+
+std::unique_ptr<ASTConsumer> QueryReplaceTool::newASTConsumer() {
+  return Refactorer.newASTConsumer();
+}
+} // namespace query
+} // namespace clang

clang-query/replace-tool/CMakeLists.txt

+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+add_clang_executable(clang-query-replace ClangQueryReplace.cpp)
+target_link_libraries(clang-query-replace
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangDynamicASTMatchers
+  clangFrontend
+  clangQuery
+  clangTooling
+  )
+
+install(TARGETS clang-query-replace RUNTIME DESTINATION bin)
 No newline at end of file

clang-query/replace-tool/ClangQueryReplace.cpp

+//===---- ClangQueryReplace.cpp - clang-query-replace tool ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This tool automatically modifies source files based on AST matchers.
+//
+// A given AST matcher is applied to the source, and a bound node in
+// this matcher is replaced by the result of substituting the text
+// corresponding to other bound nodes into a string template.
+//
+// For example:
+// clang-query-replace --spec <<SPEC foo.c
+//        Query: 'varDecl(hasName("version"),
+//                       hasInitializer(integerLiteral().bind("init")))
+//                       .bind("decl")',
+//        FromId: 'decl',
+//        ToTemplate: 'string version = "$$-${init}";' }
+// SPEC
+//
+// If foo.c contains this:
+//
+// int version = 5;
+// void bar() { long version = 9; }
+//
+// it will be replaced with:
+// string version = "$-5";
+// void bar() { string version = "$-9"; }
+// ===---------------------------------------------------------------------===//
+
+#include "QueryReplace.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include <fstream>
+#include <string>
+#include <system_error>
+
+using namespace clang;
+using namespace clang::query;
+using namespace llvm;
+
+using std::error_code;
+using clang::tooling::CommonOptionsParser;
+
+static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
+static cl::OptionCategory ClangQueryCategory("clang-query-replace options");
+
+static cl::list<std::string> Commands("spec",
+                                      cl::desc("YAML action specification"),
+                                      cl::value_desc("spec"),
+                                      cl::cat(ClangQueryCategory));
+
+int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  CommonOptionsParser OptionsParser(argc, argv, ClangQueryCategory);
+  if (Commands.empty()) {
+    llvm::errs() << "Must specify at least one --spec \n";
+    return 1;
+  }
+
+  tooling::RefactoringTool Refactor(OptionsParser.getCompilations(),
+                                    OptionsParser.getSourcePathList());
+
+  QueryReplaceTool Tool(Refactor.getReplacements());
+  for (auto &SpecYaml : Commands) {
+
+    llvm::Expected<QueryReplaceSpec> Spec =
+        QueryReplaceSpec::parseFromYAML(SpecYaml);
+    if (llvm::Error Error = Spec.takeError())
+      llvm::report_fatal_error(std::move(Error), false);
+    if (llvm::Error Error = Tool.addOperation(*Spec))
+      llvm::report_fatal_error(std::move(Error), false);
+  }
+
+  return Refactor.runAndSave(tooling::newFrontendActionFactory(&Tool).get());
+}

test/CMakeLists.txt

set(CLANG_TOOLS_TEST_DEPS
 
   # Individual tools we test.
   clang-apply-replacements
   clang-change-namespace
   clangd
   clang-include-fixer
   clang-move
   clang-query
+  clang-query-replace
   clang-rename
   clang-reorder-fields
   clang-tidy
   find-all-symbols
   modularize
   pp-trace
 
   # Unit tests

test/clang-query/replacement.c

+// RUN: cp %s %t.c
+// RUN: clang-query-replace --spec  '{Query: "varDecl(hasName(\"version\"), hasInitializer(integerLiteral().bind(\"init\"))).bind(\"decl\")", FromId: "decl", ToTemplate: "string version = \"$$-${init}\""}' %t.c --
+// RUN: FileCheck --match-full-lines %s < %t.c
+int version = 4;
+// CHECK: string version = "$-4";