Add a batch query and replace tool based on AST matchers.

[jbangert]

Summary

This requires https://reviews.llvm.org/D29613. It adds a new tool, clang-query-replace that replaces and AST node with the result of evaluating a simple template language.

[klimek]

clang-query/QueryReplace.cpp

50

Shouldn't that also just return the error?

clang-query/QueryReplace.h

35–36

This doesn't seem to come up in the test? (and I don't understand what it's trying to tell me :)

[ioeric]

clang-query/QueryReplace.cpp

1

Please add license header.

37

I'd prefer using llvm::Expected<T> with an llvm::StringError.

58

return error here?

Consider using llvm::Error with llvm::StringError.

72

Missing // namespace ...

clang-query/replace-tool/ClangQueryReplace.cpp

76

If this parses yaml string, shouldn't it be called parseFromYAML instead?

[sbenza]

clang-query/QueryReplace.h

35–36

In the other change the identifier was specified with ${identifier}, not %"identifier".
They should be consistent, no?

[jbangert]

Response to initial code review.

[jbangert]

Thank you for the feedback, addressed and reformatted using llvm style.

clang-query/QueryReplace.h

35–36

I edited the comment for consistency. This indeed has an older syntax.

[jbangert]

Fix command line tool.

clang-query/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   lineeditor
   support
   )
 
 add_clang_library(clangQuery
   Query.cpp
   QueryParser.cpp
+  QueryReplace.cpp
+  QueryReplace.h
+
 
   LINK_LIBS
   clangAST
   clangASTMatchers
   clangBasic
   clangDynamicASTMatchers
   clangFrontend
   )
 
 add_subdirectory(tool)
+add_subdirectory(replace-tool)

clang-query/QueryReplace.h

+//===--- ReplaceSpec.h - clang-query-replace ------------------------*- C++
+//-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERYREPLACE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERYREPLACE_H
+
+#include <memory>
+#include <string>
+#include <vector>
+#include "clang/AST/ASTConsumer.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/Dynamic/Parser.h"
+#include "clang/Tooling/RefactoringCallbacks.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ErrorOr.h"
+
+namespace clang {
+namespace query {
+/// \brief Describes an AST query and replace operation.
+struct QueryReplaceSpec {
+  static llvm::ErrorOr<QueryReplaceSpec> parseFromJSON(llvm::StringRef Spec);
+  /// \brief clang-query style matcher expression
+  std::string Query;
+
+  /// \brief Identifier of the bound node to replace
+  std::string FromId;
+
+  /// \brief Replacement text. %"identifier" will be substituted by the text of
+  /// an identifier.
+  std::string ToTemplate;
+};
+
+class QueryReplaceTool {
+public:
+  QueryReplaceTool(
+      std::map<std::string, tooling::Replacements> &FileToReplaces);
+  void addOperation(QueryReplaceSpec &spec);
+  std::unique_ptr<ASTConsumer> newASTConsumer();
+
+private:
+  struct ReplaceOp {
+    ast_matchers::dynamic::DynTypedMatcher Matcher;
+    std::unique_ptr<tooling::RefactoringCallback> Callback;
+  };
+  std::vector<ReplaceOp> Operations;
+
+  tooling::ASTMatchRefactorer Refactorer;
+};
+}
+}
+#endif

clang-query/QueryReplace.cpp

+#include "QueryReplace.h"
+#include "QueryParser.h"
+#include "QuerySession.h"
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/Dynamic/Parser.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/RefactoringCallbacks.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/YAMLTraits.h"
+
+using clang::query::QueryReplaceSpec;
+using clang::ast_matchers::MatchFinder;
+using clang::ast_type_traits::DynTypedNode;
+using clang::tooling::ReplaceNodeWithTemplate;
+namespace llvm {
+namespace yaml {
+template <>
+struct MappingTraits<clang::query::QueryReplaceSpec> {
+  static void mapping(IO &IO, clang::query::QueryReplaceSpec &Op) {
+    IO.mapRequired("Query", Op.Query);
+    IO.mapRequired("FromId", Op.FromId);
+    IO.mapRequired("ToTemplate", Op.ToTemplate);
+  }
+};
+}
+}
+
+namespace clang {
+namespace query {
+
+llvm::ErrorOr<QueryReplaceSpec> QueryReplaceSpec::parseFromJSON(
+    StringRef Spec) {
+  llvm::yaml::Input Input(Spec);
+  QueryReplaceSpec Op;
+  Input >> Op;
+  if (Input.error()) return Input.error();
+  return Op;
+}
+
+QueryReplaceTool::QueryReplaceTool(
+    std::map<std::string, tooling::Replacements> &FileToReplaces)
+    : Refactorer(FileToReplaces) {}
+
+void QueryReplaceTool::addOperation(clang::query::QueryReplaceSpec &Spec) {
+  ast_matchers::dynamic::Diagnostics Diag;
+  // FIXME: Support query sessions (for interactive use).
+  auto Matcher = ast_matchers::dynamic::Parser::parseMatcherExpression(
+      Spec.Query, nullptr, nullptr, &Diag);
+  if (!Matcher) {
+    llvm::errs() << "Cannot parse Query:";
+    Diag.printToStreamFull(llvm::errs());
+    llvm::errs() << "\n";
+  }
+
+  Operations.emplace_back(
+      ReplaceOp{Matcher.getValue(), llvm::make_unique<ReplaceNodeWithTemplate>(
+                                        Spec.FromId, Spec.ToTemplate)});
+
+  Refactorer.addDynamicMatcher(Operations.back().Matcher,
+                               Operations.back().Callback.get());
+}
+
+std::unique_ptr<ASTConsumer> QueryReplaceTool::newASTConsumer() {
+  return Refactorer.newASTConsumer();
+}
+}
+}

clang-query/replace-tool/CMakeLists.txt

+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+add_clang_executable(clang-query-replace ClangQueryReplace.cpp)
+target_link_libraries(clang-query-replace
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangDynamicASTMatchers
+  clangFrontend
+  clangQuery
+  clangTooling
+  )
+
+install(TARGETS clang-query-replace RUNTIME DESTINATION bin)
 No newline at end of file

clang-query/replace-tool/ClangQueryReplace.cpp

+//===---- ClangQueryReplace.cpp - clang-query-replace tool ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This tool automatically modifies source files based on AST matchers.
+//
+// A given AST matcher is applied to the source, and a bound node in
+// this matcher is replaced by the result of substituting the text
+// corresponding to other bound nodes into a string template.
+//
+// For example:
+// clang-query-replace --spec <<SPEC foo.c
+//        Query: 'varDecl(hasName("version"),
+//                       hasInitializer(integerLiteral().bind("init")))
+//                       .bind("decl")',
+//        FromId: 'decl',
+//        ToTemplate: 'string version = "$$-${init}";' }
+// SPEC
+//
+// If foo.c contains this:
+//
+// int version = 5;
+// void bar() { long version = 9; }
+//
+// it will be replaced with:
+// string version = "$-5";
+// void bar() { string version = "$-9"; }
+// ===---------------------------------------------------------------------===//
+
+#include "QueryReplace.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include <fstream>
+#include <string>
+#include <system_error>
+
+using namespace clang;
+using namespace clang::query;
+using namespace llvm;
+
+using std::error_code;
+using clang::tooling::CommonOptionsParser;
+
+static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
+static cl::OptionCategory ClangQueryCategory("clang-query-replace options");
+
+static cl::list<std::string> Commands("spec",
+                                      cl::desc("YAML action specification"),
+                                      cl::value_desc("spec"),
+                                      cl::cat(ClangQueryCategory));
+
+int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  CommonOptionsParser OptionsParser(argc, argv, ClangQueryCategory);
+  if (Commands.empty()) {
+    llvm::errs() << "Must specify at least one --spec \n";
+    return 1;
+  }
+
+  tooling::RefactoringTool Refactor(OptionsParser.getCompilations(),
+                                    OptionsParser.getSourcePathList());
+
+  QueryReplaceTool Tool(Refactor.getReplacements());
+  for (auto &SpecYaml : Commands) {
+    llvm::ErrorOr<QueryReplaceSpec> Spec =
+        QueryReplaceSpec::parseFromJSON(SpecYaml);
+    if (error_code EC = Spec.getError()) {
+      llvm::errs() << " Error " << EC.message()
+                   << " while parsing spec: " << SpecYaml << "\n";
+      return 1;
+    }
+    Tool.addOperation(*Spec);
+  }
+
+  return Refactor.runAndSave(tooling::newFrontendActionFactory(&Tool).get());
+}

test/CMakeLists.txt

set(CLANG_TOOLS_TEST_DEPS
   c-index-test
 
   # Individual tools we test.
   clang-apply-replacements
   clang-change-namespace
   clang-include-fixer
   clang-move
   clang-query
+  clang-query-replace
   clang-rename
   clang-reorder-fields
   clang-tidy
   find-all-symbols
   modularize
   pp-trace
 
   # Unit tests
   ExtraToolsUnitTests
   )
-
 add_lit_testsuite(check-clang-tools "Running the Clang extra tools' regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${CLANG_TOOLS_TEST_DEPS}
   ARGS ${CLANG_TOOLS_TEST_EXTRA_ARGS}
   )
 set_target_properties(check-clang-tools PROPERTIES FOLDER "Clang extra tools' tests")
-

test/clang-query/replacement.c

+// RUN: clang-query-replace --spec  '{Query: "varDecl(hasName(\"version\"),
+// hasInitializer(integerLiteral().bind(\"init\"))).bind(\"decl\")", FromId:
+// "decl", ToTemplate: "string version = \"$$-${init}\""}' %s -- && FileCheck %s
+// < %;
+int version = 4;
+// CHECK: string version = "$-4";