[libclang] Enhance support for source-to-source rewriting

[samvv]

Summary

Hi! This is my first time submitting a patch to LLVM/Clang, so apologies if I'm doing anything wrong. Eager to learn.

This patch exposes some additional Rewriter methods in libclang and the Python bindings, making it possible to write simple transformations in Python. This patch modifies the ABI signature of clang_CXRewriter_insertTextBefore. I do not know what the policy is regarding the modification of an existing API function in libclang, so you have been warned.

• Added clang_CXRewriter_getMainFileContents
• Added clang_CXRewriter_insertTextAfter
• Added a parameter to clang_CXRewriter_insertTextBefore to control indentation
• Modified clang_CXRewriter_insertTextBefore to accept an additional InsertIndent argument
• Added a Rewriter class in the Python bindings
• Added test_rewriter.py with some unit tests for the new Python bindings
• Applied clang-format@HEAD to the modified C++ files, which caused some of the existing lines to change

I did not discuss this change over at cfe-dev because I thought the change is fairly trivial. If all changes are required to go through cfe-dev first do let me know.

[samvv]

Attempt at re-running clang-format-13.

[ggeorgakoudis]

I'm not expert on this to review but I'm interested as a user. @samvv are you still working on this? Do you have any input from the community?

[samvv]

ggeorgakoudis no currently not. It has been stale for about a year. I also don't have a lot of time to to follow up right now, but I'll try to help if there is anything I can do.

• Original Message --------

clang/bindings/python/clang/cindex.py

def cursor(self):
         """The Cursor this Token corresponds to."""
         cursor = Cursor()
         cursor._tu = self._tu
 
         conf.lib.clang_annotateTokens(self._tu, byref(self), 1, byref(cursor))
 
         return cursor
 
+class Rewriter(ClangObject):
+
+    @staticmethod
+    def create(tu):
+        return Rewriter(conf.lib.clang_CXRewriter_create(tu))
+
+    @property
+    def contents(self):
+        return conf.lib.clang_CXRewriter_getMainFileContents(self)
+
+    def insertBefore(self, loc, text, indent=False):
+        conf.lib.clang_CXRewriter_insertTextBefore(self, loc, text, indent)
+
+    def insertAfter(self, loc, text, indent=False):
+        conf.lib.clang_CXRewriter_insertTextAfter(self, loc, text, indent)
+
+    def removeText(self, loc):
+        conf.lib.clang_CXRewriter_removeText(self, loc)
+
+    def __del__(self):
+        conf.lib.clang_CXRewriter_dispose(self)
+
 # Now comes the plumbing to hook up the C library.
 
 # Register callback types in common container.
 callbacks['translation_unit_includes'] = CFUNCTYPE(None, c_object_p,
         POINTER(SourceLocation), c_uint, py_object)
 callbacks['cursor_visit'] = CFUNCTYPE(c_int, Cursor, Cursor, py_object)
 callbacks['fields_visit'] = CFUNCTYPE(c_int, Cursor, py_object)
 
functionList = [
   ("clang_CXXMethod_isVirtual",
    [Cursor],
    bool),
 
   ("clang_CXXRecord_isAbstract",
    [Cursor],
    bool),
 
+  ("clang_CXRewriter_create",
+    [TranslationUnit],
+    c_object_p),
+
+  ("clang_CXRewriter_dispose",
+    [Rewriter]),
+
+  ("clang_CXRewriter_getMainFileContents",
+    [Rewriter],
+    _CXString,
+    _CXString.from_result),
+
+  ("clang_CXRewriter_insertTextBefore",
+    [Rewriter, SourceLocation, c_interop_string, c_bool]),
+
+  ("clang_CXRewriter_insertTextAfter",
+    [Rewriter, SourceLocation, c_interop_string, c_bool]),
+
+  ("clang_CXRewriter_removeText",
+    [Rewriter, SourceRange]),
+
   ("clang_EnumDecl_isScoped",
    [Cursor],
    bool),
 
   ("clang_defaultDiagnosticDisplayOptions",
    [],
    c_uint),
 

clang/bindings/python/tests/cindex/test_rewriter.py

+import os
+from clang.cindex import Config
+if 'CLANG_LIBRARY_PATH' in os.environ:
+    Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
+
+from clang.cindex import Rewriter
+from clang.cindex import TranslationUnit
+from clang.cindex import SourceLocation
+from clang.cindex import SourceRange
+import unittest
+from .util import get_cursor
+
+kInputsDir = os.path.join(os.path.dirname(__file__), 'INPUTS')
+
+kNewMainBody = """\
+{
+    printf("dag wereld\\n");
+    return 0;
+}"""
+
+kHelloFileRewritten = """\
+// This is an auto-generated comment
+
+#include "stdio.h"
+
+int main(int argc, char* argv[]) {
+    printf("dag wereld\\n");
+    return 0;
+}
+"""
+
+class TestRewrite(unittest.TestCase):
+    def test_create(self):
+        path = os.path.join(kInputsDir, 'hello.cpp')
+        tu = TranslationUnit.from_source(path)
+        rewiter = Rewriter.create(tu)
+
+    def test_get_contents_no_rewrite(self):
+        path = os.path.join(kInputsDir, 'hello.cpp')
+        tu = TranslationUnit.from_source(path)
+        rewriter = Rewriter.create(tu)
+        with open(path, 'r') as f:
+            text = f.read()
+        self.assertEqual(rewriter.contents, text)
+
+    def test_insert_remove(self):
+        inputPath = os.path.join(kInputsDir, 'hello.cpp')
+        outputPath = os.path.join(kInputsDir, 'hello_rewritten.cpp')
+        tu = TranslationUnit.from_source(inputPath)
+        rewriter = Rewriter.create(tu)
+        rewriter.insertAfter(tu.cursor.extent.start, '// This is an auto-generated comment\n\n')
+        main = get_cursor(tu, 'main')
+        main_xs = list(main.get_children())
+        rewriter.removeText(main_xs[2].extent)
+        rewriter.insertAfter(main_xs[2].extent.start, kNewMainBody)
+        self.assertEqual(rewriter.contents, kHelloFileRewritten)
+

clang/include/clang-c/Rewrite.h

 /*===-- clang-c/Rewrite.h - C CXRewriter   --------------------------*- C -*-===*\
 |*                                                                            *|
 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
 |* Exceptions.                                                                *|
 |* See https://llvm.org/LICENSE.txt for license information.                  *|
 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*/

 typedef void *CXRewriter;
 
 /**
  * Create CXRewriter.
  */
 CINDEX_LINKAGE CXRewriter clang_CXRewriter_create(CXTranslationUnit TU);
 
 /**
- * Insert the specified string at the specified location in the original buffer.
+ * Get the contents of the main file as a string.
  */
-CINDEX_LINKAGE void clang_CXRewriter_insertTextBefore(CXRewriter Rew, CXSourceLocation Loc,
-                                           const char *Insert);
+CXString clang_CXRewriter_getMainFileContents(CXRewriter Rew);
+
+/**
+ * Insert the given string at the specified location in the original buffer.
+ */
+CINDEX_LINKAGE void clang_CXRewriter_insertTextBefore(CXRewriter Rew,
+                                                      CXSourceLocation Loc,
+                                                      const char *Insert,
+                                                      bool IndentNewLines);
+
+/**
+ * Insert the given string right after the specified location in the original
+ * buffer.
+ */
+CINDEX_LINKAGE void clang_CXRewriter_insertTextAfter(CXRewriter Rew,
+                                                     CXSourceLocation Loc,
+                                                     const char *Insert,
+                                                     bool IndentNewLines);
 
 /**
  * Replace the specified range of characters in the input with the specified
  * replacement.
  */
-CINDEX_LINKAGE void clang_CXRewriter_replaceText(CXRewriter Rew, CXSourceRange ToBeReplaced,
+CINDEX_LINKAGE void clang_CXRewriter_replaceText(CXRewriter Rew,
+                                                 CXSourceRange ToBeReplaced,
                                                  const char *Replacement);
 
 /**
  * Remove the specified range.
  */
-CINDEX_LINKAGE void clang_CXRewriter_removeText(CXRewriter Rew, CXSourceRange ToBeRemoved);
+CINDEX_LINKAGE void clang_CXRewriter_removeText(CXRewriter Rew,
+                                                CXSourceRange ToBeRemoved);
 
 /**
  * Save all changed files to disk.
  * Returns 1 if any files were not saved successfully, returns 0 otherwise.
  */
 CINDEX_LINKAGE int clang_CXRewriter_overwriteChangedFiles(CXRewriter Rew);
 
 /**

clang/tools/libclang/Rewrite.cpp

 //===- Rewrite.cpp --------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
CXRewriter clang_CXRewriter_create(CXTranslationUnit TU) {
   }
   clang::ASTUnit *AU = clang::cxtu::getASTUnit(TU);
   assert(AU);
   return reinterpret_cast<CXRewriter>(
       new clang::Rewriter(AU->getSourceManager(), AU->getLangOpts()));
 }
 
 void clang_CXRewriter_insertTextBefore(CXRewriter Rew, CXSourceLocation Loc,
-                            const char *Insert) {
+                                       const char *Insert,
+                                       bool IndentNewLines) {
   assert(Rew);
   clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
-  R.InsertTextBefore(clang::cxloc::translateSourceLocation(Loc), Insert);
+  R.InsertText(clang::cxloc::translateSourceLocation(Loc), Insert, false,
+               IndentNewLines);
+}
+
+void clang_CXRewriter_insertTextAfter(CXRewriter Rew, CXSourceLocation Loc,
+                                      const char *Insert, bool IndentNewLines) {
+  assert(Rew);
+  clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
+  R.InsertText(clang::cxloc::translateSourceLocation(Loc), Insert, true,
+               IndentNewLines);
 }
 
 void clang_CXRewriter_replaceText(CXRewriter Rew, CXSourceRange ToBeReplaced,
                                   const char *Replacement) {
   assert(Rew);
   clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
   R.ReplaceText(clang::cxloc::translateCXRangeToCharRange(ToBeReplaced),
                 Replacement);
 }
 
 void clang_CXRewriter_removeText(CXRewriter Rew, CXSourceRange ToBeRemoved) {
   assert(Rew);
   clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
   R.RemoveText(clang::cxloc::translateCXRangeToCharRange(ToBeRemoved));
 }
 
 int clang_CXRewriter_overwriteChangedFiles(CXRewriter Rew) {
   assert(Rew);
   clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
   return R.overwriteChangedFiles();
 }
 
+CXString clang_CXRewriter_getMainFileContents(CXRewriter Rew) {
+  assert(Rew);
+  clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
+  std::string Res;
+  llvm::raw_string_ostream Out(Res);
+  R.getEditBuffer(R.getSourceMgr().getMainFileID()).write(Out);
+  return clang::cxstring::createDup(Res);
+}
+
 void clang_CXRewriter_writeMainFileToStdOut(CXRewriter Rew) {
   assert(Rew);
   clang::Rewriter &R = *reinterpret_cast<clang::Rewriter *>(Rew);
   R.getEditBuffer(R.getSourceMgr().getMainFileID()).write(llvm::outs());
 }
 
 void clang_CXRewriter_dispose(CXRewriter Rew) {
   if (Rew)
     delete reinterpret_cast<clang::Rewriter *>(Rew);
 }

clang/tools/libclang/libclang.exports

 clang_PrintingPolicy_setProperty
 clang_PrintingPolicy_dispose
 clang_install_aborting_llvm_fatal_error_handler
 clang_uninstall_llvm_fatal_error_handler
 clang_Cursor_getVarDeclInitializer
 clang_Cursor_hasVarDeclGlobalStorage
 clang_Cursor_hasVarDeclExternalStorage
 clang_CXRewriter_create
+clang_CXRewriter_getMainFileContents
 clang_CXRewriter_insertTextBefore
+clang_CXRewriter_insertTextAfter
 clang_CXRewriter_replaceText
 clang_CXRewriter_removeText
 clang_CXRewriter_overwriteChangedFiles
 clang_CXRewriter_writeMainFileToStdOut
 clang_CXRewriter_dispose