[LibTooling] Add "SourceCode" library for functions relating to source-code manipulation.

[ymandel]

Summary

Introduces a utility library in Refactoring/ to collect routines related to
source-code manipulation. In this change, we move "extended-range" functions
from the FixIt library (in clangTooling) to this new library.

We need to use this functionality in Refactoring/ and cannot access it if it
resides in Tooling/, because that would cause clangToolingRefactor to depend on
clangTooling, which would be a circular dependency.

[ymandel]

Add tests for SourceCode and remove corresponding tests from FixIt tests.

[ymandel]

Update CMakeLists.txt and rename (misnamed) test file.

[ymandel]

Various tweaks to get tests compiling/passing.

[ilya-biryukov]

Nice! A more specific name and a more specific library.
LGTM!

[ymandel]

Delete unused function.

[ymandel]

Fix trailing whitespace

clang/include/clang/Tooling/FixIt.h

 //    tooling::fixit::createReplacement(*B, *A);
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLING_FIXIT_H
 #define LLVM_CLANG_TOOLING_FIXIT_H
 
 #include "clang/AST/ASTContext.h"
-#include "clang/Basic/TokenKinds.h"
 
 namespace clang {
 namespace tooling {
 namespace fixit {
 
 namespace internal {
 StringRef getText(CharSourceRange Range, const ASTContext &Context);
 
 /// Returns the token CharSourceRange corresponding to \p Range.
 inline CharSourceRange getSourceRange(const SourceRange &Range) {
   return CharSourceRange::getTokenRange(Range);
 }
 
 /// Returns the CharSourceRange of the token at Location \p Loc.
 inline CharSourceRange getSourceRange(const SourceLocation &Loc) {
   return CharSourceRange::getTokenRange(Loc, Loc);
 }
 
 /// Returns the CharSourceRange of an given Node. \p Node is typically a
 ///        'Stmt', 'Expr' or a 'Decl'.
 template <typename T> CharSourceRange getSourceRange(const T &Node) {
   return CharSourceRange::getTokenRange(Node.getSourceRange());
 }
-
-/// Extends \p Range to include the token \p Next, if it immediately follows the
-/// end of the range. Otherwise, returns \p Range unchanged.
-CharSourceRange maybeExtendRange(CharSourceRange Range, tok::TokenKind Next,
-                                 ASTContext &Context);
 } // end namespace internal
 
 /// Returns a textual representation of \p Node.
 template <typename T>
 StringRef getText(const T &Node, const ASTContext &Context) {
   return internal::getText(internal::getSourceRange(Node), Context);
 }
 
-/// Returns the source range spanning the node, extended to include \p Next, if
-/// it immediately follows \p Node. Otherwise, returns the normal range of \p
-/// Node.  See comments on `getExtendedText()` for examples.
-template <typename T>
-CharSourceRange getExtendedRange(const T &Node, tok::TokenKind Next,
-                                 ASTContext &Context) {
-  return internal::maybeExtendRange(internal::getSourceRange(Node), Next,
-                                    Context);
-}
-
-/// Returns the source text of the node, extended to include \p Next, if it
-/// immediately follows the node. Otherwise, returns the text of just \p Node.
-///
-/// For example, given statements S1 and S2 below:
-/// \code
-///   {
-///     // S1:
-///     if (!x) return foo();
-///     // S2:
-///     if (!x) { return 3; }
-///   }
-/// \endcode
-/// then
-/// \code
-///   getText(S1, Context) = "if (!x) return foo()"
-///   getExtendedText(S1, tok::TokenKind::semi, Context)
-///     = "if (!x) return foo();"
-///   getExtendedText(*S1.getThen(), tok::TokenKind::semi, Context)
-///     = "return foo();"
-///   getExtendedText(*S2.getThen(), tok::TokenKind::semi, Context)
-///     = getText(S2, Context) = "{ return 3; }"
-/// \endcode
-template <typename T>
-StringRef getExtendedText(const T &Node, tok::TokenKind Next,
-                          ASTContext &Context) {
-  return internal::getText(getExtendedRange(Node, Next, Context), Context);
-}
-
 // Returns a FixItHint to remove \p Node.
 // TODO: Add support for related syntactical elements (i.e. comments, ...).
 template <typename T> FixItHint createRemoval(const T &Node) {
   return FixItHint::CreateRemoval(internal::getSourceRange(Node));
 }
 
 // Returns a FixItHint to replace \p Destination by \p Source.
 template <typename D, typename S>

clang/include/clang/Tooling/Refactoring/SourceCode.h

+//===--- SourceCode.h - Source code manipulation routines -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file provides functions that simplify extraction of source code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_H
+#define LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_H
+
+#include "clang/AST/ASTContext.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
+
+namespace clang {
+namespace tooling {
+
+/// Extends \p Range to include the token \p Next, if it immediately follows the
+/// end of the range. Otherwise, returns \p Range unchanged.
+CharSourceRange maybeExtendRange(CharSourceRange Range, tok::TokenKind Next,
+                                 ASTContext &Context);
+
+/// Returns the source range spanning the node, extended to include \p Next, if
+/// it immediately follows \p Node. Otherwise, returns the normal range of \p
+/// Node.  See comments on `getExtendedText()` for examples.
+template <typename T>
+CharSourceRange getExtendedRange(const T &Node, tok::TokenKind Next,
+                                 ASTContext &Context) {
+  return maybeExtendRange(CharSourceRange::getTokenRange(Node.getSourceRange()),
+                          Next, Context);
+}
+
+/// Returns the source-code text in the specified range.
+StringRef getText(CharSourceRange Range, const ASTContext &Context);
+
+/// Returns the source-code text corresponding to \p Node.
+template <typename T>
+StringRef getText(const T &Node, const ASTContext &Context) {
+  return getText(CharSourceRange::getTokenRange(Node.getSourceRange()),
+                 Context);
+}
+
+/// Returns the source text of the node, extended to include \p Next, if it
+/// immediately follows the node. Otherwise, returns the text of just \p Node.
+///
+/// For example, given statements S1 and S2 below:
+/// \code
+///   {
+///     // S1:
+///     if (!x) return foo();
+///     // S2:
+///     if (!x) { return 3; }
+///   }
+/// \endcode
+/// then
+/// \code
+///   getText(S1, Context) = "if (!x) return foo()"
+///   getExtendedText(S1, tok::TokenKind::semi, Context)
+///     = "if (!x) return foo();"
+///   getExtendedText(*S1.getThen(), tok::TokenKind::semi, Context)
+///     = "return foo();"
+///   getExtendedText(*S2.getThen(), tok::TokenKind::semi, Context)
+///     = getText(S2, Context) = "{ return 3; }"
+/// \endcode
+template <typename T>
+StringRef getExtendedText(const T &Node, tok::TokenKind Next,
+                          ASTContext &Context) {
+  return getText(getExtendedRange(Node, Next, Context), Context);
+}
+} // namespace tooling
+} // namespace clang
+#endif // LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_H

clang/lib/Tooling/FixIt.cpp

 namespace tooling {
 namespace fixit {
 
 namespace internal {
 StringRef getText(CharSourceRange Range, const ASTContext &Context) {
   return Lexer::getSourceText(Range, Context.getSourceManager(),
                               Context.getLangOpts());
 }
-
-CharSourceRange maybeExtendRange(CharSourceRange Range, tok::TokenKind Next,
-                                 ASTContext &Context) {
-  Optional<Token> Tok = Lexer::findNextToken(
-      Range.getEnd(), Context.getSourceManager(), Context.getLangOpts());
-  if (!Tok || !Tok->is(Next))
-    return Range;
-  return CharSourceRange::getTokenRange(Range.getBegin(), Tok->getLocation());
-}
 } // namespace internal
 
 } // end namespace fixit
 } // end namespace tooling
 } // end namespace clang

clang/lib/Tooling/Refactoring/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS Support)
 
 add_clang_library(clangToolingRefactor
   ASTSelection.cpp
   ASTSelectionRequirements.cpp
   AtomicChange.cpp
   Extract/Extract.cpp
   Extract/SourceExtraction.cpp
   RefactoringActions.cpp
   Rename/RenamingAction.cpp
   Rename/SymbolOccurrences.cpp
   Rename/USRFinder.cpp
   Rename/USRFindingAction.cpp
   Rename/USRLocFinder.cpp
+  SourceCode.cpp
 
   LINK_LIBS
   clangAST
   clangASTMatchers
   clangBasic
   clangFormat
   clangIndex
   clangLex
   clangRewrite
   clangToolingCore
   )

clang/lib/Tooling/Refactoring/SourceCode.cpp

-//===--- FixIt.cpp - FixIt Hint utilities -----------------------*- C++ -*-===//
+//===--- SourceCode.cpp - Source code manipulation routines -----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains implementations of utitilies to ease source code rewriting
-// by providing helper functions related to FixItHint.
+//  This file provides functions that simplify extraction of source code.
 //
 //===----------------------------------------------------------------------===//
-#include "clang/Tooling/FixIt.h"
+#include "clang/Tooling/Refactoring/SourceCode.h"
 #include "clang/Lex/Lexer.h"
 
-namespace clang {
-namespace tooling {
-namespace fixit {
+using namespace clang;
 
-namespace internal {
-StringRef getText(CharSourceRange Range, const ASTContext &Context) {
+StringRef clang::tooling::getText(CharSourceRange Range,
+                                  const ASTContext &Context) {
   return Lexer::getSourceText(Range, Context.getSourceManager(),
                               Context.getLangOpts());
 }
 
-CharSourceRange maybeExtendRange(CharSourceRange Range, tok::TokenKind Next,
+CharSourceRange clang::tooling::maybeExtendRange(CharSourceRange Range,
+                                                 tok::TokenKind Next,
                                                  ASTContext &Context) {
   Optional<Token> Tok = Lexer::findNextToken(
       Range.getEnd(), Context.getSourceManager(), Context.getLangOpts());
   if (!Tok || !Tok->is(Next))
     return Range;
   return CharSourceRange::getTokenRange(Range.getBegin(), Tok->getLocation());
 }
-} // namespace internal
-
-} // end namespace fixit
-} // end namespace tooling
-} // end namespace clang

clang/unittests/Tooling/CMakeLists.txt

add_clang_unittest(ToolingTests
   RecursiveASTVisitorTestDeclVisitor.cpp
   RecursiveASTVisitorTestPostOrderVisitor.cpp
   RecursiveASTVisitorTestTypeLocVisitor.cpp
   RefactoringActionRulesTest.cpp
   RefactoringCallbacksTest.cpp
   RefactoringTest.cpp
   ReplacementsYamlTest.cpp
   RewriterTest.cpp
+  SourceCodeTest.cpp
   ToolingTest.cpp
   )
 
 target_link_libraries(ToolingTests
   PRIVATE
   clangAST
   clangASTMatchers
   clangBasic

clang/unittests/Tooling/FixItTest.cpp

 //===- unittest/Tooling/FixitTest.cpp ------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "TestVisitor.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Tooling/FixIt.h"
 
 using namespace clang;
 
 using tooling::fixit::getText;
-using tooling::fixit::getExtendedText;
 using tooling::fixit::createRemoval;
 using tooling::fixit::createReplacement;
 
 namespace {
 
 struct CallsVisitor : TestVisitor<CallsVisitor> {
   bool VisitCallExpr(CallExpr *Expr) {
     OnCall(Expr, Context);
Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
     Expr *P1 = CE->getArg(1);
     EXPECT_EQ("x", getText(*P0, *Context));
     EXPECT_EQ("y", getText(*P1, *Context));
   };
   Visitor.runOver("#define FOO(x, y) (void)x; (void)y; foo(x, y);\n"
                   "void foo(int x, int y) { FOO(x,y) }");
 }
 
-TEST(FixItTest, getExtendedText) {
-  CallsVisitor Visitor;
-
-  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
-    EXPECT_EQ("foo(x, y);",
-              getExtendedText(*CE, tok::TokenKind::semi, *Context));
-
-    Expr *P0 = CE->getArg(0);
-    Expr *P1 = CE->getArg(1);
-    EXPECT_EQ("x", getExtendedText(*P0, tok::TokenKind::semi, *Context));
-    EXPECT_EQ("x,", getExtendedText(*P0, tok::TokenKind::comma, *Context));
-    EXPECT_EQ("y", getExtendedText(*P1, tok::TokenKind::semi, *Context));
-  };
-  Visitor.runOver("void foo(int x, int y) { foo(x, y); }");
-  Visitor.runOver("void foo(int x, int y) { if (true) foo(x, y); }");
-  Visitor.runOver("int foo(int x, int y) { if (true) return 3 + foo(x, y); }");
-  Visitor.runOver("void foo(int x, int y) { for (foo(x, y);;) ++x; }");
-  Visitor.runOver(
-      "bool foo(int x, int y) { for (;foo(x, y);) x = 1; return true; }");
-
-  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
-    EXPECT_EQ("foo()", getExtendedText(*CE, tok::TokenKind::semi, *Context));
-  };
-  Visitor.runOver("bool foo() { if (foo()) return true; return false; }");
-  Visitor.runOver("void foo() { int x; for (;; foo()) ++x; }");
-  Visitor.runOver("int foo() { return foo() + 3; }");
-}
-
 TEST(FixItTest, createRemoval) {
   CallsVisitor Visitor;
 
   Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
     FixItHint Hint = createRemoval(*CE);
     EXPECT_EQ("foo(x, y)", getText(Hint.RemoveRange.getAsRange(), *Context));
     EXPECT_TRUE(Hint.InsertFromRange.isInvalid());
     EXPECT_TRUE(Hint.CodeToInsert.empty());

clang/unittests/Tooling/SourceCodeTest.cpp

+//===- unittest/Tooling/SourceCodeTest.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestVisitor.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Tooling/Refactoring/SourceCode.h"
+
+using namespace clang;
+
+using tooling::getText;
+using tooling::getExtendedText;
+
+namespace {
+
+struct CallsVisitor : TestVisitor<CallsVisitor> {
+  bool VisitCallExpr(CallExpr *Expr) {
+    OnCall(Expr, Context);
+    return true;
+  }
+
+  std::function<void(CallExpr *, ASTContext *Context)> OnCall;
+};
+
+TEST(SourceCodeTest, getText) {
+  CallsVisitor Visitor;
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("foo(x, y)", getText(*CE, *Context));
+  };
+  Visitor.runOver("void foo(int x, int y) { foo(x, y); }");
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("APPLY(foo, x, y)", getText(*CE, *Context));
+  };
+  Visitor.runOver("#define APPLY(f, x, y) f(x, y)\n"
+                  "void foo(int x, int y) { APPLY(foo, x, y); }");
+}
+
+TEST(SourceCodeTest, getTextWithMacro) {
+  CallsVisitor Visitor;
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("F OO", getText(*CE, *Context));
+    Expr *P0 = CE->getArg(0);
+    Expr *P1 = CE->getArg(1);
+    EXPECT_EQ("", getText(*P0, *Context));
+    EXPECT_EQ("", getText(*P1, *Context));
+  };
+  Visitor.runOver("#define F foo(\n"
+                  "#define OO x, y)\n"
+                  "void foo(int x, int y) { F OO ; }");
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("", getText(*CE, *Context));
+    Expr *P0 = CE->getArg(0);
+    Expr *P1 = CE->getArg(1);
+    EXPECT_EQ("x", getText(*P0, *Context));
+    EXPECT_EQ("y", getText(*P1, *Context));
+  };
+  Visitor.runOver("#define FOO(x, y) (void)x; (void)y; foo(x, y);\n"
+                  "void foo(int x, int y) { FOO(x,y) }");
+}
+
+TEST(SourceCodeTest, getExtendedText) {
+  CallsVisitor Visitor;
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("foo(x, y);",
+              getExtendedText(*CE, tok::TokenKind::semi, *Context));
+
+    Expr *P0 = CE->getArg(0);
+    Expr *P1 = CE->getArg(1);
+    EXPECT_EQ("x", getExtendedText(*P0, tok::TokenKind::semi, *Context));
+    EXPECT_EQ("x,", getExtendedText(*P0, tok::TokenKind::comma, *Context));
+    EXPECT_EQ("y", getExtendedText(*P1, tok::TokenKind::semi, *Context));
+  };
+  Visitor.runOver("void foo(int x, int y) { foo(x, y); }");
+  Visitor.runOver("void foo(int x, int y) { if (true) foo(x, y); }");
+  Visitor.runOver("int foo(int x, int y) { if (true) return 3 + foo(x, y); }");
+  Visitor.runOver("void foo(int x, int y) { for (foo(x, y);;) ++x; }");
+  Visitor.runOver(
+      "bool foo(int x, int y) { for (;foo(x, y);) x = 1; return true; }");
+
+  Visitor.OnCall = [](CallExpr *CE, ASTContext *Context) {
+    EXPECT_EQ("foo()", getExtendedText(*CE, tok::TokenKind::semi, *Context));
+  };
+  Visitor.runOver("bool foo() { if (foo()) return true; return false; }");
+  Visitor.runOver("void foo() { int x; for (;; foo()) ++x; }");
+  Visitor.runOver("int foo() { return foo() + 3; }");
+}
+
+} // end anonymous namespace