[clang-diff] Honor macros

[johannes]

Summary

Nodes that are expanded from some macro are given a special
"Macro" kind instead of an ASTNodeKind. They are compared by
their textual value including arguments, before expansion.

[johannes]

update

include/clang/Tooling/ASTDiff/ASTDiff.h

 // is able to match large subtrees that are equivalent, with an optimal
 // algorithm to match small subtrees.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLING_ASTDIFF_ASTDIFF_H
 #define LLVM_CLANG_TOOLING_ASTDIFF_ASTDIFF_H
 
+#include "clang/Frontend/ASTUnit.h"
 #include "clang/Tooling/ASTDiff/ASTDiffInternal.h"
 
 namespace clang {
 namespace diff {
 
 enum ChangeKind {
   NoChange,
   Delete,    // (Src): delete node Src.
private:
   std::unique_ptr<Impl> DiffImpl;
 };
 
 /// SyntaxTree objects represent subtrees of the AST.
 /// They can be constructed from any Decl or Stmt.
 class SyntaxTree {
 public:
   /// Constructs a tree from a translation unit.
-  SyntaxTree(ASTContext &AST);
+  SyntaxTree(ASTUnit &AST);
   /// Constructs a tree from any AST node.
   template <class T>
-  SyntaxTree(T *Node, ASTContext &AST)
+  SyntaxTree(T *Node, ASTUnit &AST)
       : TreeImpl(llvm::make_unique<Impl>(this, Node, AST)) {}
   SyntaxTree(SyntaxTree &&Other) = default;
   ~SyntaxTree();
 
+  ASTUnit &getASTUnit() const;
   const ASTContext &getASTContext() const;
   StringRef getFilename() const;
 
   int getSize() const;
   NodeRef getRoot() const;
   using PreorderIterator = const Node *;
   PreorderIterator begin() const;
   PreorderIterator end() const;
struct Node {
   NodeId getId() const;
   SyntaxTree &getTree() const;
   const Node *getParent() const;
   NodeRef getChild(size_t Index) const;
   size_t getNumChildren() const { return Children.size(); }
   ast_type_traits::ASTNodeKind getType() const;
   StringRef getTypeLabel() const;
   bool isLeaf() const { return Children.empty(); }
+  bool isMacro() const;
   llvm::Optional<StringRef> getIdentifier() const;
   llvm::Optional<std::string> getQualifiedIdentifier() const;
 
   NodeRefIterator begin() const;
   NodeRefIterator end() const;
 
   int findPositionInParent() const;
 

lib/Tooling/ASTDiff/ASTDiff.cpp

 
 using namespace llvm;
 using namespace clang;
 
 namespace clang {
 namespace diff {
 
 bool ComparisonOptions::isMatchingAllowed(NodeRef N1, NodeRef N2) const {
-  return N1.getType().isSame(N2.getType());
+  return (N1.isMacro() && N2.isMacro()) || N1.getType().isSame(N2.getType());
 }
 
 class ASTDiff::Impl {
 private:
   std::unique_ptr<NodeId[]> SrcToDst, DstToSrc;
 
 public:
   SyntaxTree::Impl &T1, &T2;
struct NodeList {
   size_t size() { return Ids.size(); }
   void sort() { std::sort(Ids.begin(), Ids.end()); }
 };
 } // end anonymous namespace
 
 /// Represents the AST of a TranslationUnit.
 class SyntaxTree::Impl {
 public:
-  Impl(SyntaxTree *Parent, ASTContext &AST);
+  Impl(SyntaxTree *Parent, ASTUnit &AST);
   /// Constructs a tree from an AST node.
-  Impl(SyntaxTree *Parent, Decl *N, ASTContext &AST);
-  Impl(SyntaxTree *Parent, Stmt *N, ASTContext &AST);
+  Impl(SyntaxTree *Parent, Decl *N, ASTUnit &AST);
+  Impl(SyntaxTree *Parent, Stmt *N, ASTUnit &AST);
   template <class T>
   Impl(SyntaxTree *Parent,
        typename std::enable_if<std::is_base_of<Stmt, T>::value, T>::type *Node,
-       ASTContext &AST)
+       ASTUnit &AST)
       : Impl(Parent, dyn_cast<Stmt>(Node), AST) {}
   template <class T>
   Impl(SyntaxTree *Parent,
        typename std::enable_if<std::is_base_of<Decl, T>::value, T>::type *Node,
-       ASTContext &AST)
+       ASTUnit &AST)
       : Impl(Parent, dyn_cast<Decl>(Node), AST) {}
 
   SyntaxTree *Parent;
-  ASTContext &AST;
+  ASTUnit &AST;
   PrintingPolicy TypePP;
   /// Nodes in preorder.
   std::vector<Node> Nodes;
   NodeList Leaves;
   // Maps preorder indices to postorder ones.
   std::vector<int> PostorderIds;
   NodeList NodesBfs;
 

 }
 
 static bool isSpecializedNodeExcluded(const Decl *D) { return D->isImplicit(); }
 static bool isSpecializedNodeExcluded(const Stmt *S) { return false; }
 static bool isSpecializedNodeExcluded(CXXCtorInitializer *I) {
   return !I->isWritten();
 }
 
-template <class T> static bool isNodeExcluded(const SourceManager &SM, T *N) {
+template <class T> static bool isNodeExcluded(ASTUnit &AST, T *N) {
+  const SourceManager &SM = AST.getSourceManager();
   if (!N)
     return true;
   SourceLocation SLoc = N->getSourceRange().getBegin();
   if (SLoc.isValid()) {
     // Ignore everything from other files.
     if (!SM.isInMainFile(SLoc))
       return true;
-    // Ignore macros.
-    if (SLoc != SM.getSpellingLoc(SLoc))
+    const Preprocessor &PP = AST.getPreprocessor();
+    if (SLoc.isMacroID() && !PP.isAtStartOfMacroExpansion(SLoc))
       return true;
   }
   return isSpecializedNodeExcluded(N);
 }
 
+static SourceRange getSourceRange(const ASTUnit &AST, const DynTypedNode &DTN) {
+  const SourceManager &SM = AST.getSourceManager();
+  SourceRange Range = DTN.getSourceRange();
+  SourceLocation BeginLoc = Range.getBegin();
+  SourceLocation EndLoc;
+  if (BeginLoc.isMacroID())
+    EndLoc = SM.getExpansionRange(BeginLoc).second;
+  else
+    EndLoc = Range.getEnd();
+  EndLoc =
+      Lexer::getLocForEndOfToken(EndLoc, /*Offset=*/0, SM, AST.getLangOpts());
+  if (auto *ThisExpr = DTN.get<CXXThisExpr>()) {
+    if (ThisExpr->isImplicit())
+      EndLoc = BeginLoc;
+  }
+  return {SM.getExpansionLoc(BeginLoc), SM.getExpansionLoc(EndLoc)};
+}
+
 namespace {
 // Sets Height, Parent and Children for each node.
 struct PreorderVisitor : public RecursiveASTVisitor<PreorderVisitor> {
   int Id = 0, Depth = 0;
   NodeId Parent;
   SyntaxTree::Impl &Tree;
 
   PreorderVisitor(SyntaxTree::Impl &Tree) : Tree(Tree) {}
assert(N.RightMostDescendant >= Tree.getRootId() &&
            "Rightmost descendant must be a valid tree node.");
     if (N.isLeaf())
       Tree.Leaves.push_back(MyId);
     N.Height = 1;
     for (NodeId Child : N.Children)
       N.Height = std::max(N.Height, 1 + Tree.getNode(Child).Height);
   }
   bool TraverseDecl(Decl *D) {
-    if (isNodeExcluded(Tree.AST.getSourceManager(), D))
+    if (isNodeExcluded(Tree.AST, D))
       return true;
     auto SavedState = PreTraverse(D);
     RecursiveASTVisitor<PreorderVisitor>::TraverseDecl(D);
     PostTraverse(SavedState);
     return true;
   }
   bool TraverseStmt(Stmt *S) {
     if (S)
       S = S->IgnoreImplicit();
-    if (isNodeExcluded(Tree.AST.getSourceManager(), S))
+    if (isNodeExcluded(Tree.AST, S))
       return true;
     auto SavedState = PreTraverse(S);
     RecursiveASTVisitor<PreorderVisitor>::TraverseStmt(S);
     PostTraverse(SavedState);
     return true;
   }
   bool TraverseType(QualType T) { return true; }
   bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
-    if (isNodeExcluded(Tree.AST.getSourceManager(), Init))
+    if (isNodeExcluded(Tree.AST, Init))
       return true;
     auto SavedState = PreTraverse(Init);
     RecursiveASTVisitor<PreorderVisitor>::TraverseConstructorInitializer(Init);
     PostTraverse(SavedState);
     return true;
   }
 };
 } // end anonymous namespace
 
-SyntaxTree::Impl::Impl(SyntaxTree *Parent, ASTContext &AST)
+SyntaxTree::Impl::Impl(SyntaxTree *Parent, ASTUnit &AST)
     : Parent(Parent), AST(AST), TypePP(AST.getLangOpts()), Leaves(*this),
       NodesBfs(*this) {
   TypePP.AnonymousTagLocations = false;
 }
 
-SyntaxTree::Impl::Impl(SyntaxTree *Parent, Decl *N, ASTContext &AST)
+SyntaxTree::Impl::Impl(SyntaxTree *Parent, Decl *N, ASTUnit &AST)
     : Impl(Parent, AST) {
   PreorderVisitor PreorderWalker(*this);
   PreorderWalker.TraverseDecl(N);
   initTree();
 }
 
-SyntaxTree::Impl::Impl(SyntaxTree *Parent, Stmt *N, ASTContext &AST)
+SyntaxTree::Impl::Impl(SyntaxTree *Parent, Stmt *N, ASTUnit &AST)
     : Impl(Parent, AST) {
   PreorderVisitor PreorderWalker(*this);
   PreorderWalker.TraverseStmt(N);
   initTree();
 }
 
 static std::vector<NodeId> getSubtreePostorder(SyntaxTree::Impl &Tree,
                                                NodeId Root) {
if (!ContextPrefix.empty() && StringRef(Val).startswith(ContextPrefix))
     Val = Val.substr(ContextPrefix.size() + 1);
   return Val;
 }
 
 std::string SyntaxTree::Impl::getRelativeName(const NamedDecl *ND) const {
   return getRelativeName(ND, ND->getDeclContext());
 }
 
-static const DeclContext *getEnclosingDeclContext(ASTContext &AST,
-                                                  const Stmt *S) {
+static const DeclContext *getEnclosingDeclContext(ASTUnit &AST, const Stmt *S) {
   while (S) {
-    const auto &Parents = AST.getParents(*S);
+    const auto &Parents = AST.getASTContext().getParents(*S);
     if (Parents.empty())
       return nullptr;
     const auto &P = Parents[0];
     if (const auto *D = P.get<Decl>())
       return D->getDeclContext();
     S = P.get<Stmt>();
   }
   return nullptr;
 }
 
 static std::string getInitializerValue(const CXXCtorInitializer *Init,
                                        const PrintingPolicy &TypePP) {
   if (Init->isAnyMemberInitializer())
     return Init->getAnyMember()->getName();
   if (Init->isBaseInitializer())
     return QualType(Init->getBaseClass(), 0).getAsString(TypePP);
   if (Init->isDelegatingInitializer())
     return Init->getTypeSourceInfo()->getType().getAsString(TypePP);
   llvm_unreachable("Unknown initializer type");
 }
 
 std::string SyntaxTree::Impl::getNodeValue(NodeRef N) const {
   assert(&N.Tree == this);
   const DynTypedNode &DTN = N.ASTNode;
+  if (N.isMacro()) {
+    CharSourceRange Range(getSourceRange(AST, N.ASTNode), false);
+    return Lexer::getSourceText(Range, AST.getSourceManager(),
+                                AST.getLangOpts());
+  }
   if (auto *S = DTN.get<Stmt>())
     return getStmtValue(S);
   if (auto *D = DTN.get<Decl>())
     return getDeclValue(D);
   if (auto *Init = DTN.get<CXXCtorInitializer>())
     return getInitializerValue(Init, TypePP);
   llvm_unreachable("Fatal: unhandled AST node.\n");
 }

 NodeRef Node::getChild(size_t Index) const {
   return Tree.getNode(Children[Index]);
 }
 
 ast_type_traits::ASTNodeKind Node::getType() const {
   return ASTNode.getNodeKind();
 }
 
-StringRef Node::getTypeLabel() const { return getType().asStringRef(); }
+StringRef Node::getTypeLabel() const {
+  if (isMacro())
+    return "Macro";
+  return getType().asStringRef();
+}
+
+bool Node::isMacro() const {
+  return ASTNode.getSourceRange().getBegin().isMacroID();
+}
 
 llvm::Optional<std::string> Node::getQualifiedIdentifier() const {
+  if (isMacro())
+    return llvm::None;
   if (auto *ND = ASTNode.get<NamedDecl>()) {
     if (ND->getDeclName().isIdentifier())
       return ND->getQualifiedNameAsString();
   }
   return llvm::None;
 }
 
 llvm::Optional<StringRef> Node::getIdentifier() const {
+  if (isMacro())
+    return llvm::None;
   if (auto *ND = ASTNode.get<NamedDecl>()) {
     if (ND->getDeclName().isIdentifier())
       return ND->getName();
   }
   return llvm::None;
 }
 
 NodeRefIterator Node::begin() const {
   return {&Tree, isLeaf() ? nullptr : &Children[0]};
 }
 NodeRefIterator Node::end() const {
   return {&Tree, isLeaf() ? nullptr : &Children[0] + Children.size()};
 }
 
 int Node::findPositionInParent() const {
   if (!getParent())
     return 0;
   const ArrayRef<NodeId> &Siblings = getParent()->Children;
   return std::find(Siblings.begin(), Siblings.end(), getId()) -
          Siblings.begin();
 }
 
 std::pair<unsigned, unsigned> Node::getSourceRangeOffsets() const {
   const SourceManager &SM = Tree.AST.getSourceManager();
-  SourceRange Range = ASTNode.getSourceRange();
-  SourceLocation BeginLoc = Range.getBegin();
-  SourceLocation EndLoc = Lexer::getLocForEndOfToken(
-      Range.getEnd(), /*Offset=*/0, SM, Tree.AST.getLangOpts());
-  if (auto *ThisExpr = ASTNode.get<CXXThisExpr>()) {
-    if (ThisExpr->isImplicit())
-      EndLoc = BeginLoc;
-  }
-  unsigned Begin = SM.getFileOffset(SM.getExpansionLoc(BeginLoc));
-  unsigned End = SM.getFileOffset(SM.getExpansionLoc(EndLoc));
+  SourceRange Range = getSourceRange(Tree.AST, ASTNode);
+  unsigned Begin = SM.getFileOffset(Range.getBegin());
+  unsigned End = SM.getFileOffset(Range.getEnd());
   return {Begin, End};
 }
 
 namespace {
 // Compares nodes by their depth.
 struct HeightLess {
   SyntaxTree::Impl &Tree;
   HeightLess(SyntaxTree::Impl &Tree) : Tree(Tree) {}
ASTDiff::ASTDiff(SyntaxTree &T1, SyntaxTree &T2,
     : DiffImpl(llvm::make_unique<Impl>(*T1.TreeImpl, *T2.TreeImpl, Options)) {}
 
 ASTDiff::~ASTDiff() = default;
 
 const Node *ASTDiff::getMapped(NodeRef N) const {
   return DiffImpl->getMapped(N);
 }
 
-SyntaxTree::SyntaxTree(ASTContext &AST)
+SyntaxTree::SyntaxTree(ASTUnit &AST)
     : TreeImpl(llvm::make_unique<SyntaxTree::Impl>(
-          this, AST.getTranslationUnitDecl(), AST)) {}
+          this, AST.getASTContext().getTranslationUnitDecl(), AST)) {}
 
 SyntaxTree::~SyntaxTree() = default;
 
-const ASTContext &SyntaxTree::getASTContext() const { return TreeImpl->AST; }
+ASTUnit &SyntaxTree::getASTUnit() const { return TreeImpl->AST; }
+
+const ASTContext &SyntaxTree::getASTContext() const {
+  return TreeImpl->AST.getASTContext();
+}
 
 NodeRef SyntaxTree::getNode(NodeId Id) const { return TreeImpl->getNode(Id); }
 
 int SyntaxTree::getSize() const { return TreeImpl->getSize(); }
 NodeRef SyntaxTree::getRoot() const { return TreeImpl->getRoot(); }
 SyntaxTree::PreorderIterator SyntaxTree::begin() const {
   return TreeImpl->begin();
 }
 SyntaxTree::PreorderIterator SyntaxTree::end() const { return TreeImpl->end(); }
 
 std::string SyntaxTree::getNodeValue(NodeRef N) const {
   return TreeImpl->getNodeValue(N);
 }
 
 } // end namespace diff
 } // end namespace clang

test/Tooling/Inputs/clang-diff-basic-src.cpp

public:
   int id(int i) { return i; }
 };
 }
 
 void m() { int x = 0 + 0 + 0; }
 int um = 1 * 2 + 3;
 
 void f1() {{ (void) __func__;;; }}
+
+#define M1 return 1 + 1
+#define M2 return 1 + 2
+#define F(a, b) return a + b;
+
+int f2() {
+  M1;
+  M1;
+  F(1, 1);
+}

test/Tooling/clang-diff-ast.cpp

X(char s, int) : Base(), m(0) {
     int x = m;
   }
   // CHECK: CXXConstructorDecl: :X(void (char){{( __attribute__\(\(thiscall\)\))?}})(
   // CHECK: CXXCtorInitializer: X
   X(char s) : X(s, 4) {}
 };
 
 #define M (void)1
-#define MA(a, b) (void)a, b
-// CHECK: FunctionDecl
-// CHECK-NEXT: CompoundStmt
+#define F(a, b) (void)a, b
 void macros() {
+  // CHECK: Macro: M(
   M;
-  MA(1, 2);
+  // two expressions, therefore it occurs twice
+  // CHECK-NEXT: Macro: F(1, 2)(
+  // CHECK-NEXT: Macro: F(1, 2)(
+  F(1, 2);
 }
 
 #ifndef GUARD
 #define GUARD
 // CHECK-NEXT: NamespaceDecl
 namespace world {
 // nodes from other files are excluded, there should be no output here
 #include "clang-diff-ast.cpp"
 }
 // CHECK-NEXT: FunctionDecl: sentinel
 void sentinel();
 #endif

test/Tooling/clang-diff-basic.cpp

 int um = 1 + 7;
 
 namespace {
 // match with parents of different type
 // CHECK: Match FunctionDecl: f1{{.*}} to FunctionDecl: (anonymous namespace)::f1
 void f1() {{ (void) __func__;;; }}
 }
 
+// macros are always compared by their full textual value
+
+#define M1 return 1 + 1
+#define M2 return 2 * 2
+#define F(a, b) return a + b;
+
+int f2() {
+  // CHECK: Match Macro: M1{{.*}} to Macro: M1
+  M1;
+  // CHECK: Update Macro: M1{{.*}} to M2
+  M2;
+  // CHECK: Update Macro: F(1, 1){{.*}} to F(1, /*b=*/1)
+  F(1, /*b=*/1);
+}
+
 // CHECK: Delete AccessSpecDecl: public
 // CHECK: Delete CXXMethodDecl

tools/clang-diff/ClangDiff.cpp

int main(int argc, const char **argv) {
   if (ASTDump || ASTDumpJson) {
     if (!DestinationPath.empty()) {
       llvm::errs() << "Error: Please specify exactly one filename.\n";
       return 1;
     }
     std::unique_ptr<ASTUnit> AST = getAST(CommonCompilations, SourcePath);
     if (!AST)
       return 1;
-    diff::SyntaxTree Tree(AST->getASTContext());
+    diff::SyntaxTree Tree(*AST);
     if (ASTDump) {
       printTree(llvm::outs(), Tree);
       return 0;
     }
     llvm::outs() << R"({"filename":")";
     printJsonString(llvm::outs(), SourcePath);
     llvm::outs() << R"(","root":)";
     printNodeAsJson(llvm::outs(), Tree, Tree.getRoot());
if (StopAfter == "topdown")
       Options.StopAfterTopDown = true;
     else if (StopAfter == "bottomup")
       Options.StopAfterBottomUp = true;
     else {
       llvm::errs() << "Error: Invalid argument for -stop-diff-after\n";
       return 1;
     }
   }
-  diff::SyntaxTree SrcTree(Src->getASTContext());
-  diff::SyntaxTree DstTree(Dst->getASTContext());
+  diff::SyntaxTree SrcTree(*Src);
+  diff::SyntaxTree DstTree(*Dst);
   diff::ASTDiff Diff(SrcTree, DstTree, Options);
 
   if (HtmlDiff) {
     llvm::outs() << HtmlDiffHeader << "<pre>";
     llvm::outs() << "<div id='L' class='code'>";
     printHtmlForNode(llvm::outs(), Diff, true, SrcTree.getRoot(), 0);
     llvm::outs() << "</div>";
     llvm::outs() << "<div id='R' class='code'>";