Index: include/clang/Tooling/ASTDiff/ASTDiff.h =================================================================== --- /dev/null +++ include/clang/Tooling/ASTDiff/ASTDiff.h @@ -0,0 +1,119 @@ +//===- ASTDiff.h - AST differencing API -----------------------*- C++ -*- -===// +// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_ASTDIFF_ASTDIFF_H +#define LLVM_CLANG_TOOLING_ASTDIFF_ASTDIFF_H + +#include "clang/AST/ASTTypeTraits.h" +#include "clang/Frontend/ASTUnit.h" + +using namespace llvm; +using namespace clang; + +namespace clang { +namespace diff { + +/// Within a tree, this identifies a node by its postorder offset. +using NodeId = int; + +/// Sentinel value for invalid nodes. +const NodeId NoNodeId = -1; + +/// Represents a Clang AST node, alongside some additional information. +struct Node { + NodeId Parent, LeftMostDescendant; + int Depth, Height; + ast_type_traits::DynTypedNode ASTNode; + // Maybe there is a better way to store children than this. + SmallVector Children; + + ast_type_traits::ASTNodeKind getType() const { return ASTNode.getNodeKind(); } + bool hasSameType(const Node &Other) const { + return getType().isSame(Other.getType()) || + (getType().isNone() && Other.getType().isNone()); + } + const StringRef getTypeLabel() const { return getType().asStringRef(); } + bool isLeaf() const { return Children.empty(); } +}; + +// Represents the AST of a TranslationUnit. +class TreeRoot { +private: + std::vector Postorder; + void setLeftMostDescendants(); + +public: + int MaxDepth = 0; + ASTUnit &AST; + std::vector Leaves; + + TreeRoot(ASTUnit &AST); + + void setSize(size_t Size) { Postorder.resize(Size); } + int getSize() const { return Postorder.size(); } + NodeId root() const { return getSize() - 1; } + + const Node &getNode(NodeId Id) const { return Postorder[Id]; } + Node &getMutableNode(NodeId Id) { return Postorder[Id]; } + bool isValidNodeId(NodeId Id) const { return Id >= 0 && Id < getSize(); } + void addNode(Node &N) { Postorder.push_back(N); } + + /// Returns the number of leaves in the subtree. + int getSubtreePostorder(std::vector &Ids, NodeId Root) const; + std::vector getSubtreeBfs(NodeId Id) const; + int numberOfDescendants(NodeId Id) const; + + /// Returns the value of the node. + std::string label(NodeId Id) const; + void dumpTree(NodeId Id = NoNodeId) const; + std::string showNode(NodeId Id) const; + void dumpNodeAsJson(NodeId Id) const; + void dumpAsJson() const; +}; + +/// Maps nodes of the left tree to ones on the right, and vice versa. +// Supports fast insertion and lookup. +// TODO allow multiple mappings +class Mappings { + NodeId *SrcToDst, *DstToSrc; + const TreeRoot &T1, &T2; + +public: + Mappings(const TreeRoot &T1, const TreeRoot &T2); + ~Mappings(); + + void link(NodeId Src, NodeId Dst); + + NodeId getDst(NodeId Src) const { return SrcToDst[Src]; } + NodeId getSrc(NodeId Dst) const { return DstToSrc[Dst]; } + bool hasSrc(NodeId Src) const { return SrcToDst[Src] != NoNodeId; } + bool hasDst(NodeId Dst) const { return DstToSrc[Dst] != NoNodeId; } + + void dumpMapping() const; +}; + +/// (Update, src, dst, _ ): update the value of node src to match dst. +/// (Insert, src, dst, pos): insert src as child of dst at offset pos. +/// (Delete, src, _, _ ): delete node src. +/// (Move, src, dst, pos): move src to a child of dst at offset pos. +enum ChangeKind { Delete, Update, Insert, Move }; +using Change = std::tuple; + +void runDiff(ASTUnit &AST1, ASTUnit &AST2); + +} // namespace diff +} // namespace clang + +#endif Index: lib/Tooling/ASTDiff/ASTDiff.cpp =================================================================== --- /dev/null +++ lib/Tooling/ASTDiff/ASTDiff.cpp @@ -0,0 +1,865 @@ +//===- ASTDiff.cpp - AST differencing implementation-----------*- C++ -*- -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief +/// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/ASTDiff/ASTDiff.h" + +#include "clang/AST/RecursiveASTVisitor.h" +#include "llvm/ADT/PriorityQueue.h" +#include "llvm/Support/FormatVariadic.h" + +#include +#include +#include + +using namespace llvm; +using namespace clang; + +namespace clang { +namespace diff { + +static bool isRelevantNode(Decl *D) { return D != nullptr; } +static bool isRelevantNode(Stmt *S) { return S != nullptr; } + +namespace { +// Count the number of non-null nodes +struct NodeCountVisitor : public RecursiveASTVisitor { + int Count = 0; + bool TraverseDecl(Decl *D) { + if (isRelevantNode(D)) { + ++Count; + RecursiveASTVisitor::TraverseDecl(D); + } + return true; + } + bool TraverseStmt(Stmt *S) { + if (isRelevantNode(S)) { + ++Count; + RecursiveASTVisitor::TraverseStmt(S); + } + return true; + } + bool TraverseType(QualType T) { return true; } +}; +} // namespace + +using NodeMap = std::map; + +namespace { +// Initializes the nodes in the tree Root. +// +// Computes depth of each node as well as the maximum depth of the AST. +// +// Creates a mapping from AST nodes (DynTypedNode) to the postorder traversal +// number. This is stored in PostorderIdMap. +struct PostorderVisitor : public RecursiveASTVisitor { + int Id = 0, Depth = 0; + TreeRoot &Root; + NodeMap &PostorderIdMap; + PostorderVisitor(TreeRoot &Root, NodeMap &PostorderIdMap) + : Root(Root), PostorderIdMap(PostorderIdMap) {} + void PreTraverse() { ++Depth; } + template void PostTraverse(T *ASTNode) { + assert(ASTNode != nullptr); + --Depth; + Node &N = Root.getMutableNode(Id); + N.Parent = NoNodeId; + N.Depth = Depth; + N.ASTNode = ast_type_traits::DynTypedNode::create(*ASTNode); + assert(!N.ASTNode.getNodeKind().isNone()); + PostorderIdMap[N.ASTNode] = Id; + Root.MaxDepth = std::max(Root.MaxDepth, Depth); + ++Id; + } + bool TraverseDecl(Decl *D) { + if (isRelevantNode(D)) { + PreTraverse(); + RecursiveASTVisitor::TraverseDecl(D); + PostTraverse(D); + } + return true; + } + bool TraverseStmt(Stmt *S) { + if (isRelevantNode(S)) { + PreTraverse(); + RecursiveASTVisitor::TraverseStmt(S); + PostTraverse(S); + } + return true; + } + bool TraverseType(QualType T) { return true; } +}; +} // namespace + +namespace { +// Sets Height, Parent and Children for each node. +struct PreorderVisitor : public RecursiveASTVisitor { + NodeId Parent = -1; + TreeRoot &Root; + const NodeMap &PostorderIdMap; + PreorderVisitor(TreeRoot &Root, const NodeMap &PostorderIdMap) + : Root(Root), PostorderIdMap(PostorderIdMap) {} + template std::tuple PreTraverse(T *ASTNode) { + assert(ASTNode != nullptr); + auto DNode = ast_type_traits::DynTypedNode::create(*ASTNode); + assert(!DNode.getNodeKind().isNone()); + const NodeId &Id = PostorderIdMap.at(DNode); + Root.getMutableNode(Id).Parent = Parent; + if (Parent != NoNodeId) { + Node &P = Root.getMutableNode(Parent); + P.Children.push_back(Id); + } + Parent = Id; + return {Id, Root.getNode(Id).Parent}; + } + void PostTraverse(std::tuple State) { + NodeId Id, PreviousParent; + std::tie(Id, PreviousParent) = State; + Parent = PreviousParent; + if (Id != NoNodeId) { + Node &N = Root.getMutableNode(Id); + if (N.isLeaf()) { + Root.Leaves.push_back(Id); + } + N.Height = 1; + for (NodeId Child : N.Children) { + N.Height = std::max(N.Height, 1 + Root.getNode(Child).Height); + } + } + } + bool TraverseDecl(Decl *D) { + if (isRelevantNode(D)) { + auto SavedState = PreTraverse(D); + RecursiveASTVisitor::TraverseDecl(D); + PostTraverse(SavedState); + } + return true; + } + bool TraverseStmt(Stmt *S) { + if (isRelevantNode(S)) { + auto SavedState = PreTraverse(S); + RecursiveASTVisitor::TraverseStmt(S); + PostTraverse(SavedState); + } + return true; + } + bool TraverseType(QualType T) { return true; } +}; +} // namespace + +TreeRoot::TreeRoot(ASTUnit &AST) : AST(AST) { + auto *TUD = AST.getASTContext().getTranslationUnitDecl(); + // Run the above visitors to initialize the tree. + NodeCountVisitor NodeCounter; + NodeCounter.TraverseDecl(TUD); + setSize(NodeCounter.Count); + NodeMap PostorderIdMap; + PostorderVisitor PostorderWalker(*this, PostorderIdMap); + PostorderWalker.TraverseDecl(TUD); + PreorderVisitor PreorderWalker(*this, PostorderIdMap); + PreorderWalker.TraverseDecl(TUD); + setLeftMostDescendants(); +} + +int TreeRoot::getSubtreePostorder(std::vector &Ids, NodeId Root) const { + int Leaves = 0; + std::function Traverse = [&](NodeId Id) { + const Node &N = getNode(Id); + for (NodeId Child : N.Children) { + Traverse(Child); + } + if (N.isLeaf()) { + ++Leaves; + } + Ids.push_back(Id); + }; + Traverse(Root); + return Leaves; +} +std::vector TreeRoot::getSubtreeBfs(NodeId Id) const { + std::vector Ids; + size_t Expanded = 0; + Ids.push_back(Id); + while (Expanded < Ids.size()) { + for (NodeId Child : getNode(Ids[Expanded++]).Children) { + Ids.push_back(Child); + } + } + return Ids; +} + +int TreeRoot::numberOfDescendants(NodeId Id) const { + return Id - getNode(Id).LeftMostDescendant; +} + +std::string TreeRoot::label(NodeId Id) const { + assert(isValidNodeId(Id)); + const Node &N = getNode(Id); + const ast_type_traits::DynTypedNode &DTN = N.ASTNode; + if (auto *X = DTN.get()) { + return X->getName(); + } + if (auto *X = DTN.get()) { + return X->getOpcodeStr(); + } + if (auto *X = DTN.get()) { + auto Tmp = X->getValue(); + SmallString<256> Str; + Tmp.toString(Str, /*Radix=*/10, /*Signed=*/false); + return Str.str(); + } + if (auto *X = DTN.get()) { + return X->getString(); + } + if (auto *X = DTN.get()) { + return ""; + } + if (auto *X = DTN.get()) { + return ""; + } + if (auto *X = DTN.get()) { + return ""; + } + if (auto *X = DTN.get()) { + return ""; + } + if (auto *X = DTN.get()) { + return ""; + } + if (auto *X = DTN.get()) { + return ""; + } + llvm_unreachable("Fatal: unhandled AST node\n"); +} +void TreeRoot::dumpTree(NodeId Id) const { + if (Id == NoNodeId) { + Id = root(); + } + const Node &N = getNode(Id); + for (int I = 0; I < N.Depth; ++I) { + outs() << " "; + } + outs() << showNode(Id) << "\n"; + for (NodeId Child : N.Children) { + dumpTree(Child); + } +} +std::string TreeRoot::showNode(NodeId Id) const { + if (Id == NoNodeId) { + return "None"; + } + std::string Label; + if (label(Id) != "") { + Label = formatv(": {0}", label(Id)); + } + return formatv("{0}{1}({2})", getNode(Id).getTypeLabel(), Label, Id); +} +void TreeRoot::dumpNodeAsJson(NodeId Id) const { + auto N = getNode(Id); + std::string Label; + if (label(Id) != "") { + Label = formatv(R"(,"label":"{0}")", label(Id)); + } + outs() << formatv(R"({"typeLabel":"{0}"{1},"children":[)", N.getTypeLabel(), + Label); + if (N.Children.size() > 0) { + dumpNodeAsJson(N.Children[0]); + for (size_t I = 1; I < N.Children.size(); ++I) { + outs() << ","; + dumpNodeAsJson(N.Children[I]); + } + } + outs() << "]}"; +} +void TreeRoot::dumpAsJson() const { + outs() << R"({"root":)"; + dumpNodeAsJson(root()); + outs() << "}\n"; +} + +void TreeRoot::setLeftMostDescendants() { + for (NodeId Id : Leaves) { + Node &N = getMutableNode(Id); + N.LeftMostDescendant = Id; + NodeId Cur = Id; + while (N.Parent != NoNodeId && getNode(N.Parent).Children[0] == Cur) { + Cur = N.Parent; + getMutableNode(Cur).LeftMostDescendant = Id; + } + } +} + +Mappings::Mappings(const TreeRoot &T1, const TreeRoot &T2) : T1(T1), T2(T2) { + int Size = T1.getSize() + T2.getSize(); + SrcToDst = new NodeId[Size]; + DstToSrc = new NodeId[Size]; + // set everything to NoNodeId == -1 + memset(SrcToDst, NoNodeId, Size * sizeof(*SrcToDst)); + memset(DstToSrc, NoNodeId, Size * sizeof(*DstToSrc)); +} +Mappings::~Mappings() { + delete[] SrcToDst; + delete[] DstToSrc; +} +void Mappings::link(NodeId Src, NodeId Dst) { + SrcToDst[Src] = Dst; + DstToSrc[Dst] = Src; +} + +void Mappings::dumpMapping() const { + for (NodeId Id1 = 0, Id2; Id1 < T1.getSize(); ++Id1) { + if ((Id2 = getDst(Id1)) != NoNodeId) { + outs() << formatv("Match {0} to {1}\n", T1.showNode(Id1), + T2.showNode(Id2)); + } + } +} + +class Subtree { + using SNodeId = int; + const TreeRoot &Tree; + std::vector OriginalIds; + std::vector LeftMostDescendants; + int LeafCount; + +public: + std::vector KeyRoots; + + Subtree(const TreeRoot &Tree, NodeId RootId) : Tree(Tree) { + LeafCount = Tree.getSubtreePostorder(OriginalIds, RootId); + computeLeftMostDescendants(); + computeKeyRoots(); + } + + int getSizeS() const { return OriginalIds.size(); } + NodeId getOriginalId(SNodeId Id) const { + assert(Id > 0 && Id <= getSizeS()); + return OriginalIds[Id - 1]; + } + const Node &getNodeS(SNodeId Id) const { + return Tree.getNode(getOriginalId(Id)); + } + const std::string labelS(SNodeId Id) const { + return Tree.label(getOriginalId(Id)); + } + SNodeId getLeftMostDescendant(SNodeId Id) const { + assert(Id >= 0 && Id < getSizeS()); + return LeftMostDescendants[Id]; + } + +private: + void computeLeftMostDescendants() { + LeftMostDescendants.resize(getSizeS()); + for (SNodeId I = 0; I < getSizeS(); ++I) { + NodeId Cur = getOriginalId(I + 1); + while (!Tree.getNode(Cur).isLeaf()) { + Cur = Tree.getNode(Cur).Children[0]; + } + LeftMostDescendants[I] = (Cur - getOriginalId(1)); + } + } + void computeKeyRoots() { + KeyRoots.resize(LeafCount); + std::unordered_set Visited; + int K = LeafCount - 1; + for (int I = getSizeS(); I > 0; --I) { + SNodeId LeftDesc = getLeftMostDescendant(I - 1); + if (Visited.count(LeftDesc) != 0u) { + continue; + } + assert(K >= 0); + KeyRoots[K] = I; + Visited.insert(LeftDesc); + --K; + } + } +}; + +// Computes an optimal mapping between two trees +class ZsMatcher { + using Pairs = std::vector>; + +private: + Subtree S1; + Subtree S2; + double **TreeDist; + double **ForestDist; + + // TODO Use string editing distance if applicable. + double getUpdateCost(NodeId Id1, NodeId Id2) { + if (!S1.getNodeS(Id1).hasSameType(S2.getNodeS(Id2))) { + return std::numeric_limits::max(); + } + if (S1.labelS(Id1) == S2.labelS(Id2)) { + return 0; + } + return 1; + } + void computeTreeDist() { + for (NodeId Id1 : S1.KeyRoots) { + for (NodeId Id2 : S2.KeyRoots) { + computeForestDist(Id1, Id2); + } + } + } + void dumpForestDist() const { + for (int I = 0; I < S1.getSizeS() + 1; ++I) { + outs() << "["; + for (int J = 0; J < S2.getSizeS() + 1; ++J) { + outs() << formatv("{0}{1: 2}", (J == 0 ? "" : " "), ForestDist[I][J]); + } + outs() << "]\n"; + } + } + void computeForestDist(NodeId Id1, NodeId Id2) { + assert(Id1 > 0 && Id2 > 0); + NodeId LMD1 = S1.getLeftMostDescendant(Id1 - 1); + NodeId LMD2 = S2.getLeftMostDescendant(Id2 - 1); + + ForestDist[LMD1][LMD2] = 0; + for (NodeId D1 = LMD1 + 1; D1 <= Id1; ++D1) { + double DeletionCost = 1.0; + ForestDist[D1][LMD2] = ForestDist[D1 - 1][LMD2] + DeletionCost; + for (NodeId D2 = LMD2 + 1; D2 <= Id2; ++D2) { + double InsertionCost = 1; + ForestDist[LMD1][D2] = ForestDist[LMD1][D2 - 1] + InsertionCost; + NodeId DLMD1 = S1.getLeftMostDescendant(D1 - 1); + NodeId DLMD2 = S2.getLeftMostDescendant(D2 - 1); + if (DLMD1 == LMD1 && DLMD2 == LMD2) { + double UpdateCost = getUpdateCost(D1, D2); + ForestDist[D1][D2] = + std::min(std::min(ForestDist[D1 - 1][D2] + DeletionCost, + ForestDist[D1][D2 - 1] + InsertionCost), + ForestDist[D1 - 1][D2 - 1] + UpdateCost); + TreeDist[D1][D2] = ForestDist[D1][D2]; + } else { + ForestDist[D1][D2] = + std::min(std::min(ForestDist[D1 - 1][D2] + DeletionCost, + ForestDist[D1][D2 - 1] + InsertionCost), + ForestDist[DLMD1][DLMD2] + TreeDist[D1][D2]); + } + } + } + } + +public: + ZsMatcher(const TreeRoot &T1, const TreeRoot &T2, NodeId Id1, NodeId Id2) + : S1(T1, Id1), S2(T2, Id2) { + TreeDist = new double *[S1.getSizeS() + 1]; + ForestDist = new double *[S1.getSizeS() + 1]; + for (int I = 0; I < S1.getSizeS() + 1; ++I) { + TreeDist[I] = new double[S2.getSizeS() + 1](); + ForestDist[I] = new double[S2.getSizeS() + 1](); + } + } + + ~ZsMatcher() { + for (int I = 0; I < S1.getSizeS() + 1; ++I) { + delete[] TreeDist[I]; + delete[] ForestDist[I]; + } + delete[] TreeDist; + delete[] ForestDist; + } + + Pairs match() { + Pairs Matches; + + computeTreeDist(); + + bool RootNodePair = true; + Pairs TreePairs{{S1.getSizeS(), S2.getSizeS()}}; + + NodeId LastRow, LastCol, FirstRow, FirstCol, Row, Col; + while (!TreePairs.empty()) { + std::tie(LastRow, LastCol) = TreePairs.back(); + TreePairs.pop_back(); + + if (!RootNodePair) { + computeForestDist(LastRow, LastCol); + } + + RootNodePair = false; + + FirstRow = S1.getLeftMostDescendant(LastRow - 1); + FirstCol = S2.getLeftMostDescendant(LastCol - 1); + + Row = LastRow; + Col = LastCol; + + while (Row > FirstRow || Col > FirstCol) { + if (Row > FirstRow && + ForestDist[Row - 1][Col] + 1 == ForestDist[Row][Col]) { + --Row; + } else if (Col > FirstCol && + ForestDist[Row][Col - 1] + 1 == ForestDist[Row][Col]) { + --Col; + } else { + NodeId LMD1 = S1.getLeftMostDescendant(Row - 1); + NodeId LMD2 = S2.getLeftMostDescendant(Col - 1); + if (LMD1 == S1.getLeftMostDescendant(LastRow - 1) && + LMD2 == S2.getLeftMostDescendant(LastCol - 1)) { + assert(S1.getNodeS(Row).hasSameType(S2.getNodeS(Col))); + Matches.emplace_back(S1.getOriginalId(Row), S2.getOriginalId(Col)); + --Row; + --Col; + } else { + TreePairs.emplace_back(Row, Col); + Row = LMD1; + Col = LMD2; + } + } + } + } + return Matches; + } +}; + +namespace { +// Compares nodes by their depth. +struct HeightLess { + const TreeRoot &Tree; + HeightLess(const TreeRoot &Tree) : Tree(Tree) {} + bool operator()(NodeId Id1, NodeId Id2) const { + return Tree.getNode(Id1).Height < Tree.getNode(Id2).Height; + } +}; +} // namespace + +// Priority queue for nodes, sorted descendingly by their height. +class PriorityList { + const TreeRoot &Tree; + HeightLess Comparator; + std::vector Container; + PriorityQueue, HeightLess> List; + +public: + PriorityList(const TreeRoot &Tree) + : Tree(Tree), Comparator(Tree), List(Comparator, Container) {} + + void push(NodeId id) { List.push(id); } + + std::vector pop() { + int Max = peekMax(); + std::vector Result; + if (Max != 0) { + while (peekMax() == Max) { + Result.push_back(List.top()); + List.pop(); + } + } + // TODO this is here to get a stable output, not a good heuristic + std::sort(Result.begin(), Result.end()); + return Result; + } + + int peekMax() const { + if (List.empty()) { + return 0; + } + return Tree.getNode(List.top()).Height; + } + + void open(NodeId Id) { + for (NodeId Child : Tree.getNode(Id).Children) { + push(Child); + } + } +}; + +/// Computes the differences between two ASTs +class ClangDiff { +private: + TreeRoot &T1; + TreeRoot &T2; + + // Returns true if the two subtrees have the same structure + // and equal node kinds (does not compare labels). + bool isomorphic(NodeId Id1, NodeId Id2) const { + const Node &N1 = T1.getNode(Id1); + const Node &N2 = T2.getNode(Id2); + if (!N1.hasSameType(N2) || N1.Children.size() != N2.Children.size() || + T1.label(Id1) != T2.label(Id2)) { + return false; + } + for (size_t Id = 0; Id < N1.Children.size(); ++Id) { + if (!isomorphic(N1.Children[Id], N2.Children[Id])) { + return false; + } + } + return true; + } + + // TODO This is too restrictive, we want to allow multiple mapping candidates + // for nodes and resolve the ambiguity later. + bool isMappingAllowed(const Mappings &M, NodeId Id1, NodeId Id2) const { + const Node &N1 = T1.getNode(Id1); + const Node &N2 = T2.getNode(Id2); + bool AnyMapped = M.hasSrc(Id1) || M.hasDst(Id2); + bool SameType = N1.hasSameType(N2); + NodeId P1 = N1.Parent; + NodeId P2 = N2.Parent; + bool ParentsSameType = (P1 == NoNodeId && P2 == NoNodeId) || + (P1 != NoNodeId && P2 != NoNodeId && + T1.getNode(P1).hasSameType(T2.getNode(P2))); + return !AnyMapped && SameType && ParentsSameType; + } + + // Adds all corresponding subtrees of the two nodes to the mappings. + // The two nodes must be isomorphic. + void addIsomorphicSubTrees(Mappings &M, NodeId Id1, NodeId Id2) const { + M.link(Id1, Id2); + const Node &N1 = T1.getNode(Id1); + const Node &N2 = T2.getNode(Id2); + assert(isomorphic(Id1, Id2)); + assert(N1.Children.size() == N2.Children.size()); + for (size_t Id = 0; Id < N1.Children.size(); ++Id) { + addIsomorphicSubTrees(M, N1.Children[Id], N2.Children[Id]); + } + } + + // Uses an optimal albeit slow algorithm to compute mappings for two + // subtrees, but only if both have fewer nodes than MaxSize. + void addOptimalMappings(Mappings &M, NodeId Id1, NodeId Id2) const { + if (std::max(T1.numberOfDescendants(Id1), T2.numberOfDescendants(Id2)) < + MaxSize) { + ZsMatcher Matcher(T1, T2, Id1, Id2); + std::vector> R = Matcher.match(); + for (const auto Tuple : R) { + NodeId Src = Tuple.first; + NodeId Dst = Tuple.second; + if (isMappingAllowed(M, Src, Dst)) { + M.link(Src, Dst); + } + } + } + } + + // Computes the ratio of common descendants between the two nodes. + // Descendants are only considered to be equal when they are mapped in M. + double dice(const Mappings &M, NodeId Id1, NodeId Id2) const { + if (Id1 == NoNodeId || Id2 == NoNodeId) { + return 0.0; + } + int CommonDescendants = 0; + const Node &N1 = T1.getNode(Id1); + for (NodeId Id = N1.LeftMostDescendant; Id < Id1; ++Id) { + CommonDescendants += int(M.hasSrc(Id)); + } + return 2.0 * CommonDescendants / + (T1.numberOfDescendants(Id1) + T2.numberOfDescendants(Id2)); + } + + // Returns the node that has the highest degree of similarity. + NodeId findCandidate(const Mappings &M, NodeId Id1) const { + NodeId Candidate = NoNodeId; + double BestDiceValue = 0.0; + const Node &N1 = T1.getNode(Id1); + for (NodeId Id2 = 0; Id2 < T2.getSize(); ++Id2) { + const Node &N2 = T2.getNode(Id2); + if (!N1.hasSameType(N2)) { + continue; + } + if (M.hasDst(Id2)) { + continue; + } + double DiceValue = dice(M, Id1, Id2); + if (DiceValue > BestDiceValue) { + BestDiceValue = DiceValue; + Candidate = Id2; + } + } + return Candidate; + } + + // Tries to match any yet unmapped nodes, in a bottom-up fashion. + void matchBottomUp(Mappings &M) const { + for (NodeId Id1 = 0; Id1 < T1.getSize(); ++Id1) { + if (Id1 == T1.root()) { + M.link(T1.root(), T2.root()); + addOptimalMappings(M, T1.root(), T2.root()); + break; + } + assert(Id1 != NoNodeId); + const Node &N1 = T1.getNode(Id1); + bool Matched = M.hasSrc(Id1); + bool MatchedChildren = + std::any_of(N1.Children.begin(), N1.Children.end(), + [&](NodeId Child) { return M.hasSrc(Child); }); + if (Matched || !MatchedChildren) { + continue; + } + NodeId Id2 = findCandidate(M, Id1); + if (Id2 == NoNodeId || dice(M, Id1, Id2) < MinDice || + !isMappingAllowed(M, Id1, Id2)) { + continue; + } + M.link(Id1, Id2); + addOptimalMappings(M, Id1, Id2); + } + } + + // Returns a set of mappings of isomorphic subtrees. + Mappings matchTopDown() const { + PriorityList L1(T1); + PriorityList L2(T2); + + Mappings M(T1, T2); + + L1.push(T1.root()); + L2.push(T2.root()); + + int Max1, Max2; + while (std::min(Max1 = L1.peekMax(), Max2 = L2.peekMax()) > MinHeight) { + if (Max1 > Max2) { + for (NodeId Id : L1.pop()) { + L1.open(Id); + } + continue; + } + if (Max2 > Max1) { + for (NodeId Id : L2.pop()) { + L2.open(Id); + } + continue; + } + std::vector H1, H2; + H1 = L1.pop(); + H2 = L2.pop(); + for (NodeId Id1 : H1) { + for (NodeId Id2 : H2) { + if (isomorphic(Id1, Id2) && isMappingAllowed(M, Id1, Id2)) { + addIsomorphicSubTrees(M, Id1, Id2); + } + } + } + for (NodeId Id1 : H1) { + if (!M.hasSrc(Id1)) { + L1.open(Id1); + } + } + for (NodeId Id2 : H2) { + if (!M.hasDst(Id2)) { + L2.open(Id2); + } + } + } + return M; + } + + // Finds an edit script that converts T1 to T2. + std::vector computeChanges(Mappings &M) { + std::vector Changes; + for (NodeId Id2 : T2.getSubtreeBfs(T2.root())) { + const Node &N2 = T2.getNode(Id2); + NodeId Id1 = M.getSrc(Id2); + if (Id1 != NoNodeId) { + assert(T1.getNode(Id1).hasSameType(N2)); + if (T1.label(Id1) != T2.label(Id2)) { + Changes.emplace_back(Update, Id1, Id2, /*UNUSED Position=*/0); + } + continue; + } + NodeId P2 = N2.Parent; + NodeId P1 = M.getSrc(P2); + assert(P1 != NoNodeId); + Node &Parent1 = T1.getMutableNode(P1); + const Node &Parent2 = T2.getNode(P2); + auto &Siblings1 = Parent1.Children; + const auto &Siblings2 = Parent2.Children; + size_t Position; + for (Position = 0; Position < Siblings2.size(); ++Position) { + if (Siblings2[Position] == Id2 || Position >= Siblings1.size()) { + break; + } + } + Changes.emplace_back(Insert, Id2, P2, Position); + Node PatchNode; + PatchNode.Parent = P1; + PatchNode.LeftMostDescendant = N2.LeftMostDescendant; + PatchNode.Depth = N2.Depth; + PatchNode.ASTNode = N2.ASTNode; + // TODO update Depth if needed + NodeId PatchNodeId = T1.getSize(); + assert(Position <= Siblings1.size()); + // TODO maybe choose a different data structure for Children. + Siblings1.insert(Siblings1.begin() + Position, PatchNodeId); + T1.addNode(PatchNode); + M.link(PatchNodeId, Id2); + } + for (NodeId Id1 = 0; Id1 < T1.getSize(); ++Id1) { + NodeId Id2 = M.getDst(Id1); + if (Id2 == NoNodeId) { + Changes.emplace_back(Delete, Id1, Id2, /*UNUSED Position=*/0); + } + } + return Changes; + } + + // Prints an edit action. + void dumpChange(const Change &Chg) const { + ChangeKind Kind; + NodeId Id1, Id2; + size_t Position; + std::tie(Kind, Id1, Id2, Position) = Chg; + std::string S; + switch (Kind) { + case Delete: + S = formatv("Delete {0}", T1.showNode(Id1)); + break; + case Update: + S = formatv("Update {0} to {1}", T1.showNode(Id1), T2.label(Id2)); + break; + case Insert: + S = formatv("Insert {0} into {1} at {2}", T2.showNode(Id1), + T2.showNode(Id2), Position); + break; + case Move: + llvm_unreachable("TODO"); + break; + }; + outs() << S << "\n"; + } + + Mappings match() const { + Mappings M = matchTopDown(); + matchBottomUp(M); + return M; + } + +public: + int MinHeight = 2; + double MinDice = 0.2; + int MaxSize = 100; + + ClangDiff(TreeRoot &T1, TreeRoot &T2) : T1(T1), T2(T2) {} + + void printDiff() { + Mappings M = match(); + M.dumpMapping(); + auto Changes = computeChanges(M); + for (const auto &C : Changes) { + dumpChange(C); + } + } +}; + +void runDiff(ASTUnit &AST1, ASTUnit &AST2) { + TreeRoot T1(AST1); + TreeRoot T2(AST2); + ClangDiff DiffTool(T1, T2); + DiffTool.printDiff(); +} + +} // namespace diff +} // namespace clang Index: lib/Tooling/ASTDiff/CMakeLists.txt =================================================================== --- /dev/null +++ lib/Tooling/ASTDiff/CMakeLists.txt @@ -0,0 +1,12 @@ +set(LLVM_LINK_COMPONENTS + # Option + Support + ) + +add_clang_library(clangToolingASTDiff + ASTDiff.cpp + LINK_LIBS + clangBasic + clangAST + # clangToolingCore + ) Index: lib/Tooling/CMakeLists.txt =================================================================== --- lib/Tooling/CMakeLists.txt +++ lib/Tooling/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(Core) add_subdirectory(Refactoring) +add_subdirectory(ASTDiff) add_clang_library(clangTooling ArgumentsAdjusters.cpp Index: tools/CMakeLists.txt =================================================================== --- tools/CMakeLists.txt +++ tools/CMakeLists.txt @@ -2,6 +2,7 @@ add_clang_subdirectory(diagtool) add_clang_subdirectory(driver) +add_clang_subdirectory(clang-diff) add_clang_subdirectory(clang-format) add_clang_subdirectory(clang-format-vs) add_clang_subdirectory(clang-fuzzer) Index: tools/clang-diff/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-diff/CMakeLists.txt @@ -0,0 +1,16 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_executable(clang-diff + ClangDiff.cpp + ) + +target_link_libraries(clang-diff + clangAST + clangBasic + clangFrontend + clangLex + clangTooling + clangToolingASTDiff + ) Index: tools/clang-diff/ClangDiff.cpp =================================================================== --- /dev/null +++ tools/clang-diff/ClangDiff.cpp @@ -0,0 +1,58 @@ +//===- ClangDiff.cpp - compare source files by AST nodes ------*- C++ -*- -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief +/// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/ASTDiff/ASTDiff.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/CommandLine.h" + +static cl::OptionCategory ClangDiffCategory("clang-diff options"); + +static cl::opt + DumpAST("ast-dump", + cl::desc("Print the internal representation of the AST as JSON."), + cl::init(false), cl::cat(ClangDiffCategory)); + +int main(int argc, const char **argv) { + tooling::CommonOptionsParser Options(argc, argv, ClangDiffCategory); + ArrayRef Files = Options.getSourcePathList(); + + tooling::ClangTool Tool(Options.getCompilations(), Files); + + if (DumpAST) { + if (Files.size() != 1) { + errs() << "Error: please specify exactly one filename.\n"; + return 1; + } + std::vector> ASTs; + Tool.buildASTs(ASTs); + if (ASTs.size() != 1) { + return 1; + } + clang::diff::TreeRoot Tree(*ASTs[0]); + Tree.dumpAsJson(); + return 0; + } + if (Files.size() != 2) { + errs() << "Error: exactly two filenames are required.\n"; + return 1; + } + std::vector> ASTs; + Tool.buildASTs(ASTs); + if (ASTs.size() != 2) { + return 1; + } + clang::diff::runDiff(*ASTs[0], *ASTs[1]); + return 0; +}