[clang-diff] Move the JSON export function to clang-diff

[arphaman]

include/clang/Tooling/ASTDiff/ASTDiff.h

105

Please add a documentation comment that mentions what kind of value is returned by the function. getSourceRangeOffsets might be a more a descriptive name as well.

[johannes]

getSourceRangeOffsets, test

[arphaman]

test/Tooling/clang-diff-json.cpp

2

I think you have to use %python instead of python, like LLVM tests do.

[johannes]

test/Tooling/clang-diff-json.cpp

2

ok
So I have to use python 2.7? (for the test in https://reviews.llvm.org/D36182)

[arphaman]

test/Tooling/clang-diff-json.cpp

2

Yeah, 2.7 support is still a requirement.

[johannes]

test/Tooling/clang-diff-json.cpp

2

It looks like %python is expanded in the wrong way.
I get the same as when using %Sython, the directory of the test case is substituted

[arphaman]

test/Tooling/clang-diff-json.cpp

2

It might not be in the lit.cfg for Clang, let me check.

[arphaman]

@johannes , you have to add %python substitution to lit.cfg, see my attached patch that does it.

python.diff1 KBDownload

[johannes]

use %python in tests

[arphaman]

LGTM

include/clang/Tooling/ASTDiff/ASTDiff.h

 
 /// Represents a Clang AST node, alongside some additional information.
 struct Node {
   NodeId Parent, LeftMostDescendant, RightMostDescendant;
   int Depth, Height;
   ast_type_traits::DynTypedNode ASTNode;
   SmallVector<NodeId, 4> Children;
 
-  ast_type_traits::ASTNodeKind getType() const { return ASTNode.getNodeKind(); }
-  const StringRef getTypeLabel() const { return getType().asStringRef(); }
+  ast_type_traits::ASTNodeKind getType() const;
+  StringRef getTypeLabel() const;
   bool isLeaf() const { return Children.empty(); }
 };
 
 class ASTDiff {
 public:
   ASTDiff(SyntaxTree &Src, SyntaxTree &Dst, const ComparisonOptions &Options);
   ~ASTDiff();
 
public:
   SyntaxTree(const ASTContext &AST);
   /// Constructs a tree from any AST node.
   template <class T>
   SyntaxTree(T *Node, const ASTContext &AST)
       : TreeImpl(llvm::make_unique<Impl>(this, Node, AST)) {}
   SyntaxTree(SyntaxTree &&Other) = default;
   ~SyntaxTree();
 
+  const ASTContext &getASTContext() const;
+  StringRef getFilename() const;
+
   const Node &getNode(NodeId Id) const;
+  NodeId getRootId() const;
+
+  // Returns the starting and ending offset of the node in its source file.
+  std::pair<unsigned, unsigned> getSourceRangeOffsets(const Node &N) const;
 
   /// Serialize the node attributes to a string representation. This should
   /// uniquely distinguish nodes of the same kind. Note that this function just
   /// returns a representation of the node value, not considering descendants.
   std::string getNodeValue(const DynTypedNode &DTN) const;
 
-  void printAsJson(raw_ostream &OS);
-
   class Impl;
   std::unique_ptr<Impl> TreeImpl;
 };
 
 struct ComparisonOptions {
   /// During top-down matching, only consider nodes of at least this height.
   int MinHeight = 2;
 

lib/Tooling/ASTDiff/ASTDiff.cpp

public:
   /// Prints the node as "<type>[: <value>](<postorder-id)"
   void printNode(NodeId Id) const { printNode(llvm::outs(), Id); }
   void printNode(raw_ostream &OS, NodeId Id) const;
 
   void printTree() const;
   void printTree(NodeId Root) const;
   void printTree(raw_ostream &OS, NodeId Root) const;
 
-  void printAsJsonImpl(raw_ostream &OS) const;
-  void printNodeAsJson(raw_ostream &OS, NodeId Id) const;
-
 private:
   /// Nodes in preorder.
   std::vector<Node> Nodes;
 
   void initTree();
   void setLeftMostDescendants();
 };
 
if (Id.isInvalid()) {
     return;
   }
   OS << getNode(Id).getTypeLabel();
   if (getNodeValue(Id) != "")
     OS << ": " << getNodeValue(Id);
   OS << "(" << PostorderIds[Id] << ")";
 }
 
-void SyntaxTree::Impl::printNodeAsJson(raw_ostream &OS, NodeId Id) const {
-  auto N = getNode(Id);
-  OS << R"({"type":")" << N.getTypeLabel() << R"(")";
-  if (getNodeValue(Id) != "")
-    OS << R"(,"value":")" << getNodeValue(Id) << R"(")";
-  OS << R"(,"children":[)";
-  if (N.Children.size() > 0) {
-    printNodeAsJson(OS, N.Children[0]);
-    for (size_t I = 1, E = N.Children.size(); I < E; ++I) {
-      OS << ",";
-      printNodeAsJson(OS, N.Children[I]);
-    }
-  }
-  OS << "]}";
-}
-
-void SyntaxTree::Impl::printAsJsonImpl(raw_ostream &OS) const {
-  OS << R"({"root":)";
-  printNodeAsJson(OS, getRootId());
-  OS << "}\n";
-}
-
 /// Identifies a node in a subtree by its postorder offset, starting at 1.
 struct SNodeId {
   int Id = 0;
 
   explicit SNodeId(int Id) : Id(Id) {}
   explicit SNodeId() = default;
 
   operator int() const { return Id; }
for (SNodeId D1 = LMD1 + 1; D1 <= Id1; ++D1) {
                         ForestDist[D1][D2 - 1] + InsertionCost,
                         ForestDist[DLMD1][DLMD2] + TreeDist[D1][D2]});
         }
       }
     }
   }
 };
 
+ast_type_traits::ASTNodeKind Node::getType() const {
+  return ASTNode.getNodeKind();
+}
+
+StringRef Node::getTypeLabel() const { return getType().asStringRef(); }
+
 namespace {
 // Compares nodes by their depth.
 struct HeightLess {
   const SyntaxTree::Impl &Tree;
   HeightLess(const SyntaxTree::Impl &Tree) : Tree(Tree) {}
   bool operator()(NodeId Id1, NodeId Id2) const {
     return Tree.getNode(Id1).Height < Tree.getNode(Id2).Height;
   }

 }
 
 void ASTDiff::printMatch(raw_ostream &OS, const Match &M) const {
   DiffImpl->printMatchImpl(OS, M);
 }
 
 SyntaxTree::~SyntaxTree() = default;
 
-void SyntaxTree::printAsJson(raw_ostream &OS) { TreeImpl->printAsJsonImpl(OS); }
+const ASTContext &SyntaxTree::getASTContext() const { return TreeImpl->AST; }
+
+const Node &SyntaxTree::getNode(NodeId Id) const {
+  return TreeImpl->getNode(Id);
+}
+
+NodeId SyntaxTree::getRootId() const { return TreeImpl->getRootId(); }
+
+std::pair<unsigned, unsigned> SyntaxTree::getSourceRangeOffsets(const Node &N) const {
+  const SourceManager &SrcMgr = TreeImpl->AST.getSourceManager();
+  SourceRange Range = N.ASTNode.getSourceRange();
+  SourceLocation BeginLoc = Range.getBegin();
+  SourceLocation EndLoc = Lexer::getLocForEndOfToken(
+      Range.getEnd(), /*Offset=*/0, SrcMgr, TreeImpl->AST.getLangOpts());
+  if (auto *ThisExpr = N.ASTNode.get<CXXThisExpr>()) {
+    if (ThisExpr->isImplicit())
+      EndLoc = BeginLoc;
+  }
+  unsigned Begin = SrcMgr.getFileOffset(SrcMgr.getExpansionLoc(BeginLoc));
+  unsigned End = SrcMgr.getFileOffset(SrcMgr.getExpansionLoc(EndLoc));
+  return {Begin, End};
+}
 
 std::string SyntaxTree::getNodeValue(const DynTypedNode &DTN) const {
   return TreeImpl->getNodeValue(DTN);
 }
 
 } // end namespace diff
 } // end namespace clang

test/Tooling/clang-diff-json.cpp

+// RUN: clang-diff -ast-dump %s -- \
+// RUN: | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
+// RUN: | FileCheck %s
+
+// CHECK: "begin": 294,
+// CHECK: "type": "CXXRecordDecl",
+// CHECK: "type": "FieldDecl",
+// CHECK: "end": 314,
+class A {
+  int x;
+};
+
+// CHECK: "children": [
+// CHECK-NEXT: {
+// CHECK-NEXT: "begin":
+// CHECK-NEXT: "children": []
+// CHECK-NEXT: "end":
+// CHECK-NEXT: "id":
+// CHECK-NEXT: "type": "CharacterLiteral"
+// CHECK-NEXT: }
+// CHECK: ]
+// CHECK: "type": "VarDecl",
+char nl = '\n';
+
+// CHECK: "value": "abc \n\t\u0000\u001f\u0123 \ub370\ubc15"
+char s[] = "abc \n\t\0\x1f\u0123 데박";
+

test/lit.cfg

config.substitutions.append( ('%clang_cl', ' ' + config.clang +
                               ' --driver-mode=cl '))
 config.substitutions.append( ('%clangxx', ' ' + config.clang +
                               ' --driver-mode=g++ '))
 config.substitutions.append( ('%clang', ' ' + config.clang + ' ') )
 config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') )
 config.substitutions.append( ('%itanium_abi_triple', makeItaniumABITriple(config.target_triple)) )
 config.substitutions.append( ('%ms_abi_triple', makeMSABITriple(config.target_triple)) )
 config.substitutions.append( ('%resource_dir', getClangBuiltinIncludeDir(config.clang)) )
+config.substitutions.append( ('%python', config.python_executable) )
 
 # The host triple might not be set, at least if we're compiling clang from
 # an already installed llvm.
 if config.host_triple and config.host_triple != '@LLVM_HOST_TRIPLE@':
     config.substitutions.append( ('%target_itanium_abi_host_triple', '--target=%s' % makeItaniumABITriple(config.host_triple)) )
 else:
     config.substitutions.append( ('%target_itanium_abi_host_triple', '') )
 

test/lit.site.cfg.in

 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 config.clang_staticanalyzer_z3 = "@CLANG_ANALYZER_WITH_Z3@"
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
 config.enable_backtrace = @ENABLE_BACKTRACES@
 config.host_arch = "@HOST_ARCH@"
 config.enable_abi_breaking_checks = "@LLVM_ENABLE_ABI_BREAKING_CHECKS@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
 
 # Support substitution of the tools and libs dirs with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
 try:
     config.clang_tools_dir = config.clang_tools_dir % lit_config.params
     config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
     config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
     config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
 except KeyError:
     e = sys.exc_info()[1]
     key, = e.args
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg")

tools/clang-diff/ClangDiff.cpp

getAST(const std::unique_ptr<CompilationDatabase> &CommonCompilations,
   ClangTool Tool(Compilations ? *Compilations : *CommonCompilations, Files);
   std::vector<std::unique_ptr<ASTUnit>> ASTs;
   Tool.buildASTs(ASTs);
   if (ASTs.size() != Files.size())
     return nullptr;
   return std::move(ASTs[0]);
 }
 
+static char hexdigit(int N) { return N &= 0xf, N + (N < 10 ? '0' : 'a' - 10); }
+
+static void printJsonString(raw_ostream &OS, const StringRef Str) {
+  for (char C : Str) {
+    switch (C) {
+    case '"':
+      OS << R"(\")";
+      break;
+    case '\\':
+      OS << R"(\\)";
+      break;
+    case '\n':
+      OS << R"(\n)";
+      break;
+    case '\t':
+      OS << R"(\t)";
+      break;
+    default:
+      if ('\x00' <= C && C <= '\x1f') {
+        OS << R"(\u00)" << hexdigit(C >> 4) << hexdigit(C);
+      } else {
+        OS << C;
+      }
+    }
+  }
+}
+
+static void printNodeAttributes(raw_ostream &OS, diff::SyntaxTree &Tree,
+                                diff::NodeId Id) {
+  const diff::Node &N = Tree.getNode(Id);
+  OS << R"("id":)" << int(Id);
+  OS << R"(,"type":")" << N.getTypeLabel() << '"';
+  auto Offsets = Tree.getSourceRangeOffsets(N);
+  OS << R"(,"begin":)" << Offsets.first;
+  OS << R"(,"end":)" << Offsets.second;
+  std::string Value = Tree.getNodeValue(N.ASTNode);
+  if (!Value.empty()) {
+    OS << R"(,"value":")";
+    printJsonString(OS, Value);
+    OS << '"';
+  }
+}
+
+static void printNodeAsJson(raw_ostream &OS, diff::SyntaxTree &Tree,
+                            diff::NodeId Id) {
+  const diff::Node &N = Tree.getNode(Id);
+  OS << "{";
+  printNodeAttributes(OS, Tree, Id);
+  OS << R"(,"children":[)";
+  if (N.Children.size() > 0) {
+    printNodeAsJson(OS, Tree, N.Children[0]);
+    for (size_t I = 1, E = N.Children.size(); I < E; ++I) {
+      OS << ",";
+      printNodeAsJson(OS, Tree, N.Children[I]);
+    }
+  }
+  OS << "]}";
+}
+
 int main(int argc, const char **argv) {
   std::string ErrorMessage;
   std::unique_ptr<CompilationDatabase> CommonCompilations =
       FixedCompilationDatabase::loadFromCommandLine(argc, argv, ErrorMessage);
   if (!CommonCompilations && !ErrorMessage.empty())
     llvm::errs() << ErrorMessage;
   cl::HideUnrelatedOptions(ClangDiffCategory);
   if (!cl::ParseCommandLineOptions(argc, argv)) {
     cl::PrintOptionValues();
     return 1;
   }
 
   addExtraArgs(CommonCompilations);
 
   if (ASTDump) {
     if (!DestinationPath.empty()) {
       llvm::errs() << "Error: Please specify exactly one filename.\n";
       return 1;
     }
     std::unique_ptr<ASTUnit> AST = getAST(CommonCompilations, SourcePath);
     if (!AST)
       return 1;
     diff::SyntaxTree Tree(AST->getASTContext());
-    Tree.printAsJson(llvm::outs());
+    llvm::outs() << R"({"filename":")";
+    printJsonString(llvm::outs(), SourcePath);
+    llvm::outs() << R"(","root":)";
+    printNodeAsJson(llvm::outs(), Tree, Tree.getRootId());
+    llvm::outs() << "}\n";
     return 0;
   }
 
   if (DestinationPath.empty()) {
     llvm::errs() << "Error: Exactly two paths are required.\n";
     return 1;
   }