Add JSON output option to llvm-remark-size-diff

[paquette]

Summary

This adds JSON output to llvm-remark-size-diff.

The goal here is to make it easy for external tools to consume output from llvm-remark-size-diff. These tools could be used for automated size analysis. (E.g. in CI).

To specify JSON output, use --report_style=json. JSON output can be pretty-printed via --pretty.

With automation in mind, the schema looks like this:

"Files": {
    "A": <filename_a>
    "B": <filename_b>
},

"InBoth": [
    {
    "FunctionName": <function name>,
    "InstCount": [
        <count_in_a>,
        <count_in_b>
        ],
    "StackSize": [
        <count_in_a>,
        <count_in_b>
        ]
    },
    ...
]

"OnlyInA": [
    {
    "FunctionName": <function name>,
    "InstCount": [
        <count_in_a>,
        0
        ],
    "StackSize": [
        <count_in_a>,
        0
        ]
    },
    ...
]

"OnlyInB": [
    {
    "FunctionName": <function name>,
    "InstCount": [
        0,
        <count_in_b>
        ],
    "StackSize": [
        0,
        <count_in_b>
        ]
    },
    ...
]

A few notes:

• Filenames are included, because tools may want to combine many outputs together in some way (a big JSON file, a big CSV, or something.)

• Counts are represented as [a, b] so that a diff can be calculated via b - a. The original counts may be useful for size analysis (e.g. was this function extremely large before?) and so both are preserved.

• OnlyInA and OnlyInB have a 0 for one of the counts always. This is to make it easier for tools to share code between OnlyInA, OnlyInB, and InBoth.

[echristo]

LGTM, one nit :)

llvm/tools/llvm-remark-size-diff/RemarkSizeDiff.cpp

483

:)

[paquette]

llvm/tools/llvm-remark-size-diff/RemarkSizeDiff.cpp

483

My brain is really small today, but I'm going to guess that this should be a switch

llvm/test/tools/llvm-remark-size-diff/json-add-remove-func.test

+RUN: llvm-remark-size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/2-identical-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=ADD
+RUN: llvm-remark-size-diff %p/Inputs/2-identical-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml  --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=REMOVE
+
+; The "two-identical-one-instr-funcs" file contains a single-instruction
+; function which does not appear in the other file.
+
+; ADD-LABEL: "Files":
+; ADD: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; ADD-NEXT: "B":{{.*}}2-identical-func-1-instr-1-stack.yaml
+
+; ADD-LABEL: "InBoth": [
+; ADD: "FunctionName": "func0",
+; ADD-NEXT: "InstCount": [
+; ADD-NEXT: 1,
+; ADD-NEXT: 1
+; ADD-NEXT: ],
+; ADD-NEXT: "StackSize": [
+; ADD-NEXT: 1,
+; ADD-NEXT: 1
+
+; ADD-LABEL: "OnlyInA": [],
+
+; ADD-LABEL: "OnlyInB": [
+; ADD: "FunctionName": "func1",
+; ADD-NEXT: "InstCount": [
+; ADD-NEXT: 0,
+; ADD-NEXT: 1
+; ADD-NEXT: ],
+; ADD-NEXT: "StackSize": [
+; ADD-NEXT: 0,
+; ADD-NEXT: 1
+
+; REMOVE-LABEL: "Files":
+; REMOVE: "A":{{.*}}2-identical-func-1-instr-1-stack.yaml
+; REMOVE-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; REMOVE-LABEL: "InBoth": [
+; REMOVE: "FunctionName": "func0",
+; REMOVE-NEXT: "InstCount": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 1
+; REMOVE-NEXT: ],
+; REMOVE-NEXT: "StackSize": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 1
+
+; REMOVE-LABEL: "OnlyInA": [
+; REMOVE: "FunctionName": "func1",
+; REMOVE-NEXT: "InstCount": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 0
+; REMOVE-NEXT: ],
+; REMOVE-NEXT: "StackSize": [
+; REMOVE-NEXT: 1,
+; REMOVE-NEXT: 0
+
+; REMOVE-LABEL: "OnlyInB": []

llvm/test/tools/llvm-remark-size-diff/json-increase-decrease-inst-count.test

+RUN: llvm-remark-size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-2-instr-2-stack.yaml --parser=yaml  --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=INCREASE
+RUN: llvm-remark-size-diff %p/Inputs/1-func-2-instr-2-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml  --report_style=json --pretty | FileCheck -strict-whitespace %s --check-prefix=DECREASE
+
+; Test a size increase/decrease of one instruction + 1 stack byte.
+
+; INCREASE-LABEL: "Files":
+; INCREASE: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; INCREASE-NEXT: "B":{{.*}}1-func-2-instr-2-stack.yaml
+
+; INCREASE-LABEL: "InBoth": [
+; INCREASE: "FunctionName": "func0"
+; INCREASE-NEXT: "InstCount":
+; INCREASE-NEXT: 1,
+; INCREASE-NEXT: 2
+; INCREASE-NEXT: ],
+; INCREASE-NEXT: "StackSize":
+; INCREASE-NEXT: 1,
+; INCREASE-NEXT: 2
+
+; INCREASE: "OnlyInA": [],
+; INCREASE: "OnlyInB": []
+
+; DECREASE-LABEL: "Files":
+; DECREASE: "A":{{.*}}1-func-2-instr-2-stack.yaml
+; DECREASE-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; DECREASE-LABEL: "InBoth": [
+; DECREASE: "FunctionName": "func0"
+; DECREASE-NEXT: "InstCount":
+; DECREASE-NEXT: 2,
+; DECREASE-NEXT: 1
+; DECREASE-NEXT: ],
+; DECREASE-NEXT: "StackSize":
+; DECREASE-NEXT: 2,
+; DECREASE-NEXT: 1
+
+; DECREASE: "OnlyInA": [],
+; DECREASE: "OnlyInB": []

llvm/test/tools/llvm-remark-size-diff/json-no-difference.test

+RUN: llvm-remark-size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml --report_style=json --pretty | FileCheck -strict-whitespace %s
+
+; CHECK-LABEL: "Files":
+; CHECK: "A":{{.*}}1-func-1-instr-1-stack.yaml
+; CHECK-NEXT: "B":{{.*}}1-func-1-instr-1-stack.yaml
+
+; CHECK-LABEL: "InBoth":
+; CHECK: "FunctionName": "func0",
+; CHECK-NEXT: "InstCount":
+; CHECK-NEXT: 1,
+; CHECK-NEXT: 1
+; CHECK-NEXT: ],
+; CHECK-NEXT: "StackSize": [
+; CHECK-NEXT: 1,
+; CHECK-NEXT: 1
+
+; CHECK: "OnlyInA": []
+; CHECK: "OnlyInB": []

llvm/tools/llvm-remark-size-diff/RemarkSizeDiff.cpp

 //===-------------- llvm-remark-size-diff/RemarkSizeDiff.cpp --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// Diffs instruction count and stack size remarks between two remark files.
 ///
 /// This is intended for use by compiler developers who want to see how their
 /// changes impact program code size.
 ///
-/// TODO: Add structured output (JSON, or YAML, or something...)
-///
 //===----------------------------------------------------------------------===//
 
 #include "llvm-c/Remarks.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Remarks/Remark.h"
 #include "llvm/Remarks/RemarkParser.h"
 #include "llvm/Remarks/RemarkSerializer.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
 enum ParserFormatOptions { yaml, bitstream };
+enum ReportStyleOptions { human_output, json_output };
 static cl::OptionCategory SizeDiffCategory("llvm-remark-size-diff options");
 static cl::opt<std::string> InputFileNameA(cl::Positional, cl::Required,
                                            cl::cat(SizeDiffCategory),
                                            cl::desc("remarks_a"));
 static cl::opt<std::string> InputFileNameB(cl::Positional, cl::Required,
                                            cl::cat(SizeDiffCategory),
                                            cl::desc("remarks_b"));
 static cl::opt<std::string> OutputFilename("o", cl::init("-"),
                                            cl::cat(SizeDiffCategory),
                                            cl::desc("Output"),
                                            cl::value_desc("file"));
 static cl::opt<ParserFormatOptions>
     ParserFormat("parser", cl::cat(SizeDiffCategory), cl::init(bitstream),
                  cl::desc("Set the remark parser format:"),
                  cl::values(clEnumVal(yaml, "YAML format"),
                             clEnumVal(bitstream, "Bitstream format")));
+static cl::opt<ReportStyleOptions> ReportStyle(
+    "report_style", cl::cat(SizeDiffCategory),
+    cl::init(ReportStyleOptions::human_output),
+    cl::desc("Choose the report output format:"),
+    cl::values(clEnumValN(human_output, "human", "Human-readable format"),
+               clEnumValN(json_output, "json", "JSON format")));
+static cl::opt<bool> PrettyPrint("pretty", cl::cat(SizeDiffCategory),
+                                 cl::init(false),
+                                 cl::desc("Pretty-print JSON"));
 
 /// Contains information from size remarks.
 // This is a little nicer to read than a std::pair.
 struct InstCountAndStackSize {
   int64_t InstCount = 0;
   int64_t StackSize = 0;
 };
 
static ErrorOr<std::unique_ptr<ToolOutputFile>> getOutputStream() {
   std::error_code EC;
   auto Out = std::make_unique<ToolOutputFile>(OutputFilename, EC,
                                               sys::fs::OF_TextWithCRLF);
   if (!EC)
     return std::move(Out);
   return EC;
 }
 
-/// Output all diffs in \p DiffsByFilesPresent.
+/// \return a json::Array representing all FunctionDiffs in \p FunctionDiffs.
+/// \p WhichFiles represents which files the functions in \p FunctionDiffs
+/// appeared in (A, B, or both).
+json::Array
+getFunctionDiffListAsJSON(const SmallVector<FunctionDiff> &FunctionDiffs,
+                          const FilesPresent &WhichFiles) {
+  json::Array FunctionDiffsAsJSON;
+  int64_t InstCountA, InstCountB, StackSizeA, StackSizeB;
+  for (auto &Diff : FunctionDiffs) {
+    InstCountA = InstCountB = StackSizeA = StackSizeB = 0;
+    switch (WhichFiles) {
+    case BOTH:
+      LLVM_FALLTHROUGH;
+    case A:
+      InstCountA = Diff.getInstCountA();
+      StackSizeA = Diff.getStackSizeA();
+      if (WhichFiles != BOTH)
+        break;
+      LLVM_FALLTHROUGH;
+    case B:
+      InstCountB = Diff.getInstCountB();
+      StackSizeB = Diff.getStackSizeB();
+      break;
+    }
+    // Each metric we care about is represented like:
+    //   "Val": [A, B]
+    // This allows any consumer of the JSON to calculate the diff using B - A.
+    // This is somewhat wasteful for OnlyInA and OnlyInB (we only need A or B).
+    // However, this should make writing consuming tools easier, since the tool
+    // writer doesn't need to think about slightly different formats in each
+    // section.
+    json::Object FunctionObject({{"FunctionName", Diff.FuncName},
+                                 {"InstCount", {InstCountA, InstCountB}},
+                                 {"StackSize", {StackSizeA, StackSizeB}}});
+    FunctionDiffsAsJSON.push_back(std::move(FunctionObject));
+  }
+  return FunctionDiffsAsJSON;
+}
+
+/// Output all diffs in \p DiffsByFilesPresent as a JSON report. This is
+/// intended for consumption by external tools.
+///
+/// \p InputFileNameA - File A used to produce the report.
+/// \p InputFileNameB - File B used ot produce the report.
+/// \p OS - Output stream.
+///
+/// JSON output includes:
+///  - \p InputFileNameA and \p InputFileNameB under "Files".
+///  - Functions present in both files under "InBoth".
+///  - Functions present only in A in "OnlyInA".
+///  - Functions present only in B in "OnlyInB".
+///  - Instruction count and stack size differences for each function.
+///
+/// Differences are represented using [count_a, count_b]. The actual difference
+/// can be computed via count_b - count_a.
+static void
+outputJSONForAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+                      const DiffsCategorizedByFilesPresent &DiffsByFilesPresent,
+                      llvm::raw_ostream &OS) {
+  json::Object Output;
+  // Include file names in the report.
+  json::Object Files(
+      {{"A", InputFileNameA.str()}, {"B", InputFileNameB.str()}});
+  Output["Files"] = std::move(Files);
+  Output["OnlyInA"] = getFunctionDiffListAsJSON(DiffsByFilesPresent.OnlyInA, A);
+  Output["OnlyInB"] = getFunctionDiffListAsJSON(DiffsByFilesPresent.OnlyInB, B);
+  Output["InBoth"] =
+      getFunctionDiffListAsJSON(DiffsByFilesPresent.InBoth, BOTH);
+  json::OStream JOS(OS, PrettyPrint ? 2 : 0);
+  JOS.value(std::move(Output));
+  OS << '\n';
+}
+
+/// Output all diffs in \p DiffsByFilesPresent using the desired output style.
 /// \returns Error::success() on success, and an Error otherwise.
+/// \p InputFileNameA - Name of input file A; may be used in the report.
+/// \p InputFileNameB - Name of input file B; may be used in the report.
 static Error
-outputAllDiffs(DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
+outputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+               DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
   auto MaybeOF = getOutputStream();
   if (std::error_code EC = MaybeOF.getError())
     return errorCodeToError(EC);
   std::unique_ptr<ToolOutputFile> OF = std::move(*MaybeOF);
+  switch (ReportStyle) {
+  case human_output:
     printDiffsCategorizedByFilesPresent(DiffsByFilesPresent, OF->os());
+    break;
+  case json_output:
+    outputJSONForAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent,
+                          OF->os());
+    break;
+  }
   OF->keep();
   return Error::success();
 }
 
 /// Boolean wrapper for outputDiff which handles errors.
 static bool
-tryOutputAllDiffs(DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
-  if (Error E = outputAllDiffs(DiffsByFilesPresent)) {
+tryOutputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
+                  DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
+  if (Error E =
+          outputAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent)) {
     handleAllErrors(std::move(E), [&](const ErrorInfoBase &PE) {
       PE.log(WithColor::error());
       errs() << '\n';
     });
     return false;
   }
   return true;
 }
 
 int main(int argc, const char **argv) {
   InitLLVM X(argc, argv);
   cl::HideUnrelatedOptions(SizeDiffCategory);
   cl::ParseCommandLineOptions(argc, argv,
                               "Diff instruction count and stack size remarks "
                               "between two remark files.\n");
   StringMap<InstCountAndStackSize> FuncNameToSizeInfoA;
   StringMap<InstCountAndStackSize> FuncNameToSizeInfoB;
   if (!tryReadFileAndProcessRemarks(InputFileNameA, FuncNameToSizeInfoA) ||
       !tryReadFileAndProcessRemarks(InputFileNameB, FuncNameToSizeInfoB))
     return 1;
   DiffsCategorizedByFilesPresent DiffsByFilesPresent;
   computeDiff(FuncNameToSizeInfoA, FuncNameToSizeInfoB, DiffsByFilesPresent);
-  if (!tryOutputAllDiffs(DiffsByFilesPresent))
+  if (!tryOutputAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent))
     return 1;
 }