[BOLT] Refactor heatmap to be standalone tool

[yota9]

Summary

Separate heatmap from bolt and build it as standalone tool.

[Amir]

The changes make sense to me. Running internal checks.
@maksfb, is that what you meant by refactoring the heatmap tool into its own tool?

[yota9]

@Amir Great thanks! Please let me know the tests results :)

[Amir]

The test succeeded.

In D118946#3297841, @yota9 wrote:

@Amir Great thanks! Please let me know the tests results :)

[yota9]

Fix shared libs build mode

[yota9]

Make InputFilename static

[maksfb]

Thanks for fixing the heatmap, @yota9. It will be good to refactor heatmap-specific code and options, but we can do it once basic functionality is restored.

bolt/docs/Heatmaps.md

 or if you want to monitor the existing process(es):
 ```bash
 $ perf record -e cycles:u -j any,u [-p PID|-a] -- sleep <interval>
 ```
 
 Note that at the moment running with LBR (`-j any,u` or `-b`) is
 a requirement.
 
-Once the run is complete, and `perf.data` is generated, run BOLT in
-a heatmap mode:
+Once the run is complete, and `perf.data` is generated, run llvm-bolt-heatmap:
 
 ```bash
-$ llvm-bolt heatmap -p perf.data <executable>
+$ llvm-bolt-heatmap -p perf.data <executable>
 ```
 
 By default the heatmap will be dumped to *stdout*. You can change it
 with `-o <heatmapfile>` option. Each character/block in the heatmap
 shows the execution data accumulated for corresponding 64 bytes of
 code. You can change this granularity with a `-block-size` option.
 E.g. set it to 4096 to see code usage grouped by 4K pages.
 Other useful options are:

bolt/include/bolt/Utils/CommandLineOpts.h

 
 extern llvm::cl::OptionCategory BoltCategory;
 extern llvm::cl::OptionCategory BoltDiffCategory;
 extern llvm::cl::OptionCategory BoltOptCategory;
 extern llvm::cl::OptionCategory BoltRelocCategory;
 extern llvm::cl::OptionCategory BoltOutputCategory;
 extern llvm::cl::OptionCategory AggregatorCategory;
 extern llvm::cl::OptionCategory BoltInstrCategory;
-
-extern llvm::cl::SubCommand HeatmapCommand;
+extern llvm::cl::OptionCategory HeatmapCategory;
 
 extern llvm::cl::opt<unsigned> AlignText;
 extern llvm::cl::opt<bool> AggregateOnly;
 extern llvm::cl::opt<unsigned> BucketsPerLine;
 extern llvm::cl::opt<bool> DiffOnly;
 extern llvm::cl::opt<bool> EnableBAT;
 extern llvm::cl::opt<bool> RemoveSymtab;
 extern llvm::cl::opt<unsigned> ExecutionCountThreshold;
 extern llvm::cl::opt<unsigned> HeatmapBlock;
-extern llvm::cl::opt<std::string> HeatmapFile;
 extern llvm::cl::opt<unsigned long long> HeatmapMaxAddress;
 extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;
 extern llvm::cl::opt<bool> Instrument;
 extern llvm::cl::opt<std::string> OutputFilename;
 extern llvm::cl::opt<std::string> PerfData;

bolt/lib/Profile/DataAggregator.cpp

std::error_code DataAggregator::printLBRHeatMap() {
   if (HM.getNumInvalidRanges())
     outs() << "HEATMAP: invalid traces: " << HM.getNumInvalidRanges() << '\n';
 
   if (!HM.size()) {
     errs() << "HEATMAP-ERROR: no valid traces registered\n";
     exit(1);
   }
 
-  HM.print(opts::HeatmapFile);
-  if (opts::HeatmapFile == "-")
-    HM.printCDF(opts::HeatmapFile);
+  HM.print(opts::OutputFilename);
+  if (opts::OutputFilename == "-")
+    HM.printCDF(opts::OutputFilename);
   else
-    HM.printCDF(opts::HeatmapFile + ".csv");
+    HM.printCDF(opts::OutputFilename + ".csv");
 
   return std::error_code();
 }
 
 std::error_code DataAggregator::parseBranchEvents() {
   outs() << "PERF2BOLT: parse branch events...\n";
   NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,
                      TimerGroupDesc, opts::TimeAggregator);

bolt/lib/Utils/CommandLineOpts.cpp

 
 cl::OptionCategory BoltCategory("BOLT generic options");
 cl::OptionCategory BoltDiffCategory("BOLTDIFF generic options");
 cl::OptionCategory BoltOptCategory("BOLT optimization options");
 cl::OptionCategory BoltRelocCategory("BOLT options in relocation mode");
 cl::OptionCategory BoltOutputCategory("Output options");
 cl::OptionCategory AggregatorCategory("Data aggregation options");
 cl::OptionCategory BoltInstrCategory("BOLT instrumentation options");
-
-cl::SubCommand HeatmapCommand("heatmap", "generate heatmap");
+cl::OptionCategory HeatmapCategory("Heatmap options");
 
 cl::opt<unsigned>
 AlignText("align-text",
   cl::desc("alignment of .text section"),
   cl::ZeroOrMore,
   cl::Hidden,
   cl::cat(BoltCategory));
 
 cl::opt<bool>
 AggregateOnly("aggregate-only",
   cl::desc("exit after writing aggregated data file"),
   cl::Hidden,
   cl::cat(AggregatorCategory));
 
 cl::opt<unsigned>
     BucketsPerLine("line-size",
                    cl::desc("number of entries per line (default 256)"),
-  cl::init(256),
-  cl::Optional,
-  cl::sub(HeatmapCommand));
+                   cl::init(256), cl::Optional, cl::cat(HeatmapCategory));
 
 cl::opt<bool>
 DiffOnly("diff-only",
   cl::desc("stop processing once we have enough to compare two binaries"),
   cl::Hidden,
   cl::cat(BoltDiffCategory));
 
 cl::opt<bool>
ExecutionCountThreshold("execution-count-threshold",
   cl::desc("perform profiling accuracy-sensitive optimizations only if "
            "function execution count >= the threshold (default: 0)"),
   cl::init(0),
   cl::ZeroOrMore,
   cl::Hidden,
   cl::cat(BoltOptCategory));
 
 cl::opt<unsigned>
     HeatmapBlock("block-size",
                  cl::desc("size of a heat map block in bytes (default 64)"),
-  cl::init(64),
-  cl::sub(HeatmapCommand));
+                 cl::init(64), cl::cat(HeatmapCategory));
 
-cl::opt<std::string>
-HeatmapFile("o",
-  cl::init("-"),
-  cl::desc("heatmap output file (default stdout)"),
-  cl::Optional,
-  cl::sub(HeatmapCommand));
-
-cl::opt<unsigned long long>
-HeatmapMaxAddress("max-address",
-  cl::init(0xffffffff),
+cl::opt<unsigned long long> HeatmapMaxAddress(
+    "max-address", cl::init(0xffffffff),
     cl::desc("maximum address considered valid for heatmap (default 4GB)"),
-  cl::Optional,
-  cl::sub(HeatmapCommand));
+    cl::Optional, cl::cat(HeatmapCategory));
 
-cl::opt<unsigned long long>
-HeatmapMinAddress("min-address",
-  cl::init(0x0),
+cl::opt<unsigned long long> HeatmapMinAddress(
+    "min-address", cl::init(0x0),
     cl::desc("minimum address considered valid for heatmap (default 0)"),
-  cl::Optional,
-  cl::sub(HeatmapCommand));
+    cl::Optional, cl::cat(HeatmapCategory));
 
 cl::opt<bool>
 HotData("hot-data",
   cl::desc("hot data symbols support (relocation mode)"),
   cl::ZeroOrMore,
   cl::cat(BoltCategory));
 
 cl::opt<bool>

bolt/test/heatmap.test

 # Verifies basic functioning of heatmap mode
 
 REQUIRES: system-linux
-XFAIL: *
 
-RUN: llvm-bolt heatmap --help | FileCheck %s --check-prefix=CHECK-SUBCOMMAND
-CHECK-SUBCOMMAND: SUBCOMMAND 'heatmap': generate heatmap
-CHECK-SUBCOMMAND: USAGE: llvm-bolt heatmap [options] <executable>
-
-RUN: llvm-bolt-heatmap --help | FileCheck %s --check-prefix=CHECK-STANDALONE
-CHECK-STANDALONE: SUBCOMMAND 'heatmap': generate heatmap
-CHECK-STANDALONE: USAGE: llvm-bolt-heatmap heatmap [options] <executable>
+RUN: llvm-bolt-heatmap --help | FileCheck %s
+CHECK: USAGE: llvm-bolt-heatmap [options] <executable>

bolt/tools/CMakeLists.txt

 add_subdirectory(driver)
 add_subdirectory(merge-fdata)
+add_subdirectory(heatmap)

bolt/tools/driver/CMakeLists.txt

add_llvm_tool(llvm-bolt
   llvm-bolt.cpp
 
   DEPENDS
   ${BOLT_DRIVER_DEPS}
   )
 
 add_llvm_tool_symlink(perf2bolt llvm-bolt)
 add_llvm_tool_symlink(llvm-boltdiff llvm-bolt)
-add_llvm_tool_symlink(llvm-bolt-heatmap llvm-bolt)
 
 set(BOLT_DEPENDS
   llvm-bolt
   perf2bolt
   llvm-boltdiff
-  llvm-bolt-heatmap
   )
 
 add_custom_target(bolt DEPENDS ${BOLT_DEPENDS})
 install(PROGRAMS
   ${CMAKE_BINARY_DIR}/bin/llvm-bolt
   ${CMAKE_BINARY_DIR}/bin/perf2bolt
   ${CMAKE_BINARY_DIR}/bin/llvm-boltdiff
-  ${CMAKE_BINARY_DIR}/bin/llvm-bolt-heatmap
   TYPE BIN
   COMPONENT bolt
   )
 add_llvm_install_targets(install-bolt DEPENDS bolt COMPONENT bolt)
 set_target_properties(bolt PROPERTIES FOLDER "BOLT")
 set_target_properties(install-bolt PROPERTIES FOLDER "BOLT")
 
 include_directories( ${BOLT_SOURCE_DIR}/lib )

bolt/tools/driver/llvm-bolt.cpp

void perf2boltMode(int argc, char **argv) {
   }
   if (opts::OutputFilename.empty()) {
     errs() << ToolName << ": expected -o=<output file> option.\n";
     exit(1);
   }
   opts::AggregateOnly = true;
 }
 
-void heatmapMode(int argc, char **argv) {
-  // Insert a fake subcommand if invoked via a command alias.
-  std::unique_ptr<char *[]> FakeArgv;
-  if (argc == 1 || strcmp(argv[1], "heatmap")) {
-    ++argc;
-    FakeArgv.reset(new char *[argc + 1]);
-    FakeArgv[0] = argv[0];
-    FakeArgv[1] = const_cast<char *>("heatmap");
-    for (int I = 2; I < argc; ++I)
-      FakeArgv[I] = argv[I - 1];
-    FakeArgv[argc] = nullptr;
-    argv = FakeArgv.get();
-  }
-
-  cl::ParseCommandLineOptions(argc, argv, "");
-
-  if (!sys::fs::exists(opts::InputFilename))
-    report_error(opts::InputFilename, errc::no_such_file_or_directory);
-
-  if (opts::PerfData.empty()) {
-    errs() << ToolName << ": expected -perfdata=<filename> option.\n";
-    exit(1);
-  }
-
-  opts::HeatmapMode = true;
-  opts::AggregateOnly = true;
-}
-
 void boltDiffMode(int argc, char **argv) {
   cl::HideUnrelatedOptions(makeArrayRef(opts::BoltDiffCategories));
   cl::AddExtraVersionPrinter(printBoltRevision);
   cl::ParseCommandLineOptions(
       argc, argv,
       "llvm-boltdiff - BOLT binary diff tool\n"
       "\nEXAMPLE: llvm-boltdiff -data=a.fdata -data2=b.fdata exec1 exec2\n");
   if (opts::InputDataFilename2.empty()) {
cl::ParseCommandLineOptions(argc, argv,
                               "BOLT - Binary Optimization and Layout Tool\n");
 
   if (opts::OutputFilename.empty()) {
     errs() << ToolName << ": expected -o=<output file> option.\n";
     exit(1);
   }
 }
 
-std::string GetExecutablePath(const char *Argv0) {
-  SmallString<128> ExecutablePath(Argv0);
+static std::string GetExecutablePath(const char *Argv0) {
+  SmallString<256> ExecutablePath(Argv0);
   // Do a PATH lookup if Argv0 isn't a valid path.
   if (!llvm::sys::fs::exists(ExecutablePath))
     if (llvm::ErrorOr<std::string> P =
             llvm::sys::findProgramByName(ExecutablePath))
       ExecutablePath = *P;
   return std::string(ExecutablePath.str());
 }
 
int main(int argc, char **argv) {
   llvm::InitializeAllAsmParsers();
   llvm::InitializeAllDisassemblers();
 
   llvm::InitializeAllTargets();
   llvm::InitializeAllAsmPrinters();
 
   ToolName = argv[0];
 
-  // Pre-process subcommands.
-  if (argc > 1 && *argv[1] != '-') {
-    if (!strcmp(argv[1], "heatmap"))
-      opts::HeatmapMode = true;
-  }
-
   if (llvm::sys::path::filename(ToolName) == "perf2bolt")
     perf2boltMode(argc, argv);
   else if (llvm::sys::path::filename(ToolName) == "llvm-boltdiff")
     boltDiffMode(argc, argv);
-  else if (llvm::sys::path::filename(ToolName) == "llvm-bolt-heatmap" ||
-           opts::HeatmapMode)
-    heatmapMode(argc, argv);
   else
     boltMode(argc, argv);
 
   if (!sys::fs::exists(opts::InputFilename))
     report_error(opts::InputFilename, errc::no_such_file_or_directory);
 
   // Attempt to open the binary.
   if (!opts::DiffOnly) {

bolt/tools/heatmap/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  BOLTProfile
+  BOLTRewrite
+  BOLTUtils
+  MC
+  Object
+  Support
+  )
+
+add_llvm_tool(llvm-bolt-heatmap
+  heatmap.cpp
+  )
+
+set_target_properties(llvm-bolt-heatmap PROPERTIES FOLDER "BOLT")

bolt/tools/heatmap/heatmap.cpp

+#include "bolt/Profile/DataAggregator.h"
+#include "bolt/Rewrite/RewriteInstance.h"
+#include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TargetSelect.h"
+
+using namespace llvm;
+using namespace bolt;
+
+namespace opts {
+
+static cl::OptionCategory *HeatmapCategories[] = {&HeatmapCategory,
+                                                  &BoltOutputCategory};
+
+static cl::opt<std::string> InputFilename(cl::Positional,
+                                          cl::desc("<executable>"),
+                                          cl::Required,
+                                          cl::cat(HeatmapCategory));
+
+} // namespace opts
+
+static StringRef ToolName;
+
+static void report_error(StringRef Message, std::error_code EC) {
+  assert(EC);
+  errs() << ToolName << ": '" << Message << "': " << EC.message() << ".\n";
+  exit(1);
+}
+
+static void report_error(StringRef Message, Error E) {
+  assert(E);
+  errs() << ToolName << ": '" << Message << "': " << toString(std::move(E))
+         << ".\n";
+  exit(1);
+}
+
+static std::string GetExecutablePath(const char *Argv0) {
+  SmallString<256> ExecutablePath(Argv0);
+  // Do a PATH lookup if Argv0 isn't a valid path.
+  if (!llvm::sys::fs::exists(ExecutablePath))
+    if (llvm::ErrorOr<std::string> P =
+            llvm::sys::findProgramByName(ExecutablePath))
+      ExecutablePath = *P;
+  return std::string(ExecutablePath.str());
+}
+
+int main(int argc, char **argv) {
+  cl::HideUnrelatedOptions(makeArrayRef(opts::HeatmapCategories));
+  cl::ParseCommandLineOptions(argc, argv, "");
+
+  if (opts::PerfData.empty()) {
+    errs() << ToolName << ": expected -perfdata=<filename> option.\n";
+    exit(1);
+  }
+
+  opts::HeatmapMode = true;
+  opts::AggregateOnly = true;
+  if (!sys::fs::exists(opts::InputFilename))
+    report_error(opts::InputFilename, errc::no_such_file_or_directory);
+
+  // Output to stdout by default
+  if (opts::OutputFilename.empty())
+    opts::OutputFilename = "-";
+
+  // Initialize targets and assembly printers/parsers.
+  llvm::InitializeAllTargetInfos();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllAsmParsers();
+  llvm::InitializeAllDisassemblers();
+
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllAsmPrinters();
+
+  ToolName = argv[0];
+  std::string ToolPath = GetExecutablePath(argv[0]);
+  Expected<OwningBinary<Binary>> BinaryOrErr =
+      createBinary(opts::InputFilename);
+  if (Error E = BinaryOrErr.takeError())
+    report_error(opts::InputFilename, std::move(E));
+  Binary &Binary = *BinaryOrErr.get().getBinary();
+
+  if (auto *e = dyn_cast<ELFObjectFileBase>(&Binary)) {
+    RewriteInstance RI(e, argc, argv, ToolPath);
+    if (Error E = RI.setProfile(opts::PerfData))
+      report_error(opts::PerfData, std::move(E));
+
+    RI.run();
+  } else {
+    report_error(opts::InputFilename, object_error::invalid_file_type);
+  }
+
+  return EXIT_SUCCESS;
+}