[llvm-exegesis] Refactor FunctionExecutorImpl and create factory

[aidengrossman]

Summary

In order to better support adding in new implementations of
FunctionExecutor, this patch makes some small changes so that it is
easier to add new ones in. FunctionExecutorImpl is renamed to
InProcessFunctionExecutorImpl to better reflect how it will be placed
relative to the soon-to-be introduced subprocess executor and a new
function is created to create executors so selection can be done more
easily. In addition, a new CLI flag, -execution-mode, which can be used
to select between the different executors.

[courbet]

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

195

A runnable configuration exists outside of how it's executed, it feels weird to have this here. Why isn't this a member of BenchmarkRunner ?

204

Why do we need Key ?

205

[nit] use switch, even if it's only one option for now, I think it's more readable for enums.

[aidengrossman]

Address reviewer feedback.

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

195

Good point. I've switched it over to being a member of BenchmarkRunner.

204

I'm using a BenchmarkKey in https://reviews.llvm.org/D151025 (and touching it in some previous patches in the stack) to pass around memory annotations, and the way I have it structured currently, I pass the key to the FunctionExecutorImpl for setup. I can move this to another diff/change this around if desired.

[courbet]

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

128

[nit] createFunctionExecutor ?

[aidengrossman]

Address reviewer feedback.

[aidengrossman]

Format

[aidengrossman]

Update

[aidengrossman]

Rebase

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

 #include <vector>
 
 namespace llvm {
 namespace exegesis {
 
 // Common code for all benchmark modes.
 class BenchmarkRunner {
 public:
-  explicit BenchmarkRunner(const LLVMState &State,
-                           Benchmark::ModeE Mode,
-                           BenchmarkPhaseSelectorE BenchmarkPhaseSelector);
+  enum ExecutionModeE { InProcess };
+
+  explicit BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
+                           BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                           ExecutionModeE ExecutionMode);
 
   virtual ~BenchmarkRunner();
 
   class RunnableConfiguration {
     friend class BenchmarkRunner;
 
   public:
     ~RunnableConfiguration() = default;
protected:
     virtual Expected<llvm::SmallVector<int64_t, 4>>
     runWithCounter(StringRef CounterName) const = 0;
   };
 
 protected:
   const LLVMState &State;
   const Benchmark::ModeE Mode;
   const BenchmarkPhaseSelectorE BenchmarkPhaseSelector;
+  const ExecutionModeE ExecutionMode;
 
 private:
   virtual Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const = 0;
 
   Expected<SmallString<0>> assembleSnippet(const BenchmarkCode &BC,
                                            const SnippetRepetitor &Repetitor,
                                            unsigned MinInstructions,
                                            unsigned LoopBodySize) const;
 
   Expected<std::string> writeObjectFile(StringRef Buffer,
                                         StringRef FileName) const;
 
   const std::unique_ptr<ScratchSpace> Scratch;
+
+  Expected<std::unique_ptr<FunctionExecutor>>
+  createFunctionExecutor(object::OwningBinary<object::ObjectFile> Obj,
+                         const BenchmarkKey &Key) const;
 };
 
 } // namespace exegesis
 } // namespace llvm
 
 #endif // LLVM_TOOLS_LLVM_EXEGESIS_BENCHMARKRUNNER_H

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Program.h"
 
 namespace llvm {
 namespace exegesis {
 
-BenchmarkRunner::BenchmarkRunner(const LLVMState &State,
-                                 Benchmark::ModeE Mode,
-                                 BenchmarkPhaseSelectorE BenchmarkPhaseSelector)
+BenchmarkRunner::BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
+                                 BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                                 ExecutionModeE ExecutionMode)
     : State(State), Mode(Mode), BenchmarkPhaseSelector(BenchmarkPhaseSelector),
-      Scratch(std::make_unique<ScratchSpace>()) {}
+      ExecutionMode(ExecutionMode), Scratch(std::make_unique<ScratchSpace>()) {}
 
 BenchmarkRunner::~BenchmarkRunner() = default;
 
 void BenchmarkRunner::FunctionExecutor::accumulateCounterValues(
     const llvm::SmallVectorImpl<int64_t> &NewValues,
     llvm::SmallVectorImpl<int64_t> *Result) {
   const size_t NumValues = std::max(NewValues.size(), Result->size());
   if (NumValues > Result->size())
for (auto &CounterName : CounterNames) {
     if (!ValueOrError)
       return ValueOrError.takeError();
     accumulateCounterValues(ValueOrError.get(), &CounterValues);
   }
   return CounterValues;
 }
 
 namespace {
-class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
+class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
 public:
-  FunctionExecutorImpl(const LLVMState &State,
+  InProcessFunctionExecutorImpl(const LLVMState &State,
                                 object::OwningBinary<object::ObjectFile> Obj,
                                 BenchmarkRunner::ScratchSpace *Scratch)
       : State(State), Function(State.createTargetMachine(), std::move(Obj)),
         Scratch(Scratch) {}
 
 private:
   static void
   accumulateCounterValues(const llvm::SmallVector<int64_t, 4> &NewValues,
                           llvm::SmallVector<int64_t, 4> *Result) {
     const size_t NumValues = std::max(NewValues.size(), Result->size());
auto Snippet = assembleSnippet(BC, Repetitor, InstrBenchmark.NumRepetitions,
                                    LoopBodySize);
     if (Error E = Snippet.takeError())
       return std::move(E);
     RC.ObjectFile = getObjectFromBuffer(*Snippet);
   }
 
   return std::move(RC);
 }
 
+Expected<std::unique_ptr<BenchmarkRunner::FunctionExecutor>>
+BenchmarkRunner::createFunctionExecutor(
+    object::OwningBinary<object::ObjectFile> ObjectFile,
+    const BenchmarkKey &Key) const {
+  switch (ExecutionMode) {
+  case ExecutionModeE::InProcess:
+    return std::make_unique<InProcessFunctionExecutorImpl>(
+        State, std::move(ObjectFile), Scratch.get());
+  }
+  llvm_unreachable("ExecutionMode is outside expected range");
+}
+
 Expected<Benchmark> BenchmarkRunner::runConfiguration(
     RunnableConfiguration &&RC,
     const std::optional<StringRef> &DumpFile) const {
   Benchmark &InstrBenchmark = RC.InstrBenchmark;
   object::OwningBinary<object::ObjectFile> &ObjectFile = RC.ObjectFile;
 
   if (DumpFile && BenchmarkPhaseSelector >
                       BenchmarkPhaseSelectorE::PrepareAndAssembleSnippet) {
     auto ObjectFilePath =
         writeObjectFile(ObjectFile.getBinary()->getData(), *DumpFile);
     if (Error E = ObjectFilePath.takeError()) {
       InstrBenchmark.Error = toString(std::move(E));
       return std::move(InstrBenchmark);
     }
     outs() << "Check generated assembly with: /usr/bin/objdump -d "
            << *ObjectFilePath << "\n";
   }
 
   if (BenchmarkPhaseSelector < BenchmarkPhaseSelectorE::Measure) {
     InstrBenchmark.Error = "actual measurements skipped.";
     return std::move(InstrBenchmark);
   }
 
-  const FunctionExecutorImpl Executor(State, std::move(ObjectFile),
-                                      Scratch.get());
-  auto NewMeasurements = runMeasurements(Executor);
+  Expected<std::unique_ptr<BenchmarkRunner::FunctionExecutor>> Executor =
+      createFunctionExecutor(std::move(ObjectFile), RC.InstrBenchmark.Key);
+  if (!Executor)
+    return Executor.takeError();
+  auto NewMeasurements = runMeasurements(**Executor);
+
   if (Error E = NewMeasurements.takeError()) {
     if (!E.isA<SnippetCrash>())
       return std::move(E);
     InstrBenchmark.Error = toString(std::move(E));
     return std::move(InstrBenchmark);
   }
   assert(InstrBenchmark.NumRepetitions > 0 && "invalid NumRepetitions");
   for (BenchmarkMeasure &BM : *NewMeasurements) {

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h

 
 #include "BenchmarkRunner.h"
 
 namespace llvm {
 namespace exegesis {
 
 class LatencyBenchmarkRunner : public BenchmarkRunner {
 public:
-  LatencyBenchmarkRunner(
-      const LLVMState &State, Benchmark::ModeE Mode,
+  LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
                          BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode);
+                         Benchmark::ResultAggregationModeE ResultAggMode,
+                         ExecutionModeE ExecutionMode);
   ~LatencyBenchmarkRunner() override;
 
 private:
   Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const override;
 
   Benchmark::ResultAggregationModeE ResultAggMode;
 };
 } // namespace exegesis
 } // namespace llvm
 
 #endif // LLVM_TOOLS_LLVM_EXEGESIS_LATENCY_H

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp

 #include <cmath>
 
 namespace llvm {
 namespace exegesis {
 
 LatencyBenchmarkRunner::LatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAgg)
-    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector) {
-  assert((Mode == Benchmark::Latency ||
-          Mode == Benchmark::InverseThroughput) &&
+    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode)
+    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
+  assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
          "invalid mode");
   ResultAggMode = ResultAgg;
 }
 
 LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
 
 static double computeVariance(const llvm::SmallVector<int64_t, 4> &Values) {
   if (Values.empty())

llvm/tools/llvm-exegesis/lib/Target.h

public:
   std::unique_ptr<SnippetGenerator>
   createSnippetGenerator(Benchmark::ModeE Mode,
                          const LLVMState &State,
                          const SnippetGenerator::Options &Opts) const;
   // Creates a benchmark runner for the given mode.
   Expected<std::unique_ptr<BenchmarkRunner>> createBenchmarkRunner(
       Benchmark::ModeE Mode, const LLVMState &State,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode =
-          Benchmark::Min) const;
+      BenchmarkRunner::ExecutionModeE ExecutionMode,
+      Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
 
   // Returns the ExegesisTarget for the given triple or nullptr if the target
   // does not exist.
   static const ExegesisTarget *lookup(Triple TT);
   // Returns the default (unspecialized) ExegesisTarget.
   static const ExegesisTarget &getDefault();
   // Registers a target. Not thread safe.
   static void registerTarget(ExegesisTarget *T);
private:
   // implementing these.
   std::unique_ptr<SnippetGenerator> virtual createSerialSnippetGenerator(
       const LLVMState &State, const SnippetGenerator::Options &Opts) const;
   std::unique_ptr<SnippetGenerator> virtual createParallelSnippetGenerator(
       const LLVMState &State, const SnippetGenerator::Options &Opts) const;
   std::unique_ptr<BenchmarkRunner> virtual createLatencyBenchmarkRunner(
       const LLVMState &State, Benchmark::ModeE Mode,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode) const;
+      Benchmark::ResultAggregationModeE ResultAggMode,
+      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
       const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode) const;
+      Benchmark::ResultAggregationModeE ResultAggMode,
+      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
 
   const ExegesisTarget *Next = nullptr;
   const ArrayRef<CpuAndPfmCounters> CpuPfmCounters;
 };
 
 } // namespace exegesis
 } // namespace llvm
 
 #endif // LLVM_TOOLS_LLVM_EXEGESIS_TARGET_H

llvm/tools/llvm-exegesis/lib/Target.cpp

std::unique_ptr<SnippetGenerator> ExegesisTarget::createSnippetGenerator(
   }
   return nullptr;
 }
 
 Expected<std::unique_ptr<BenchmarkRunner>>
 ExegesisTarget::createBenchmarkRunner(
     Benchmark::ModeE Mode, const LLVMState &State,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+    BenchmarkRunner::ExecutionModeE ExecutionMode,
     Benchmark::ResultAggregationModeE ResultAggMode) const {
   PfmCountersInfo PfmCounters = State.getPfmCounters();
   switch (Mode) {
   case Benchmark::Unknown:
     return nullptr;
   case Benchmark::Latency:
   case Benchmark::InverseThroughput:
     if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
         !PfmCounters.CycleCounter) {
       const char *ModeName = Mode == Benchmark::Latency
                                  ? "latency"
                                  : "inverse_throughput";
       return make_error<Failure>(
           Twine("can't run '")
               .concat(ModeName)
               .concat(
                   "' mode, sched model does not define a cycle counter. You "
                   "can pass --benchmark-phase=... to skip the actual "
                   "benchmarking or --use-dummy-perf-counters to not query "
                   "the kernel for real event counts."));
     }
     return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector,
-                                        ResultAggMode);
+                                        ResultAggMode, ExecutionMode);
   case Benchmark::Uops:
     if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
         !PfmCounters.UopsCounter && !PfmCounters.IssueCounters)
       return make_error<Failure>(
           "can't run 'uops' mode, sched model does not define uops or issue "
           "counters. You can pass --benchmark-phase=... to skip the actual "
           "benchmarking or --use-dummy-perf-counters to not query the kernel "
           "for real event counts.");
     return createUopsBenchmarkRunner(State, BenchmarkPhaseSelector,
-                                     ResultAggMode);
+                                     ResultAggMode, ExecutionMode);
   }
   return nullptr;
 }
 
 std::unique_ptr<SnippetGenerator> ExegesisTarget::createSerialSnippetGenerator(
     const LLVMState &State, const SnippetGenerator::Options &Opts) const {
   return std::make_unique<SerialSnippetGenerator>(State, Opts);
 }
 
 std::unique_ptr<SnippetGenerator> ExegesisTarget::createParallelSnippetGenerator(
     const LLVMState &State, const SnippetGenerator::Options &Opts) const {
   return std::make_unique<ParallelSnippetGenerator>(State, Opts);
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAggMode) const {
+    Benchmark::ResultAggregationModeE ResultAggMode,
+    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
   return std::make_unique<LatencyBenchmarkRunner>(
-      State, Mode, BenchmarkPhaseSelector, ResultAggMode);
+      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode);
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
     const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE /*unused*/) const {
-  return std::make_unique<UopsBenchmarkRunner>(State, BenchmarkPhaseSelector);
+    Benchmark::ResultAggregationModeE /*unused*/,
+    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
+  return std::make_unique<UopsBenchmarkRunner>(State, BenchmarkPhaseSelector,
+                                               ExecutionMode);
 }
 
 static_assert(std::is_trivial_v<PfmCountersInfo>,
               "We shouldn't have dynamic initialization here");
 const PfmCountersInfo PfmCountersInfo::Default = {nullptr, nullptr, nullptr,
                                                   0u};
 const PfmCountersInfo PfmCountersInfo::Dummy = {
     pfm::PerfEvent::DummyEventString, pfm::PerfEvent::DummyEventString, nullptr,

llvm/tools/llvm-exegesis/lib/UopsBenchmarkRunner.h

 #include "BenchmarkRunner.h"
 
 namespace llvm {
 namespace exegesis {
 
 class UopsBenchmarkRunner : public BenchmarkRunner {
 public:
   UopsBenchmarkRunner(const LLVMState &State,
-                      BenchmarkPhaseSelectorE BenchmarkPhaseSelector)
-      : BenchmarkRunner(State, Benchmark::Uops,
-                        BenchmarkPhaseSelector) {}
+                      BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                      ExecutionModeE ExecutionMode)
+      : BenchmarkRunner(State, Benchmark::Uops, BenchmarkPhaseSelector,
+                        ExecutionMode) {}
   ~UopsBenchmarkRunner() override;
 
   static constexpr const size_t kMinNumDifferentAddresses = 6;
 
 private:
   Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const override;
 };
 
 } // namespace exegesis
 } // namespace llvm
 
 #endif // LLVM_TOOLS_LLVM_EXEGESIS_UOPSBENCHMARKRUNNER_H

llvm/tools/llvm-exegesis/llvm-exegesis.cpp

MCPU("mcpu",
          cl::value_desc("cpu-name"), cl::cat(Options), cl::init("native"));
 
 static cl::opt<std::string>
     DumpObjectToDisk("dump-object-to-disk",
                      cl::desc("dumps the generated benchmark object to disk "
                               "and prints a message to access it"),
                      cl::ValueOptional, cl::cat(BenchmarkOptions));
 
+static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
+    "execution-mode",
+    cl::desc("Selects the execution mode to use for running snippets"),
+    cl::cat(BenchmarkOptions),
+    cl::values(clEnumValN(BenchmarkRunner::ExecutionModeE::InProcess,
+                          "inprocess",
+                          "Executes the snippets within the same process")),
+    cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
+
 static ExitOnError ExitOnErr("llvm-exegesis error: ");
 
 // Helper function that logs the error(s) and exits.
 template <typename... ArgTs> static void ExitWithError(ArgTs &&... Args) {
   ExitOnErr(make_error<Failure>(std::forward<ArgTs>(Args)...));
 }
 
 // Check Err. If it's in a failure state log the file error(s) and exit.
#endif
 
   // Preliminary check to ensure features needed for requested
   // benchmark mode are present on target CPU and/or OS.
   if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure)
     ExitOnErr(State.getExegesisTarget().checkFeatureSupport());
 
   const std::unique_ptr<BenchmarkRunner> Runner =
       ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
-          BenchmarkMode, State, BenchmarkPhaseSelector, ResultAggMode));
+          BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode,
+          ResultAggMode));
   if (!Runner) {
     ExitWithError("cannot create benchmark runner");
   }
 
   const auto Opcodes = getOpcodesOrDie(State);
 
   SmallVector<std::unique_ptr<const SnippetRepetitor>, 2> Repetitors;
   if (RepetitionMode != Benchmark::RepetitionModeE::AggregateMin)