[llvm-exegesis] Refactor FunctionExecutorImpl and create factory

[aidengrossman]

Summary

In order to better support adding in new implementations of
FunctionExecutor, this patch makes some small changes so that it is
easier to add new ones in. FunctionExecutorImpl is renamed to
InProcessFunctionExecutorImpl to better reflect how it will be placed
relative to the soon-to-be introduced subprocess executor and a new
function is created to create executors so selection can be done more
easily. In addition, a new CLI flag, -execution-mode, which can be used
to select between the different executors.

[courbet]

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

195

A runnable configuration exists outside of how it's executed, it feels weird to have this here. Why isn't this a member of BenchmarkRunner ?

204

Why do we need Key ?

205

[nit] use switch, even if it's only one option for now, I think it's more readable for enums.

[aidengrossman]

Address reviewer feedback.

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

195

Good point. I've switched it over to being a member of BenchmarkRunner.

204

I'm using a BenchmarkKey in https://reviews.llvm.org/D151025 (and touching it in some previous patches in the stack) to pass around memory annotations, and the way I have it structured currently, I pass the key to the FunctionExecutorImpl for setup. I can move this to another diff/change this around if desired.

[courbet]

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

128

[nit] createFunctionExecutor ?

[aidengrossman]

Address reviewer feedback.

[aidengrossman]

Format

[aidengrossman]

Update

[aidengrossman]

Rebase

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

 // Common code for all benchmark modes.
 class BenchmarkRunner {
 public:
+  enum ExecutionModeE { InProcess };
+
   explicit BenchmarkRunner(const LLVMState &State,
                            Benchmark::ModeE Mode,
-                           BenchmarkPhaseSelectorE BenchmarkPhaseSelector);
+                           BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                           ExecutionModeE ExecutionMode);
 
   virtual ~BenchmarkRunner();
 

   const LLVMState &State;
   const Benchmark::ModeE Mode;
   const BenchmarkPhaseSelectorE BenchmarkPhaseSelector;
+  const ExecutionModeE ExecutionMode;
 
 private:
   virtual Expected<std::vector<BenchmarkMeasure>>

                                         StringRef FileName) const;
 
   const std::unique_ptr<ScratchSpace> Scratch;
+
+  Expected<std::unique_ptr<FunctionExecutor>>
+  createFunctionExecutor(object::OwningBinary<object::ObjectFile> Obj,
+                         const BenchmarkKey &Key) const;
 };
 
 } // namespace exegesis

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

 
 BenchmarkRunner::BenchmarkRunner(const LLVMState &State,
                                  Benchmark::ModeE Mode,
-                                 BenchmarkPhaseSelectorE BenchmarkPhaseSelector)
+                                 BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                                 ExecutionModeE ExecutionMode)
     : State(State), Mode(Mode), BenchmarkPhaseSelector(BenchmarkPhaseSelector),
-      Scratch(std::make_unique<ScratchSpace>()) {}
+      ExecutionMode(ExecutionMode), Scratch(std::make_unique<ScratchSpace>()) {}
 
 BenchmarkRunner::~BenchmarkRunner() = default;
 

 }
 
 namespace {
-class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
+class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
 public:
-  FunctionExecutorImpl(const LLVMState &State,
+  InProcessFunctionExecutorImpl(const LLVMState &State,
                        object::OwningBinary<object::ObjectFile> Obj,
                        BenchmarkRunner::ScratchSpace *Scratch)
       : State(State), Function(State.createTargetMachine(), std::move(Obj)),

   return std::move(RC);
 }
 
+Expected<std::unique_ptr<BenchmarkRunner::FunctionExecutor>>
+BenchmarkRunner::createFunctionExecutor(
+    object::OwningBinary<object::ObjectFile> ObjectFile,
+    const BenchmarkKey &Key) const {
+  switch (ExecutionMode) {
+  case ExecutionModeE::InProcess:
+    return std::make_unique<InProcessFunctionExecutorImpl>(
+        State, std::move(ObjectFile), Scratch.get());
+  }
+  llvm_unreachable("ExecutionMode is outside expected range");
+}
+
 Expected<Benchmark> BenchmarkRunner::runConfiguration(
     RunnableConfiguration &&RC,
     const std::optional<StringRef> &DumpFile) const {

     return std::move(InstrBenchmark);
   }
 
-  const FunctionExecutorImpl Executor(State, std::move(ObjectFile),
-                                      Scratch.get());
-  auto NewMeasurements = runMeasurements(Executor);
+  Expected<std::unique_ptr<BenchmarkRunner::FunctionExecutor>> Executor =
+      createFunctionExecutor(std::move(ObjectFile), RC.InstrBenchmark.Key);
+  if (!Executor)
+    return Executor.takeError();
+  auto NewMeasurements = runMeasurements(**Executor);
+
   if (Error E = NewMeasurements.takeError()) {
     if (!E.isA<SnippetCrash>())
       return std::move(E);

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h

   LatencyBenchmarkRunner(
       const LLVMState &State, Benchmark::ModeE Mode,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode);
+      Benchmark::ResultAggregationModeE ResultAggMode,
+      ExecutionModeE ExecutionMode);
   ~LatencyBenchmarkRunner() override;
 
 private:

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp

 LatencyBenchmarkRunner::LatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAgg)
-    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector) {
-  assert((Mode == Benchmark::Latency ||
-          Mode == Benchmark::InverseThroughput) &&
+    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode)
+    : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
+  assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
          "invalid mode");
   ResultAggMode = ResultAgg;
 }

llvm/tools/llvm-exegesis/lib/Target.h

   Expected<std::unique_ptr<BenchmarkRunner>> createBenchmarkRunner(
       Benchmark::ModeE Mode, const LLVMState &State,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode =
-          Benchmark::Min) const;
+      BenchmarkRunner::ExecutionModeE ExecutionMode,
+      Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
 
   // Returns the ExegesisTarget for the given triple or nullptr if the target
   // does not exist.

   std::unique_ptr<BenchmarkRunner> virtual createLatencyBenchmarkRunner(
       const LLVMState &State, Benchmark::ModeE Mode,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode) const;
+      Benchmark::ResultAggregationModeE ResultAggMode,
+      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
       const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-      Benchmark::ResultAggregationModeE ResultAggMode) const;
+      Benchmark::ResultAggregationModeE ResultAggMode,
+      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
 
   const ExegesisTarget *Next = nullptr;
   const ArrayRef<CpuAndPfmCounters> CpuPfmCounters;

llvm/tools/llvm-exegesis/lib/Target.cpp

 ExegesisTarget::createBenchmarkRunner(
     Benchmark::ModeE Mode, const LLVMState &State,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+    BenchmarkRunner::ExecutionModeE ExecutionMode,
     Benchmark::ResultAggregationModeE ResultAggMode) const {
   PfmCountersInfo PfmCounters = State.getPfmCounters();
   switch (Mode) {

                   "the kernel for real event counts."));
     }
     return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector,
-                                        ResultAggMode);
+                                        ResultAggMode, ExecutionMode);
   case Benchmark::Uops:
     if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
         !PfmCounters.UopsCounter && !PfmCounters.IssueCounters)

           "benchmarking or --use-dummy-perf-counters to not query the kernel "
           "for real event counts.");
     return createUopsBenchmarkRunner(State, BenchmarkPhaseSelector,
-                                     ResultAggMode);
+                                     ResultAggMode, ExecutionMode);
   }
   return nullptr;
 }

 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAggMode) const {
+    Benchmark::ResultAggregationModeE ResultAggMode,
+    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
   return std::make_unique<LatencyBenchmarkRunner>(
-      State, Mode, BenchmarkPhaseSelector, ResultAggMode);
+      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode);
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
     const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE /*unused*/) const {
-  return std::make_unique<UopsBenchmarkRunner>(State, BenchmarkPhaseSelector);
+    Benchmark::ResultAggregationModeE /*unused*/,
+    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
+  return std::make_unique<UopsBenchmarkRunner>(State, BenchmarkPhaseSelector,
+                                               ExecutionMode);
 }
 
 static_assert(std::is_trivial_v<PfmCountersInfo>,

llvm/tools/llvm-exegesis/lib/UopsBenchmarkRunner.h

 class UopsBenchmarkRunner : public BenchmarkRunner {
 public:
   UopsBenchmarkRunner(const LLVMState &State,
-                      BenchmarkPhaseSelectorE BenchmarkPhaseSelector)
-      : BenchmarkRunner(State, Benchmark::Uops,
-                        BenchmarkPhaseSelector) {}
+                      BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
+                      ExecutionModeE ExecutionMode)
+      : BenchmarkRunner(State, Benchmark::Uops, BenchmarkPhaseSelector,
+                        ExecutionMode) {}
   ~UopsBenchmarkRunner() override;
 
   static constexpr const size_t kMinNumDifferentAddresses = 6;

llvm/tools/llvm-exegesis/llvm-exegesis.cpp

                               "and prints a message to access it"),
                      cl::ValueOptional, cl::cat(BenchmarkOptions));
 
+static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
+    "execution-mode",
+    cl::desc("Selects the execution mode to use for running snippets"),
+    cl::cat(BenchmarkOptions),
+    cl::values(clEnumValN(BenchmarkRunner::ExecutionModeE::InProcess,
+                          "inprocess",
+                          "Executes the snippets within the same process")),
+    cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
+
 static ExitOnError ExitOnErr("llvm-exegesis error: ");
 
 // Helper function that logs the error(s) and exits.

 
   const std::unique_ptr<BenchmarkRunner> Runner =
       ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
-          BenchmarkMode, State, BenchmarkPhaseSelector, ResultAggMode));
+          BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode,
+          ResultAggMode));
   if (!Runner) {
     ExitWithError("cannot create benchmark runner");
   }