[mlir][NFC] Use options struct in ExecutionEngine::create

[cota]

Summary

Its number of optional parameters has grown too large,
which makes adding new optional parameters quite a chore.

Fix this by using an options struct.

[mehdi_amini]

mlir/include/mlir/ExecutionEngine/ExecutionEngine.h

73

Can you improve the comment to explain where will the cache be located?

[cota]

mlir/include/mlir/ExecutionEngine/ExecutionEngine.h

73

Can you please suggest a rephrase? I am struggling to see how the location of the cache is unclear given the comment, but perhaps I'm missing something. Thanks!

[mehdi_amini]

mlir/include/mlir/ExecutionEngine/ExecutionEngine.h

73

Aren't the object will be stored on disk? How is the user turning this boolean to true supposed to get access to the objects?

I'm also not sure why this is true by default?

[cota]

mlir/include/mlir/ExecutionEngine/ExecutionEngine.h

73

Thanks for clarifying.
The enableObjectCache option was added in https://reviews.llvm.org/rG06e8101034e, defaulting to false. However, the init code added there got its logic reversed (cache(enableObjectCache ? nullptr : new SimpleObjectCache()), which was fixed in https://reviews.llvm.org/rGd1186fcb04 by setting the default to true (i.e. preserving the existing behavior even if it perhaps was unintentional).
I'll ask the authors of those patches about what the default should be and will get back to this in a follow-up patch.
In the meantime I'll commit this patch since I have other changes pending on it.

mlir/examples/toy/Ch6/toyc.cpp

int runJit(mlir::ModuleOp module) {
 
   // An optimization pipeline to use within the execution engine.
   auto optPipeline = mlir::makeOptimizingTransformer(
       /*optLevel=*/enableOpt ? 3 : 0, /*sizeLevel=*/0,
       /*targetMachine=*/nullptr);
 
   // Create an MLIR execution engine. The execution engine eagerly JIT-compiles
   // the module.
-  auto maybeEngine = mlir::ExecutionEngine::create(
-      module, /*llvmModuleBuilder=*/nullptr, optPipeline);
+  mlir::ExecutionEngineOptions engineOptions;
+  engineOptions.transformer = optPipeline;
+  auto maybeEngine = mlir::ExecutionEngine::create(module, engineOptions);
   assert(maybeEngine && "failed to construct an execution engine");
   auto &engine = maybeEngine.get();
 
   // Invoke the JIT-compiled function.
   auto invocationResult = engine->invokePacked("main");
   if (invocationResult) {
     llvm::errs() << "JIT invocation failed\n";
     return -1;

mlir/examples/toy/Ch7/toyc.cpp

int runJit(mlir::ModuleOp module) {
 
   // An optimization pipeline to use within the execution engine.
   auto optPipeline = mlir::makeOptimizingTransformer(
       /*optLevel=*/enableOpt ? 3 : 0, /*sizeLevel=*/0,
       /*targetMachine=*/nullptr);
 
   // Create an MLIR execution engine. The execution engine eagerly JIT-compiles
   // the module.
-  auto maybeEngine = mlir::ExecutionEngine::create(
-      module, /*llvmModuleBuilder=*/nullptr, optPipeline);
+  mlir::ExecutionEngineOptions engineOptions;
+  engineOptions.transformer = optPipeline;
+  auto maybeEngine = mlir::ExecutionEngine::create(module, engineOptions);
   assert(maybeEngine && "failed to construct an execution engine");
   auto &engine = maybeEngine.get();
 
   // Invoke the JIT-compiled function.
   auto invocationResult = engine->invokePacked("main");
   if (invocationResult) {
     llvm::errs() << "JIT invocation failed\n";
     return -1;

mlir/include/mlir/ExecutionEngine/ExecutionEngine.h

public:
 
   /// Dump cached object to output file `filename`.
   void dumpToObjectFile(StringRef filename);
 
 private:
   llvm::StringMap<std::unique_ptr<llvm::MemoryBuffer>> cachedObjects;
 };
 
+struct ExecutionEngineOptions {
+  /// If `llvmModuleBuilder` is provided, it will be used to create LLVM module
+  /// from the given MLIR module. Otherwise, a default `translateModuleToLLVMIR`
+  /// function will be used to translate MLIR module to LLVM IR.
+  llvm::function_ref<std::unique_ptr<llvm::Module>(ModuleOp,
+                                                   llvm::LLVMContext &)>
+      llvmModuleBuilder = nullptr;
+
+  /// If `transformer` is provided, it will be called on the LLVM module during
+  /// JIT-compilation and can be used, e.g., for reporting or optimization.
+  llvm::function_ref<llvm::Error(llvm::Module *)> transformer = {};
+
+  /// `jitCodeGenOptLevel`, when provided, is used as the optimization level for
+  /// target code generation.
+  Optional<llvm::CodeGenOpt::Level> jitCodeGenOptLevel = llvm::None;
+
+  /// If `sharedLibPaths` are provided, the underlying JIT-compilation will
+  /// open and link the shared libraries for symbol resolution.
+  ArrayRef<StringRef> sharedLibPaths = {};
+
+  /// If `enableObjectCache` is set, the JIT compiler will create one to store
+  /// the object generated for the given module.
+  bool enableObjectCache = true;
+
+  /// If enable `enableGDBNotificationListener` is set, the JIT compiler will
+  /// notify the llvm's global GDB notification listener.
+  bool enableGDBNotificationListener = true;
+
+  /// If `enablePerfNotificationListener` is set, the JIT compiler will notify
+  /// the llvm's global Perf notification listener.
+  bool enablePerfNotificationListener = true;
+};
+
 /// JIT-backed execution engine for MLIR modules.  Assumes the module can be
 /// converted to LLVM IR.  For each function, creates a wrapper function with
 /// the fixed interface
 ///
 ///     void _mlir_funcName(void **)
 ///
 /// where the only argument is interpreted as a list of pointers to the actual
 /// arguments of the function, followed by a pointer to the result.  This allows
 /// the engine to provide the caller with a generic function pointer that can
 /// be used to invoke the JIT-compiled function.
 class ExecutionEngine {
 public:
   ExecutionEngine(bool enableObjectCache, bool enableGDBNotificationListener,
                   bool enablePerfNotificationListener);
 
   /// Creates an execution engine for the given module.
-  ///
-  /// If `llvmModuleBuilder` is provided, it will be used to create LLVM module
-  /// from the given MLIR module. Otherwise, a default `translateModuleToLLVMIR`
-  /// function will be used to translate MLIR module to LLVM IR.
-  ///
-  /// If `transformer` is provided, it will be called on the LLVM module during
-  /// JIT-compilation and can be used, e.g., for reporting or optimization.
-  ///
-  /// `jitCodeGenOptLevel`, when provided, is used as the optimization level for
-  /// target code generation.
-  ///
-  /// If `sharedLibPaths` are provided, the underlying JIT-compilation will
-  /// open and link the shared libraries for symbol resolution.
-  ///
-  /// If `enableObjectCache` is set, the JIT compiler will create one to store
-  /// the object generated for the given module.
-  ///
-  /// If enable `enableGDBNotificationListener` is set, the JIT compiler will
-  /// notify the llvm's global GDB notification listener.
-  ///
-  /// If `enablePerfNotificationListener` is set, the JIT compiler will notify
-  /// the llvm's global Perf notification listener.
   static llvm::Expected<std::unique_ptr<ExecutionEngine>>
-  create(ModuleOp m,
-         llvm::function_ref<std::unique_ptr<llvm::Module>(ModuleOp,
-                                                          llvm::LLVMContext &)>
-             llvmModuleBuilder = nullptr,
-         llvm::function_ref<llvm::Error(llvm::Module *)> transformer = {},
-         Optional<llvm::CodeGenOpt::Level> jitCodeGenOptLevel = llvm::None,
-         ArrayRef<StringRef> sharedLibPaths = {}, bool enableObjectCache = true,
-         bool enableGDBNotificationListener = true,
-         bool enablePerfNotificationListener = true);
+  create(ModuleOp m, const ExecutionEngineOptions &options = {});
 
   /// Looks up a packed-argument function wrapping the function with the given
   /// name and returns a pointer to it. Propagates errors in case of failure.
   llvm::Expected<void (*)(void **)> lookupPacked(StringRef name) const;
 
   /// Looks up the original function with the given name and returns a
   /// pointer to it. This is not necesarily a packed function. Propagates
   /// errors in case of failure.

mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp

mlirExecutionEngineCreate(MlirModule op, int optLevel, int numPaths,
   for (unsigned i = 0; i < static_cast<unsigned>(numPaths); ++i)
     libPaths.push_back(sharedLibPaths[i].data);
 
   // Create a transformer to run all LLVM optimization passes at the
   // specified optimization level.
   auto llvmOptLevel = static_cast<llvm::CodeGenOpt::Level>(optLevel);
   auto transformer = mlir::makeLLVMPassesTransformer(
       /*passes=*/{}, llvmOptLevel, /*targetMachine=*/tmOrError->get());
-  auto jitOrError =
-      ExecutionEngine::create(unwrap(op), /*llvmModuleBuilder=*/{}, transformer,
-                              llvmOptLevel, libPaths);
+  ExecutionEngineOptions jitOptions;
+  jitOptions.transformer = transformer;
+  jitOptions.jitCodeGenOptLevel = llvmOptLevel;
+  jitOptions.sharedLibPaths = libPaths;
+  auto jitOrError = ExecutionEngine::create(unwrap(op), jitOptions);
   if (!jitOrError) {
     consumeError(jitOrError.takeError());
     return MlirExecutionEngine{nullptr};
   }
   return wrap(jitOrError->release());
 }
 
 extern "C" void mlirExecutionEngineDestroy(MlirExecutionEngine jit) {

mlir/lib/ExecutionEngine/ExecutionEngine.cpp

if (enablePerfNotificationListener) {
     if (auto *listener = llvm::JITEventListener::createPerfJITEventListener())
       perfListener = listener;
     else if (auto *listener =
                  llvm::JITEventListener::createIntelJITEventListener())
       perfListener = listener;
   }
 }
 
-Expected<std::unique_ptr<ExecutionEngine>> ExecutionEngine::create(
-    ModuleOp m,
-    llvm::function_ref<std::unique_ptr<llvm::Module>(ModuleOp,
-                                                     llvm::LLVMContext &)>
-        llvmModuleBuilder,
-    llvm::function_ref<Error(llvm::Module *)> transformer,
-    Optional<llvm::CodeGenOpt::Level> jitCodeGenOptLevel,
-    ArrayRef<StringRef> sharedLibPaths, bool enableObjectCache,
-    bool enableGDBNotificationListener, bool enablePerfNotificationListener) {
+Expected<std::unique_ptr<ExecutionEngine>>
+ExecutionEngine::create(ModuleOp m, const ExecutionEngineOptions &options) {
   auto engine = std::make_unique<ExecutionEngine>(
-      enableObjectCache, enableGDBNotificationListener,
-      enablePerfNotificationListener);
+      options.enableObjectCache, options.enableGDBNotificationListener,
+      options.enablePerfNotificationListener);
 
   std::unique_ptr<llvm::LLVMContext> ctx(new llvm::LLVMContext);
-  auto llvmModule = llvmModuleBuilder ? llvmModuleBuilder(m, *ctx)
-                                      : translateModuleToLLVMIR(m, *ctx);
+  auto llvmModule = options.llvmModuleBuilder
+                        ? options.llvmModuleBuilder(m, *ctx)
   if (!llvmModule)
     return makeStringError("could not convert to LLVM IR");
   // FIXME: the triple should be passed to the translation or dialect conversion
   // instead of this.  Currently, the LLVM module created above has no triple
   // associated with it.
   setupTargetTriple(llvmModule.get());
   packFunctionArguments(llvmModule.get());
 
auto objectLinkingLayerCreator = [&](ExecutionSession &session,
     // cf llvm/lib/ExecutionEngine/Orc/LLJIT.cpp LLJIT::createObjectLinkingLayer
     llvm::Triple targetTriple(llvm::Twine(llvmModule->getTargetTriple()));
     if (targetTriple.isOSBinFormatCOFF()) {
       objectLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
       objectLayer->setAutoClaimResponsibilityForObjectSymbols(true);
     }
 
     // Resolve symbols from shared libraries.
-    for (auto libPath : sharedLibPaths) {
+    for (auto libPath : options.sharedLibPaths) {
       auto mb = llvm::MemoryBuffer::getFile(libPath);
       if (!mb) {
         errs() << "Failed to create MemoryBuffer for: " << libPath
                << "\nError: " << mb.getError().message() << "\n";
         continue;
       }
       auto &jd = session.createBareJITDylib(std::string(libPath));
       auto loaded = DynamicLibrarySearchGenerator::Load(
auto objectLinkingLayerCreator = [&](ExecutionSession &session,
 
     return objectLayer;
   };
 
   // Callback to inspect the cache and recompile on demand. This follows Lang's
   // LLJITWithObjectCache example.
   auto compileFunctionCreator = [&](JITTargetMachineBuilder jtmb)
       -> Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> {
-    if (jitCodeGenOptLevel)
-      jtmb.setCodeGenOptLevel(jitCodeGenOptLevel.getValue());
+    if (options.jitCodeGenOptLevel)
+      jtmb.setCodeGenOptLevel(options.jitCodeGenOptLevel.getValue());
     auto tm = jtmb.createTargetMachine();
     if (!tm)
       return tm.takeError();
     return std::make_unique<TMOwningSimpleCompiler>(std::move(*tm),
                                                     engine->cache.get());
   };
 
   // Create the LLJIT by calling the LLJITBuilder with 2 callbacks.
   auto jit =
       cantFail(llvm::orc::LLJITBuilder()
                    .setCompileFunctionCreator(compileFunctionCreator)
                    .setObjectLinkingLayerCreator(objectLinkingLayerCreator)
                    .create());
 
   // Add a ThreadSafemodule to the engine and return.
   ThreadSafeModule tsm(std::move(llvmModule), std::move(ctx));
-  if (transformer)
+  if (options.transformer)
     cantFail(tsm.withModuleDo(
-        [&](llvm::Module &module) { return transformer(&module); }));
+        [&](llvm::Module &module) { return options.transformer(&module); }));
   cantFail(jit->addIRModule(std::move(tsm)));
   engine->jit = std::move(jit);
 
   // Resolve symbols that are statically linked in the current process.
   llvm::orc::JITDylib &mainJD = engine->jit->getMainJITDylib();
   mainJD.addGenerator(
       cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess(
           dataLayout.getGlobalPrefix())));

mlir/lib/ExecutionEngine/JitRunner.cpp

auto runtimeSymbolMap = [&](llvm::orc::MangleAndInterner interner) {
     auto symbolMap = config.runtimeSymbolMap ? config.runtimeSymbolMap(interner)
                                              : llvm::orc::SymbolMap();
     for (auto &exportSymbol : exportSymbols)
       symbolMap[interner(exportSymbol.getKey())] =
           llvm::JITEvaluatedSymbol::fromPointer(exportSymbol.getValue());
     return symbolMap;
   };
 
-  auto expectedEngine = mlir::ExecutionEngine::create(
-      module, config.llvmModuleBuilder, config.transformer, jitCodeGenOptLevel,
-      executionEngineLibs);
+  mlir::ExecutionEngineOptions engineOptions;
+  engineOptions.llvmModuleBuilder = config.llvmModuleBuilder;
+  engineOptions.transformer = config.transformer;
+  engineOptions.jitCodeGenOptLevel = jitCodeGenOptLevel;
+  engineOptions.sharedLibPaths = executionEngineLibs;
+  auto expectedEngine = mlir::ExecutionEngine::create(module, engineOptions);
   if (!expectedEngine)
     return expectedEngine.takeError();
 
   auto engine = std::move(*expectedEngine);
   engine->registerSymbols(runtimeSymbolMap);
 
   auto expectedFPtr = engine->lookupPacked(entryPoint);
   if (!expectedFPtr)