[WIP] [MLIR] [RFC] Async dialect targeting LLVM coroutines

[ezhulenev]

Summary

[WIP] [MLIR] [RFC] Async dialect targeting LLVM coroutines

[mehdi_amini]

mlir/test/mlir-cpu-runner/async.mlir

75

I don't quite get what guarantee we have that it'll print printed before print123? Aren't these running concurrently and any order is possible?

[ezhulenev]

mlir/test/mlir-cpu-runner/async.mlir

75

Yes, this is a bad test, 123 could be printed first in theory.

[mehdi_amini]

What's the status here? Is this ready for review or do you have some update to do after the public meeting? Are we waiting on some of the things on Stephan's side to converge/land?
Are there some high-level design discussion we still need to have?

[ezhulenev]

In D82704#2186560, @mehdi_amini wrote:

What's the status here?

POC that will never be submitted as is (can't find a button to revert this PR).

When we'll converge to some common IR design with Stephan I'll do the work in separate PRs, this one will be just a reference for the new implementation.

[rriddle]

In D82704#2187834, @ezhulenev wrote:

In D82704#2186560, @mehdi_amini wrote:

What's the status here?

POC that will never be submitted as is (can't find a button to revert this PR).

(Add Action -> Abandon Revision)

[mehdi_amini]

LG, thanks!

mlir/include/mlir/Conversion/AsyncToLLVM/ConvertAsyncToLLVM.h

+//===- ConvertAsyncToLLVM.h - Conversion Patterns from Async to LLVM ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_ASYNCTOLLVM_CONVERTASYNCTOLLVM_H_
+#define MLIR_CONVERSION_ASYNCTOLLVM_CONVERTASYNCTOLLVM_H_
+
+#include <memory>
+
+namespace mlir {
+class ModuleOp;
+template <typename T>
+class OperationPass;
+class OwningRewritePatternList;
+
+/// Collect a set of patterns to convert from the Async dialect to LLVM.
+void populateAsyncToLLVMConversionPatterns(OwningRewritePatternList &patterns);
+
+/// Create a pass to convert Async operations to the LLVMIR dialect.
+std::unique_ptr<OperationPass<ModuleOp>> createConvertAsyncToLLVMPass();
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_ASYNCTOLLVM_CONVERTASYNCTOLLVM_H_

mlir/include/mlir/Conversion/Passes.td

 
 def ConvertAVX512ToLLVM : Pass<"convert-avx512-to-llvm", "ModuleOp"> {
   let summary = "Convert the operations from the avx512 dialect into the LLVM "
                 "dialect";
   let constructor = "mlir::createConvertAVX512ToLLVMPass()";
 }
 
 //===----------------------------------------------------------------------===//
+// AsyncToLLVM
+//===----------------------------------------------------------------------===//
+
+def ConvertAsyncToLLVM : Pass<"convert-async-to-llvm", "ModuleOp"> {
+  let summary = "Convert the operations from the async dialect into the LLVM "
+                "dialect";
+  let constructor = "mlir::createConvertAsyncToLLVMPass()";
+}
+
+//===----------------------------------------------------------------------===//
 // GPUCommon
 //===----------------------------------------------------------------------===//
 
 def ConvertGpuLaunchFuncToGpuRuntimeCalls : Pass<"launch-func-to-gpu-runtime",
                                                  "ModuleOp"> {
   let summary = "Convert all launch_func ops to GPU runtime calls";
   let constructor = "mlir::createConvertGpuLaunchFuncToGpuRuntimeCallsPass()";
   let options = [

mlir/include/mlir/Dialect/Async/Async.h

+//===- Async.h - Asynchronous Operations ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines asynchronous operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ASYNC_H_
+#define MLIR_DIALECT_ASYNC_H_
+
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+
+namespace mlir {
+namespace async {
+
+namespace AsyncTypes {
+enum Kind {
+  Runtime = Type::FIRST_SHAPE_TYPE,
+  Handle,
+  LAST_SHAPE_TYPE = Handle
+};
+} // namespace AsyncTypes
+
+class RuntimeType : public Type::TypeBase<RuntimeType, Type> {
+public:
+  using Base::Base;
+
+  static RuntimeType get(MLIRContext *context) {
+    return Base::get(context, AsyncTypes::Runtime);
+  }
+  /// Used to implement llvm-style cast.
+  static bool kindof(unsigned kind) { return kind == AsyncTypes::Runtime; }
+};
+
+class HandleType : public Type::TypeBase<HandleType, Type> {
+public:
+  using Base::Base;
+
+  static HandleType get(MLIRContext *context) {
+    return Base::get(context, AsyncTypes::Handle);
+  }
+  /// Used to implement llvm-style cast.
+  static bool kindof(unsigned kind) { return kind == AsyncTypes::Handle; }
+};
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Async/AsyncOps.h.inc"
+
+#include "mlir/Dialect/Async/AsyncOpsDialect.h.inc"
+
+} // end namespace async
+} // end namespace mlir
+#endif // MLIR_DIALECT_ASYNC_H_

mlir/include/mlir/Dialect/Async/AsyncOps.td

+//===- AsyncOps.td - Asynchronous operations ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines MLIR asynchronous operations
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ASYNC_ASYNCOPS
+#define MLIR_DIALECT_ASYNC_ASYNCOPS
+
+include "mlir/IR/OpBase.td"
+
+//===----------------------------------------------------------------------===//
+// Async Dialect definition
+//===----------------------------------------------------------------------===//
+
+def Async_Dialect : Dialect {
+  let name = "async";
+  let cppNamespace = "";
+}
+
+//===----------------------------------------------------------------------===//
+// Async type definitions
+//===----------------------------------------------------------------------===//
+
+def AsyncIsRuntimeTypePred : CPred<"$_self.isa<RuntimeType>()">;
+def AsyncRuntimeType : DialectType<Async_Dialect, AsyncIsRuntimeTypePred,
+                                  "runtime">;
+
+def AsyncIsHandleTypePred : CPred<"$_self.isa<HandleType>()">;
+def AsyncHandleType : DialectType<Async_Dialect, AsyncIsHandleTypePred,
+                                  "handle">;
+
+//===----------------------------------------------------------------------===//
+// Async op definitions
+//===----------------------------------------------------------------------===//
+
+class Async_Op<string mnemonic, list<OpTrait> traits = []> :
+  Op<Async_Dialect, mnemonic, !listconcat(traits, [IsolatedFromAbove])> {
+
+  // Each registered op in the Async dialect needs to provide all
+  // of a printer, parser and verifier.
+  let parser = [{ return mlir::tfrt::parse$cppClass(parser, result); }];
+}
+
+def AsyncDefaultRuntimeOp : Async_Op<"default_runtime"> {
+  let summary = "Async default_runtime operation";
+  let description = [{
+  }];
+
+  let arguments = (ins );
+  let results = (outs AsyncRuntimeType:$runtime);
+
+  let assemblyFormat = [{
+    attr-dict `:` type($runtime)
+  }];
+}
+
+def AsyncCreateHandleOp : Async_Op<"create_handle"> {
+  let summary = "Async create_handle operation";
+  let description = [{
+  }];
+
+  let arguments = (ins AsyncRuntimeType:$runtime);
+  let results = (outs AsyncHandleType:$handle);
+
+  let assemblyFormat = [{
+    $runtime attr-dict `:` functional-type($runtime, $handle)
+  }];
+}
+
+def AsyncEmplaceHandleOp : Async_Op<"emplace_handle"> {
+  let summary = "Async emplace_handle operation";
+  let description = [{
+  }];
+
+  let arguments = (ins AsyncHandleType:$handle);
+  let results = (outs);
+
+  let assemblyFormat = [{
+    $handle attr-dict `:` type($handle)
+  }];
+}
+
+def AsyncCallOp : Async_Op<"call"> {
+  let summary = "Async call operation";
+  let description = [{
+  }];
+
+  let arguments = (ins AsyncRuntimeType:$runtime,
+                       FlatSymbolRefAttr:$callee,
+                       Variadic<AnyType>:$operands);
+  let results = (outs AsyncHandleType:$handle);
+
+  let assemblyFormat = [{
+    $runtime `:` type($runtime) $callee `(` $operands `)` attr-dict `:` functional-type($operands, results)
+  }];
+}
+
+def AsyncAwaitOp : Async_Op<"await"> {
+  let summary = "Async await operation";
+  let description = [{
+  }];
+
+  let arguments = (ins AsyncHandleType:$handle);
+  let results = (outs);
+
+  let assemblyFormat = [{
+    $handle attr-dict `:` type($handle)
+  }];
+}
+
+def AsyncSyncAwaitOp : Async_Op<"sync_await"> {
+  let summary = "Async sync_await operation";
+  let description = [{
+  }];
+
+  let arguments = (ins AsyncHandleType:$handle);
+  let results = (outs);
+
+  let assemblyFormat = [{
+    $handle attr-dict `:` type($handle)
+  }];
+}
+
+#endif  // MLIR_DIALECT_ASYNC_ASYNCOPS

mlir/include/mlir/Dialect/Async/CMakeLists.txt

+add_mlir_dialect(AsyncOps async Ops)
+add_mlir_doc(AsyncOps -gen-dialect-doc AsyncDialect Dialects/)

mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h

getStructTy(LLVMType elt1, Args... elts) {
     return getStructTy(&elt1.getDialect(), fields);
   }
   static LLVMType getVectorTy(LLVMType elementType, unsigned numElements);
 
   /// Void type utilities.
   static LLVMType getVoidTy(LLVMDialect *dialect);
   bool isVoidTy();
 
+  /// Token type utilities.
+  static LLVMType getTokenTy(LLVMDialect *dialect);
+  bool isTokenTy();
+
   // Creation and setting of LLVM's identified struct types
   static LLVMType createStructTy(LLVMDialect *dialect,
                                  ArrayRef<LLVMType> elements,
                                  Optional<StringRef> name,
                                  bool isPacked = false);
 
   static LLVMType createStructTy(LLVMDialect *dialect,
                                  Optional<StringRef> name) {

mlir/include/mlir/ExecutionEngine/AsyncRuntime.h

+//===- AsyncRuntime.h - Async runtime reference implementation ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares basic Async runtime API for supporting Async dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef EXECUTIONENGINE_ASYNCRUNTIME_H_
+#define EXECUTIONENGINE_ASYNCRUNTIME_H_
+
+#ifdef _WIN32
+#ifndef MLIR_ASYNCRUNTIME_EXPORT
+#ifdef mlir_c_runner_utils_EXPORTS
+/* We are building this library */
+#define MLIR_ASYNCRUNTIME_EXPORT __declspec(dllexport)
+#define MLIR_ASYNCRUNTIME_DEFINE_FUNCTIONS
+#else
+/* We are using this library */
+#define MLIR_ASYNCRUNTIME_EXPORT __declspec(dllimport)
+#endif // mlir_c_runner_utils_EXPORTS
+#endif // MLIR_ASYNCRUNTIME_EXPORT
+#else
+#define MLIR_ASYNCRUNTIME_EXPORT
+#define MLIR_ASYNCRUNTIME_DEFINE_FUNCTIONS
+#endif // _WIN32
+
+//===----------------------------------------------------------------------===//
+// MLIRAsync runtime API.
+//===----------------------------------------------------------------------===//
+// clang-format off
+
+typedef struct MLIR_AsyncRuntime MLIR_AsyncRuntime;
+typedef struct MLIR_AsyncHandle MLIR_AsyncHandle;
+
+using TaskFunction = void (*)();     // asynchronous task task function
+using CoroHandle = void *;           // coroutine handle
+using CoroResume = void (*)(void *); // coroutine resume function
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+MLIR_AsyncRuntime *MLIR_AsyncRT_DefaultRuntime();
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+MLIR_AsyncHandle *MLIR_AsyncRT_Call(MLIR_AsyncRuntime *, TaskFunction);
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+void MLIR_AsyncRT_Await(MLIR_AsyncHandle *, CoroHandle, CoroResume);
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+void MLIR_AsyncRT_SyncAwait(MLIR_AsyncHandle *);
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+MLIR_AsyncHandle * MLIR_AsyncRT_CreateHandle(MLIR_AsyncRuntime *);
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT
+void MLIR_AsyncRT_EmplaceHandle(MLIR_AsyncHandle *);
+
+// clang-format on
+#endif // EXECUTIONENGINE_ASYNCRUNTIME_H_

mlir/include/mlir/InitAllDialects.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef MLIR_INITALLDIALECTS_H_
 #define MLIR_INITALLDIALECTS_H_
 
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Async/Async.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/Dialect/Quant/QuantOps.h"
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SDBM/SDBMDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/IR/Dialect.h"
 
 namespace mlir {
 
 // This function should be called before creating any MLIRContext if one expect
 // all the possible dialects to be made available to the context automatically.
 inline void registerAllDialects() {
   static bool init_once = []() {
     registerDialect<AffineDialect>();
     registerDialect<avx512::AVX512Dialect>();
+    registerDialect<async::AsyncDialect>();
     registerDialect<gpu::GPUDialect>();
     registerDialect<LLVM::LLVMAVX512Dialect>();
     registerDialect<LLVM::LLVMDialect>();
     registerDialect<linalg::LinalgDialect>();
     registerDialect<scf::SCFDialect>();
     registerDialect<omp::OpenMPDialect>();
     registerDialect<quant::QuantizationDialect>();
     registerDialect<spirv::SPIRVDialect>();

mlir/include/mlir/InitAllPasses.h

 // passes to the system.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef MLIR_INITALLPASSES_H_
 #define MLIR_INITALLPASSES_H_
 
 #include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
+#include "mlir/Conversion/AsyncToLLVM/ConvertAsyncToLLVM.h"
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
 #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
 #include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
 #include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
 #include "mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h"
 #include "mlir/Conversion/LinalgToStandard/LinalgToStandard.h"

mlir/lib/Conversion/AsyncToLLVM/CMakeLists.txt

+add_mlir_conversion_library(MLIRASYNCToLLVM
+  ConvertAsyncToLLVM.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AsyncToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRASYNC
+  MLIRLLVMIR
+  MLIRStandardToLLVM
+  MLIRTransforms
+  )

mlir/lib/Conversion/AsyncToLLVM/ConvertAsyncToLLVM.cpp

+//===- ConvertAsyncToLLVM.cpp - Convert Async to the LLVM dialect ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Each function that has `async.await` operation in the function body converted
+// to a coroutine. See https://llvm.org/docs/Coroutines.html for documentation.
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Conversion/AsyncToLLVM/ConvertAsyncToLLVM.h"
+
+#include "../PassDetail.h"
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
+#include "mlir/Dialect/Async/Async.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/Module.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+using namespace mlir::async;
+
+namespace {
+static constexpr const char *kResume = "__resume";
+static constexpr const char *kMalloc = "malloc";
+static constexpr const char *kFree = "free";
+
+// clang-format off
+static constexpr const char *kAsyncCall      = "MLIR_AsyncRT_Call";
+static constexpr const char *kAwait          = "MLIR_AsyncRT_Await";
+static constexpr const char *kSyncAwait      = "MLIR_AsyncRT_SyncAwait";
+static constexpr const char *kCreateHandle   = "MLIR_AsyncRT_CreateHandle";
+static constexpr const char *kEmplaceHandle  = "MLIR_AsyncRT_EmplaceHandle";
+static constexpr const char *kDefaultRuntime = "MLIR_AsyncRT_DefaultRuntime";
+
+static constexpr const char* kCoroId         = "llvm.coro.id";
+static constexpr const char* kCoroSizeI64    = "llvm.coro.size.i64";
+static constexpr const char* kCoroBegin      = "llvm.coro.begin";
+static constexpr const char* kCoroSave       = "llvm.coro.save";
+static constexpr const char* kCoroSuspend    = "llvm.coro.suspend";
+static constexpr const char* kCoroEnd        = "llvm.coro.end";
+static constexpr const char* kCoroFree       = "llvm.coro.free";
+static constexpr const char* kCoroResume     = "llvm.coro.resume";
+// clang-format on
+
+struct ConvertAsyncToLLVMPass
+    : public ConvertAsyncToLLVMBase<ConvertAsyncToLLVMPass> {
+  void runOnOperation() override;
+};
+
+// Function targeted for coroutine transformation has two additional blocks at
+// the end: coroutine cleanup and coroutine suspension.
+//
+// async.await op lowering additionaly creates a resume block for each
+// operation.
+struct CoroMachinery {
+  Value coroHandle;
+  Block *cleanup;
+  Block *suspend;
+};
+
+/// LLVM function types for Async Runtime API.
+struct API {
+  static LLVM::LLVMType resumeFunctionType(LLVM::LLVMDialect *llvm) {
+    auto voidTy = LLVM::LLVMType::getVoidTy(llvm);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(voidTy, {i8Ptr}, false);
+  }
+
+  static LLVM::LLVMType asyncFunctionType(LLVM::LLVMDialect *llvm) {
+    auto voidTy = LLVM::LLVMType::getVoidTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(voidTy, {}, false);
+  }
+
+  static LLVM::LLVMType asyncCallFunctionType(LLVM::LLVMDialect *llvm) {
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    auto calleePtr = asyncFunctionType(llvm).getPointerTo();
+    return LLVM::LLVMType::getFunctionTy(i8Ptr, {i8Ptr, calleePtr}, false);
+  }
+
+  static LLVM::LLVMType awaitFunctionType(LLVM::LLVMDialect *llvm) {
+    auto voidTy = LLVM::LLVMType::getVoidTy(llvm);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    auto resumePtr = resumeFunctionType(llvm).getPointerTo();
+    return LLVM::LLVMType::getFunctionTy(voidTy, {i8Ptr, i8Ptr, resumePtr},
+                                         false);
+  }
+
+  static LLVM::LLVMType syncAwaitFunctionType(LLVM::LLVMDialect *llvm) {
+    auto voidTy = LLVM::LLVMType::getVoidTy(llvm);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(voidTy, {i8Ptr}, false);
+  }
+
+  static LLVM::LLVMType createHandleFunctionType(LLVM::LLVMDialect *llvm) {
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(i8Ptr, {i8Ptr}, false);
+  }
+
+  static LLVM::LLVMType emplaceHandleFunctionType(LLVM::LLVMDialect *llvm) {
+    auto voidTy = LLVM::LLVMType::getVoidTy(llvm);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(voidTy, {i8Ptr}, false);
+  }
+
+  static LLVM::LLVMType defaultRuntimeFunctionType(LLVM::LLVMDialect *llvm) {
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    return LLVM::LLVMType::getFunctionTy(i8Ptr, {}, false);
+  }
+};
+
+LLVM::LLVMDialect *LlvmDialect(MLIRContext *ctx) {
+  return ctx->getRegisteredDialect<LLVM::LLVMDialect>();
+}
+
+//===----------------------------------------------------------------------===//
+// Convert Async dialect types to opaque LLVM pointers.
+//===----------------------------------------------------------------------===//
+
+class AsyncRuntimeTypeConverted : public TypeConverter {
+public:
+  AsyncRuntimeTypeConverted() {
+    addConversion([](Type type) {
+      auto *llvm = LlvmDialect(type.getContext());
+      auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+
+      if (type.isa<RuntimeType>() || type.isa<HandleType>())
+        return Type(i8Ptr);
+
+      return type;
+    });
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// async.call op lowering to MLIR_AsyncRT_Call function call.
+//===----------------------------------------------------------------------===//
+
+class AsyncCallOpLowering : public ConversionPattern {
+public:
+  explicit AsyncCallOpLowering(MLIRContext *ctx)
+      : ConversionPattern(AsyncCallOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    AsyncCallOp async_call = cast<AsyncCallOp>(op);
+
+    MLIRContext *ctx = op->getContext();
+    Location loc = op->getLoc();
+
+    ModuleOp module = op->getParentOfType<ModuleOp>();
+    FuncOp callee = module.lookupSymbol<FuncOp>(async_call.callee());
+
+    auto *llvm = LlvmDialect(ctx);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+
+    auto asyncFnTy = API::asyncFunctionType(llvm);
+    auto callePtr = rewriter.create<LLVM::ConstantOp>(
+        loc, asyncFnTy.getPointerTo(),
+        rewriter.getSymbolRefAttr(callee.getOperation()));
+
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+        op, i8Ptr, rewriter.getSymbolRefAttr(kAsyncCall),
+        ArrayRef<Value>({op->getOperand(0), callePtr}));
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// async.await op lowering to MLIR_AsyncRT_Await function call.
+//===----------------------------------------------------------------------===//
+
+class AsyncAwaitOpLowering : public ConversionPattern {
+public:
+  explicit AsyncAwaitOpLowering(MLIRContext *ctx, CoroMachinery coroMachinery)
+      : ConversionPattern(AsyncAwaitOp::getOperationName(), 1, ctx),
+        coroMachinery(coroMachinery) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    MLIRContext *ctx = op->getContext();
+    Location loc = op->getLoc();
+
+    auto *llvm = LlvmDialect(ctx);
+
+    auto token_ty = LLVM::LLVMType::getTokenTy(llvm);
+    auto i1 = LLVM::LLVMType::getInt1Ty(llvm);
+    auto i8 = LLVM::LLVMType::getInt8Ty(llvm);
+
+    auto resumeFnTy = API::resumeFunctionType(llvm);
+    auto resumePtr = rewriter.create<LLVM::ConstantOp>(
+        loc, resumeFnTy.getPointerTo(), rewriter.getSymbolRefAttr(kResume));
+
+    // Save coroutine state: @llvm.coro.save
+    auto coroSave = rewriter.create<LLVM::CallOp>(
+        loc, token_ty, rewriter.getSymbolRefAttr(kCoroSave),
+        ArrayRef<Value>({coroMachinery.coroHandle}));
+
+    // Call MLIR async runtime await API with an async handle, coroutine handle,
+    // and a pointer to resume function.
+    rewriter.create<LLVM::CallOp>(
+        loc, Type(), rewriter.getSymbolRefAttr(kAwait),
+        ArrayRef<Value>(
+            {op->getOperand(0), coroMachinery.coroHandle, resumePtr}));
+    rewriter.eraseOp(op);
+
+    // Everything after await call goes into the coroutine resume block.
+    Block *resume = op->getBlock()->splitBlock(op->getNextNode());
+
+    // Branch into the resume block unconditionally. This branch will be
+    // replaced with a conditional branch after we'll add suspension point.
+    OpBuilder builder = OpBuilder::atBlockEnd(resume->getPrevNode());
+    builder.create<BranchOp>(loc, resume);
+
+    // Add a coroutine suspension point right after the await call.
+    builder = OpBuilder::atBlockTerminator(resume->getPrevNode());
+
+    auto constFalse =
+        builder.create<LLVM::ConstantOp>(loc, i1, builder.getBoolAttr(false));
+
+    // Suspend a coroutine: @llvm.coro.suspend
+    auto coroSuspend = builder.create<LLVM::CallOp>(
+        loc, i8, builder.getSymbolRefAttr(kCoroSuspend),
+        ArrayRef<Value>({coroSave.getResult(0), constFalse}));
+
+    // After a suspension point decide if we should branch into resume, cleanup
+    // or suspend block of the coroutine (see @llvm.coro.suspend return code
+    // documentation).
+    auto constZero =
+        builder.create<LLVM::ConstantOp>(loc, i8, builder.getI8IntegerAttr(0));
+    auto constNegOne =
+        builder.create<LLVM::ConstantOp>(loc, i8, builder.getI8IntegerAttr(-1));
+
+    auto isZero = builder.create<LLVM::ICmpOp>(
+        loc, LLVM::ICmpPredicate::eq, coroSuspend.getResult(0), constZero);
+    auto isNegOne = builder.create<LLVM::ICmpOp>(
+        loc, LLVM::ICmpPredicate::eq, coroSuspend.getResult(0), constNegOne);
+
+    Block *resumeOrCleanup = builder.createBlock(resume);
+    // Suspend the coroutine ...?
+    builder = OpBuilder::atBlockTerminator(resumeOrCleanup->getPrevNode());
+    builder.create<LLVM::CondBrOp>(loc, isNegOne, coroMachinery.suspend,
+                                   ArrayRef<Value>(), resumeOrCleanup,
+                                   ArrayRef<Value>());
+    resumeOrCleanup->getPrevNode()->getTerminator()->erase();
+
+    // ... or resume or cleanup the coroutine?
+    builder = OpBuilder::atBlockBegin(resumeOrCleanup);
+    builder.create<LLVM::CondBrOp>(loc, isZero, resume, ArrayRef<Value>(),
+                                   coroMachinery.cleanup, ArrayRef<Value>());
+
+    return success();
+  }
+
+private:
+  CoroMachinery coroMachinery;
+};
+
+//===----------------------------------------------------------------------===//
+// async.sync_await op lowering to MLIR_AsyncRT_SyncAwait function call.
+//===----------------------------------------------------------------------===//
+
+class AsyncSyncAwaitOpLowering : public ConversionPattern {
+public:
+  explicit AsyncSyncAwaitOpLowering(MLIRContext *ctx)
+      : ConversionPattern(AsyncSyncAwaitOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.create<LLVM::CallOp>(op->getLoc(), Type(),
+                                  rewriter.getSymbolRefAttr(kSyncAwait),
+                                  ArrayRef<Value>({op->getOperand(0)}));
+    rewriter.eraseOp(op);
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// async.create_handle op lowering to MLIR_AsyncRT_CreateHandle function call.
+//===----------------------------------------------------------------------===//
+
+class AsyncCreateHandleOpLowering : public ConversionPattern {
+public:
+  explicit AsyncCreateHandleOpLowering(MLIRContext *ctx)
+      : ConversionPattern(AsyncCreateHandleOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    MLIRContext *ctx = op->getContext();
+
+    auto *llvm = LlvmDialect(ctx);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+        op, i8Ptr, rewriter.getSymbolRefAttr(kCreateHandle),
+        ArrayRef<Value>({op->getOperand(0)}));
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// async.emplace_handle op lowering to MLIR_AsyncRT_EmplaceHandle function call.
+//===----------------------------------------------------------------------===//
+
+class AsyncEmplaceHandleOpLowering : public ConversionPattern {
+public:
+  explicit AsyncEmplaceHandleOpLowering(MLIRContext *ctx)
+      : ConversionPattern(AsyncEmplaceHandleOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.create<LLVM::CallOp>(op->getLoc(), Type(),
+                                  rewriter.getSymbolRefAttr(kEmplaceHandle),
+                                  ArrayRef<Value>({op->getOperand(0)}));
+    rewriter.eraseOp(op);
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// async.default_runtime op lowering to MLIR_AsyncRT_DefaultRuntime function
+// call.
+//===----------------------------------------------------------------------===//
+
+class AsyncDefaultRuntimeOpLowering : public ConversionPattern {
+public:
+  explicit AsyncDefaultRuntimeOpLowering(MLIRContext *ctx)
+      : ConversionPattern(AsyncDefaultRuntimeOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    MLIRContext *ctx = op->getContext();
+
+    auto *llvm = LlvmDialect(ctx);
+    auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+    rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+        op, i8Ptr, rewriter.getSymbolRefAttr(kDefaultRuntime),
+        ArrayRef<Value>());
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Convert types for all call operations to lower async types.
+//===----------------------------------------------------------------------===//
+
+class CallOpOpConversion : public ConversionPattern {
+public:
+  explicit CallOpOpConversion(MLIRContext *ctx)
+      : ConversionPattern(CallOp::getOperationName(), 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    AsyncRuntimeTypeConverted converter;
+
+    SmallVector<Type, 5> resultTypes;
+    converter.convertTypes(op->getResultTypes(), resultTypes);
+
+    CallOp call = cast<CallOp>(op);
+    rewriter.replaceOpWithNewOp<CallOp>(op, resultTypes, call.callee(),
+                                        call.getOperands());
+
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+
+/// Add Async Runtime C API declarations to the module.
+void AddAsyncRuntimeApiDeclarations(ModuleOp module) {
+  MLIRContext *ctx = module.getContext();
+  Location loc = module.getLoc();
+
+  OpBuilder builder(module.getBody()->getTerminator());
+  auto *llvm = LlvmDialect(ctx);
+
+  if (!module.lookupSymbol(kAsyncCall))
+    builder.create<LLVM::LLVMFuncOp>(loc, kAsyncCall,
+                                     API::asyncCallFunctionType(llvm));
+
+  if (!module.lookupSymbol(kAwait))
+    builder.create<LLVM::LLVMFuncOp>(loc, kAwait, API::awaitFunctionType(llvm));
+
+  if (!module.lookupSymbol(kSyncAwait))
+    builder.create<LLVM::LLVMFuncOp>(loc, kSyncAwait,
+                                     API::syncAwaitFunctionType(llvm));
+
+  if (!module.lookupSymbol(kCreateHandle))
+    builder.create<LLVM::LLVMFuncOp>(loc, kCreateHandle,
+                                     API::createHandleFunctionType(llvm));
+
+  if (!module.lookupSymbol(kEmplaceHandle))
+    builder.create<LLVM::LLVMFuncOp>(loc, kEmplaceHandle,
+                                     API::emplaceHandleFunctionType(llvm));
+
+  if (!module.lookupSymbol(kDefaultRuntime))
+    builder.create<LLVM::LLVMFuncOp>(loc, kDefaultRuntime,
+                                     API::defaultRuntimeFunctionType(llvm));
+}
+
+/// Adds malloc/free declarations to the module.
+void AddCRuntimeDeclarations(ModuleOp module) {
+  using namespace mlir::LLVM;
+
+  MLIRContext *ctx = module.getContext();
+  Location loc = module.getLoc();
+
+  OpBuilder builder(module.getBody()->getTerminator());
+  auto *llvm = LlvmDialect(ctx);
+
+  auto voidTy = LLVMType::getVoidTy(llvm);
+  auto i64 = LLVMType::getInt64Ty(llvm);
+  auto i8Ptr = LLVMType::getInt8PtrTy(llvm);
+
+  if (!module.lookupSymbol(kMalloc))
+    builder.create<LLVM::LLVMFuncOp>(
+        loc, kMalloc, LLVMType::getFunctionTy(i8Ptr, {i64}, false));
+
+  if (!module.lookupSymbol(kFree))
+    builder.create<LLVM::LLVMFuncOp>(
+        loc, kFree, LLVMType::getFunctionTy(voidTy, i8Ptr, false));
+}
+
+/// Adds coroutine intrinsics declarations to the module.
+void AddCoroutineIntrinsicsDeclarations(ModuleOp module) {
+  using namespace mlir::LLVM;
+
+  MLIRContext *ctx = module.getContext();
+  Location loc = module.getLoc();
+
+  OpBuilder builder(module.getBody()->getTerminator());
+  auto *llvm = LlvmDialect(ctx);
+
+  auto token = LLVMType::getTokenTy(llvm);
+  auto voidTy = LLVMType::getVoidTy(llvm);
+
+  auto i8 = LLVMType::getInt8Ty(llvm);
+  auto i1 = LLVMType::getInt1Ty(llvm);
+  auto i32 = LLVMType::getInt32Ty(llvm);
+  auto i64 = LLVMType::getInt64Ty(llvm);
+  auto i8Ptr = LLVMType::getInt8PtrTy(llvm);
+
+  if (!module.lookupSymbol(kCoroId)) {
+    builder.create<LLVMFuncOp>(
+        loc, kCoroId,
+        LLVMType::getFunctionTy(token, {i32, i8Ptr, i8Ptr, i8Ptr}, false));
+  }
+
+  if (!module.lookupSymbol(kCoroSizeI64)) {
+    builder.create<LLVMFuncOp>(loc, kCoroSizeI64,
+                               LLVMType::getFunctionTy(i64, false));
+  }
+
+  if (!module.lookupSymbol(kCoroBegin)) {
+    builder.create<LLVMFuncOp>(
+        loc, kCoroBegin, LLVMType::getFunctionTy(i8Ptr, {token, i8Ptr}, false));
+  }
+
+  if (!module.lookupSymbol(kCoroSave)) {
+    builder.create<LLVMFuncOp>(loc, kCoroSave,
+                               LLVMType::getFunctionTy(token, i8Ptr, false));
+  }
+
+  if (!module.lookupSymbol(kCoroSuspend)) {
+    builder.create<LLVMFuncOp>(loc, kCoroSuspend,
+                               LLVMType::getFunctionTy(i8, {token, i1}, false));
+  }
+
+  if (!module.lookupSymbol(kCoroEnd)) {
+    builder.create<LLVMFuncOp>(loc, kCoroEnd,
+                               LLVMType::getFunctionTy(i1, {i8Ptr, i1}, false));
+  }
+
+  if (!module.lookupSymbol(kCoroFree)) {
+    builder.create<LLVMFuncOp>(
+        loc, kCoroFree, LLVMType::getFunctionTy(i8Ptr, {token, i8Ptr}, false));
+  }
+
+  if (!module.lookupSymbol(kCoroResume)) {
+    builder.create<LLVMFuncOp>(loc, kCoroResume,
+                               LLVMType::getFunctionTy(voidTy, i8Ptr, false));
+  }
+}
+
+/// Adds a function that passes input coroutine handle to llvm.coro.resume
+/// intrinsic. We need this function to be able to take an address and pass
+/// it to async.await callback.
+void AddResumeFunction(ModuleOp module) {
+  OpBuilder moduleBuilder(module.getBody()->getTerminator());
+  Location loc = module.getLoc();
+
+  if (module.lookupSymbol(kResume))
+    return;
+
+  auto *llvm = LlvmDialect(module.getContext());
+  auto i8Ptr = LLVM::LLVMType::getInt8PtrTy(llvm);
+
+  auto resumeOp = moduleBuilder.create<FuncOp>(
+      loc, kResume, moduleBuilder.getFunctionType({i8Ptr}, {}),
+      ArrayRef<NamedAttribute>{});
+  SymbolTable::setSymbolVisibility(resumeOp, SymbolTable::Visibility::Private);
+
+  auto *block = resumeOp.addEntryBlock();
+  OpBuilder blockBuilder = OpBuilder::atBlockEnd(block);
+
+  SmallVector<Value, 1> args = {resumeOp.getArgument(0)};
+  blockBuilder.create<LLVM::CallOp>(
+      loc, Type(), blockBuilder.getSymbolRefAttr("llvm.coro.resume"), args);
+
+  blockBuilder.create<ReturnOp>(loc);
+}
+
+/// Returns `true` if the function has operations from the Async dialect in the
+/// body.
+bool IsAsyncFunction(FuncOp func) {
+  auto *async = func.getContext()->getRegisteredDialect<AsyncDialect>();
+  for (auto &block : func.getBlocks()) {
+    for (auto &op : block.getOperations())
+      if (op.getDialect() == async)
+        return true;
+  }
+  return false;
+}
+
+/// Returns `true` if the function has `async.await` operation in the body,
+/// which requires the function to be converted into the coroutine.
+bool IsAsyncCoroFunction(FuncOp func) {
+  for (auto &block : func.getBlocks()) {
+    for (auto &op : block.getOperations())
+      if (isa<AsyncAwaitOp>(op))
+        return true;
+  }
+  return false;
+}
+
+/// Adds a coroutine initialization, cleanup and suspention points to the
+/// function. This will allow to use LLVM coroutines passes to convert a
+/// function to the coroutine.
+CoroMachinery AddCoroMachinery(FuncOp func) {
+  using namespace mlir::LLVM;
+
+  OpBuilder builder(func.getBody());
+  Location loc = func.getBody().getLoc();
+
+  auto *llvm = LlvmDialect(func.getContext());
+
+  auto token = LLVMType::getTokenTy(llvm);
+  auto i1 = LLVMType::getInt1Ty(llvm);
+  auto i32 = LLVMType::getInt32Ty(llvm);
+  auto i64 = LLVMType::getInt64Ty(llvm);
+  auto i8Ptr = LLVMType::getInt8PtrTy(llvm);
+
+  // ------------------------------------------------------------------------ //
+  // Initialize coroutine: allocate frame, get coroutine handle.
+  // ------------------------------------------------------------------------ //
+
+  // Constants for initializing coroutine frame.
+  auto constZero =
+      builder.create<LLVM::ConstantOp>(loc, i32, builder.getI32IntegerAttr(0));
+  auto constFalse =
+      builder.create<LLVM::ConstantOp>(loc, i1, builder.getBoolAttr(false));
+  auto nullPtr = builder.create<LLVM::NullOp>(loc, i8Ptr);
+
+  // Get coroutine id: @llvm.coro.id
+  auto coroId = builder.create<LLVM::CallOp>(
+      loc, token, builder.getSymbolRefAttr(kCoroId),
+      ArrayRef<Value>({constZero, nullPtr, nullPtr, nullPtr}));
+
+  // Get coroutine frame size: @llvm.coro.size.i64
+  auto coroSize = builder.create<LLVM::CallOp>(
+      loc, i64, builder.getSymbolRefAttr(kCoroSizeI64), ArrayRef<Value>());
+
+  // Allocate memory for coroutine frame.
+  auto coroAlloc = builder.create<LLVM::CallOp>(
+      loc, i8Ptr, builder.getSymbolRefAttr(kMalloc),
+      ArrayRef<Value>({coroSize.getResult(0)}));
+
+  // Begin a coroutine: @llvm.coro.begin
+  auto coroHdl = builder.create<LLVM::CallOp>(
+      loc, i8Ptr, builder.getSymbolRefAttr(kCoroBegin),
+      ArrayRef<Value>({coroId.getResult(0), coroAlloc.getResult(0)}));
+
+  Block *cleanupBlock = func.addBlock();
+  Block *suspendBlock = func.addBlock();
+
+  // ------------------------------------------------------------------------ //
+  // Coroutine cleanup block: deallocate coroutine frame, free the memory.
+  // ------------------------------------------------------------------------ //
+  builder = OpBuilder::atBlockBegin(cleanupBlock);
+
+  // Get a pointer to the coroutine frame memory: @llvm.coro.free.
+  auto coroMem = builder.create<LLVM::CallOp>(
+      loc, i8Ptr, builder.getSymbolRefAttr(kCoroFree),
+      ArrayRef<Value>({coroId.getResult(0), coroHdl.getResult(0)}));
+
+  // Free the memory.
+  builder.create<LLVM::CallOp>(loc, Type(), builder.getSymbolRefAttr(kFree),
+                               ArrayRef<Value>({coroMem.getResult(0)}));
+  // Branch into the return block.
+  builder.create<BranchOp>(loc, suspendBlock);
+
+  // ------------------------------------------------------------------------ //
+  // Coroutine suspend block: return allocated async handle, and mark the
+  // end of a coroutine.
+  // ------------------------------------------------------------------------ //
+  builder = OpBuilder::atBlockBegin(suspendBlock);
+
+  // Mark the end of a coroutine: @llvm.coro.end.
+  builder.create<LLVM::CallOp>(
+      loc, i1, builder.getSymbolRefAttr(kCoroEnd),
+      ArrayRef<Value>({coroHdl.getResult(0), constFalse}));
+
+  // Clone return statement into the suspend block.
+  auto *ret = func.getBody().front().getTerminator();
+  builder.clone(*ret);
+
+  // And replace the original return with a branch to cleanup.
+  ret->erase();
+  builder = OpBuilder::atBlockEnd(&func.getBody().front());
+  builder.create<BranchOp>(loc, cleanupBlock);
+
+  // `async.await` op lowering will create resume blocks for async
+  // continuations, and will conditionally branch to cleanup or suspend blocks.
+
+  return {coroHdl.getResult(0), cleanupBlock, suspendBlock};
+}
+
+} // namespace
+
+/// Populate the given list with patterns that convert from Async to LLVM.
+void mlir::populateAsyncToLLVMConversionPatterns(
+    OwningRewritePatternList &patterns) {}
+
+void ConvertAsyncToLLVMPass::runOnOperation() {
+  ModuleOp module = getOperation();
+  MLIRContext *ctx = module.getContext();
+
+  AddAsyncRuntimeApiDeclarations(module);
+  AddCRuntimeDeclarations(module);
+  AddCoroutineIntrinsicsDeclarations(module);
+  AddResumeFunction(module);
+
+  module.walk([&](FuncOp func) -> WalkResult {
+    // Skip functions that do not have operations from async dialect.
+    if (!IsAsyncFunction(func))
+      return WalkResult::advance();
+
+    AsyncRuntimeTypeConverted converter;
+    OwningRewritePatternList patterns;
+
+    // Trivial lowerings for async types and operations.
+    populateFuncOpTypeConversionPattern(patterns, ctx, converter);
+    populateAsyncToLLVMConversionPatterns(patterns);
+    patterns.insert<AsyncCallOpLowering>(ctx);
+    patterns.insert<AsyncSyncAwaitOpLowering>(ctx);
+    patterns.insert<AsyncCreateHandleOpLowering>(ctx);
+    patterns.insert<AsyncEmplaceHandleOpLowering>(ctx);
+    patterns.insert<AsyncDefaultRuntimeOpLowering>(ctx);
+    patterns.insert<CallOpOpConversion>(ctx);
+
+    // Add lowering from async.await to LLVM coroutines.
+    if (IsAsyncCoroFunction(func)) {
+      auto coro = AddCoroMachinery(func);
+      patterns.insert<AsyncAwaitOpLowering>(ctx, coro);
+    }
+
+    // Apply all the conversion patterns.
+    ConversionTarget target(*ctx);
+    target.addLegalDialect<LLVM::LLVMDialect>();
+    target.addIllegalDialect<AsyncDialect>();
+    target.addDynamicallyLegalOp<FuncOp>(
+        [&](FuncOp op) { return converter.isSignatureLegal(op.getType()); });
+    target.addDynamicallyLegalOp<CallOp>(
+        [&](CallOp op) { return converter.isLegal(op.getResultTypes()); });
+
+    if (failed(applyPartialConversion(func, target, patterns))) {
+      signalPassFailure();
+      return WalkResult::interrupt();
+    }
+
+    return WalkResult::advance();
+  });
+}
+
+std::unique_ptr<OperationPass<ModuleOp>> mlir::createConvertAsyncToLLVMPass() {
+  return std::make_unique<ConvertAsyncToLLVMPass>();
+}

mlir/lib/Conversion/CMakeLists.txt

 add_subdirectory(AffineToStandard)
 add_subdirectory(AVX512ToLLVM)
+add_subdirectory(AsyncToLLVM)
 add_subdirectory(GPUCommon)
 add_subdirectory(GPUToNVVM)
 add_subdirectory(GPUToROCDL)
 add_subdirectory(GPUToSPIRV)
 add_subdirectory(GPUToVulkan)
 add_subdirectory(LinalgToLLVM)
 add_subdirectory(LinalgToSPIRV)
 add_subdirectory(LinalgToStandard)

mlir/lib/Dialect/Async/Async.cpp

+//===- Async.cpp - Asynchronous Operations --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Async/Async.h"
+
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/Function.h"
+#include "mlir/IR/Module.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/IR/Types.h"
+#include "mlir/Support/LogicalResult.h"
+
+using namespace mlir;
+using namespace mlir::async;
+
+async::AsyncDialect::AsyncDialect(MLIRContext *context)
+    : Dialect(getDialectNamespace(), context) {
+  addTypes<RuntimeType, HandleType>();
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/Async/AsyncOps.cpp.inc"
+      >();
+}
+
+Type mlir::async::AsyncDialect::parseType(DialectAsmParser &parser) const {
+  StringRef keyword;
+  if (parser.parseKeyword(&keyword))
+    return Type();
+  MLIRContext *context = getContext();
+
+  if (keyword == "runtime")
+    return RuntimeType::get(context);
+  if (keyword == "handle")
+    return HandleType::get(context);
+
+  parser.emitError(parser.getNameLoc(), "unknown Async type: " + keyword);
+  return Type();
+}
+
+static void print(RuntimeType rt, DialectAsmPrinter &os) { os << "runtime"; }
+static void print(HandleType rt, DialectAsmPrinter &os) { os << "handle"; }
+
+void mlir::async::AsyncDialect::printType(Type type,
+                                          DialectAsmPrinter &os) const {
+  switch (type.getKind()) {
+  default:
+    llvm_unreachable("Unhandled Async type");
+  case AsyncTypes::Runtime:
+    print(type.cast<RuntimeType>(), os);
+    break;
+  case AsyncTypes::Handle:
+    print(type.cast<HandleType>(), os);
+    break;
+  }
+}
+
+namespace mlir {
+namespace async {
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Async/AsyncOps.cpp.inc"
+} // namespace async
+} // namespace mlir

mlir/lib/Dialect/Async/CMakeLists.txt

+add_mlir_dialect_library(MLIRASYNC
+  Async.cpp
+
+  DEPENDS
+  MLIRAsyncOpsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIREDSC
+  MLIRIR
+  )

mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp

struct LLVMDialectImpl {
   llvm::LLVMContext llvmContext;
   llvm::Module module;
 
   /// A set of LLVMTypes that are cached on construction to avoid any lookups or
   /// locking.
   LLVMType int1Ty, int8Ty, int16Ty, int32Ty, int64Ty, int128Ty;
   LLVMType doubleTy, floatTy, bfloatTy, halfTy, fp128Ty, x86_fp80Ty;
   LLVMType voidTy;
+  LLVMType tokenTy;
 
   /// A smart mutex to lock access to the llvm context. Unlike MLIR, LLVM is not
   /// multi-threaded and requires locked access to prevent race conditions.
   llvm::sys::SmartMutex<true> mutex;
 };
 } // end namespace detail
 } // end namespace LLVM
 } // end namespace mlir
#include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
   impl->floatTy = LLVMType::get(context, llvm::Type::getFloatTy(llvmContext));
   impl->bfloatTy = LLVMType::get(context, llvm::Type::getBFloatTy(llvmContext));
   impl->halfTy = LLVMType::get(context, llvm::Type::getHalfTy(llvmContext));
   impl->fp128Ty = LLVMType::get(context, llvm::Type::getFP128Ty(llvmContext));
   impl->x86_fp80Ty =
       LLVMType::get(context, llvm::Type::getX86_FP80Ty(llvmContext));
   /// Other Types.
   impl->voidTy = LLVMType::get(context, llvm::Type::getVoidTy(llvmContext));
+  impl->tokenTy = LLVMType::get(context, llvm::Type::getTokenTy(llvmContext));
 }
 
 LLVMDialect::~LLVMDialect() {}
 
 #define GET_OP_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
 
 llvm::LLVMContext &LLVMDialect::getLLVMContext() { return impl->llvmContext; }

 }
 
 LLVMType LLVMType::getVoidTy(LLVMDialect *dialect) {
   return dialect->impl->voidTy;
 }
 
 bool LLVMType::isVoidTy() { return getUnderlyingType()->isVoidTy(); }
 
+LLVMType LLVMType::getTokenTy(LLVMDialect *dialect) {
+  return dialect->impl->tokenTy;
+}
+
+bool LLVMType::isTokenTy() { return getUnderlyingType()->isTokenTy(); }
+
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
 
 Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
                                      StringRef name, StringRef value,
                                      LLVM::Linkage linkage,
                                      LLVM::LLVMDialect *llvmDialect) {

mlir/lib/ExecutionEngine/AsyncRuntime.cpp

+//===- AsyncRuntime.cpp - Async runtime reference implementation ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements basic Async runtime API for supporting Async dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/ExecutionEngine/AsyncRuntime.h"
+
+#ifdef MLIR_ASYNCRUNTIME_DEFINE_FUNCTIONS
+
+#include <thread>
+#include <vector>
+
+struct MLIR_AsyncRuntime {};
+
+struct MLIR_AsyncHandle {
+  bool completed = false;
+  std::mutex mu;
+  std::condition_variable cv;
+  std::vector<std::function<void()>> awaiters;
+};
+
+MLIR_AsyncRuntime *DefaultInstance() {
+  static MLIR_AsyncRuntime *instance = new MLIR_AsyncRuntime();
+  return instance;
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT MLIR_AsyncRuntime *
+MLIR_AsyncRT_DefaultRuntime() {
+  return DefaultInstance();
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT MLIR_AsyncHandle *
+MLIR_AsyncRT_Call(MLIR_AsyncRuntime *runtime, TaskFunction task) {
+  MLIR_AsyncHandle *handle = new MLIR_AsyncHandle;
+
+  // Launch task in a separate thread.
+  std::thread worker([handle, task]() {
+    (*task)();
+    MLIR_AsyncRT_EmplaceHandle(handle);
+  });
+
+  worker.detach();
+
+  return handle;
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT MLIR_AsyncHandle *
+MLIR_AsyncRT_CreateHandle(MLIR_AsyncRuntime *) {
+  MLIR_AsyncHandle *handle = new MLIR_AsyncHandle;
+  return handle;
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT void
+MLIR_AsyncRT_EmplaceHandle(MLIR_AsyncHandle *handle) {
+  std::unique_lock<std::mutex> lock(handle->mu);
+  handle->completed = true;
+  handle->cv.notify_all();
+  for (auto &awaiter : handle->awaiters) {
+    awaiter();
+  }
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT void
+MLIR_AsyncRT_Await(MLIR_AsyncHandle *handle, CoroHandle coro_handle,
+                   CoroResume coro_resume) {
+  std::unique_lock<std::mutex> lock(handle->mu);
+
+  if (handle->completed) {
+    (*coro_resume)(coro_handle);
+    delete handle;
+  } else {
+    handle->awaiters.push_back([coro_handle, coro_resume, handle]() {
+      (*coro_resume)(coro_handle);
+      delete handle;
+    });
+  }
+}
+
+extern "C" MLIR_ASYNCRUNTIME_EXPORT void
+MLIR_AsyncRT_SyncAwait(MLIR_AsyncHandle *handle) {
+  std::unique_lock<std::mutex> lock(handle->mu);
+
+  if (!handle->completed)
+    handle->cv.wait(lock, [handle] { return handle->completed; });
+
+  delete handle;
+}
+
+#endif

mlir/lib/ExecutionEngine/OptUtils.cpp

 #include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Coroutines.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include <climits>
 #include <mutex>
 
 // Run the module and function passes managed by the module manager.
 static void runPasses(llvm::legacy::PassManager &modulePM,
                       llvm::legacy::FunctionPassManager &funcPM,
void mlir::initializeLLVMPasses() {
   llvm::initializeCore(registry);
   llvm::initializeTransformUtils(registry);
   llvm::initializeScalarOpts(registry);
   llvm::initializeIPO(registry);
   llvm::initializeInstCombine(registry);
   llvm::initializeAggressiveInstCombine(registry);
   llvm::initializeAnalysis(registry);
   llvm::initializeVectorization(registry);
+  llvm::initializeCoroutines(registry);
 }
 
 // Populate pass managers according to the optimization and size levels.
 // This behaves similarly to LLVM opt.
 static void populatePassManagers(llvm::legacy::PassManager &modulePM,
                                  llvm::legacy::FunctionPassManager &funcPM,
                                  unsigned optLevel, unsigned sizeLevel,
                                  llvm::TargetMachine *targetMachine) {
   llvm::PassManagerBuilder builder;
   builder.OptLevel = optLevel;
   builder.SizeLevel = sizeLevel;
   builder.Inliner = llvm::createFunctionInliningPass(
       optLevel, sizeLevel, /*DisableInlineHotCallSite=*/false);
   builder.LoopVectorize = optLevel > 1 && sizeLevel < 2;
   builder.SLPVectorize = optLevel > 1 && sizeLevel < 2;
   builder.DisableUnrollLoops = (optLevel == 0);
 
+  // Add all coroutine passes to the builder.
+  addCoroutinePassesToExtensionPoints(builder);
+
   if (targetMachine) {
     // Add pass to initialize TTI for this specific target. Otherwise, TTI will
     // be initialized to NoTTIImpl by default.
     modulePM.add(createTargetTransformInfoWrapperPass(
         targetMachine->getTargetIRAnalysis()));
     funcPM.add(createTargetTransformInfoWrapperPass(
         targetMachine->getTargetIRAnalysis()));
   }

mlir/test/Conversion/AsyncToLLVM/convert-to-llvm.mlir

+// RUN: mlir-opt %s -convert-async-to-llvm | mlir-opt | FileCheck %s
+
+func @callee() {
+}
+
+// CHECK: func @create_and_emplace_handle
+func @create_and_emplace_handle() {
+  // CHECK: %[[R0:.*]] = llvm.call @MLIR_AsyncRT_DefaultRuntime()
+  // CHECK: %[[R1:.*]] = llvm.call @MLIR_AsyncRT_CreateHandle(%[[R0]])
+  // CHECK: llvm.call @MLIR_AsyncRT_EmplaceHandle(%[[R1]])
+  // CHECK: llvm.call @MLIR_AsyncRT_SyncAwait(%[[R1]])
+  %runtime = async.default_runtime : !async.runtime
+  %handle = async.create_handle %runtime : (!async.runtime) -> !async.handle
+  async.emplace_handle %handle : !async.handle
+  async.sync_await %handle : !async.handle
+  return
+}
+
+// CHECK: func @async_call(%arg0: !llvm<"i8*">)
+func @async_call(%arg0: !async.runtime) {
+  // CHECK: %[[R0:.*]] = llvm.mlir.constant(@callee)
+  // CHECK: %[[R1:.*]] = llvm.call @MLIR_AsyncRT_Call(%arg0, %[[R0]])
+  %handle = async.call %arg0 : !async.runtime @callee() : () -> !async.handle
+  return
+}
+
+// CHECK: func @async_await(%arg0: !llvm<"i8*">)
+func @async_await(%arg0: !async.runtime) -> !async.handle {
+  // CHECK: %[[ID:.*]] = llvm.call @llvm.coro.id
+  // CHECK: %[[SIZE:.*]] = llvm.call @llvm.coro.size.i64
+  // CHECK: %[[HDL:.*]] = llvm.call @llvm.coro.begin
+
+  // CHECK: %[[AHDL:.*]] = llvm.call @MLIR_AsyncRT_CreateHandle
+  %ret_handle = async.create_handle %arg0 : (!async.runtime) -> !async.handle
+
+  %handle = async.call %arg0 : !async.runtime @callee() : () -> !async.handle
+
+  // CHECK: %[[SAVED:.*]] = llvm.call @llvm.coro.save
+  // CHECK: llvm.call @MLIR_AsyncRT_Await
+  // CHECK: %[[SUSPENDED:.*]] = llvm.call @llvm.coro.suspend
+  async.await %handle : !async.handle
+
+  // CHECK: llvm.call @MLIR_AsyncRT_EmplaceHandle
+  async.emplace_handle %ret_handle : !async.handle
+  return %ret_handle : !async.handle
+}
+
+// CHECK: llvm.func @MLIR_AsyncRT_Call
+// CHECK: llvm.func @MLIR_AsyncRT_Await
+// CHECK: llvm.func @MLIR_AsyncRT_SyncAwait
+// CHECK: llvm.func @MLIR_AsyncRT_CreateHandle
+// CHECK: llvm.func @MLIR_AsyncRT_EmplaceHandle
+// CHECK: llvm.func @MLIR_AsyncRT_DefaultRuntime

mlir/test/mlir-cpu-runner/async.mlir

+// RUN:   mlir-opt %s -convert-async-to-llvm -convert-std-to-llvm              \
+// RUN: | mlir-cpu-runner                                                      \
+// RUN:     -e main0 -entry-point-result=void -O0                              \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext  \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_async_runtime%shlibext   \
+// RUN: | FileCheck %s -check-prefix=CHECK0
+
+// RUN:   mlir-opt %s -convert-async-to-llvm -convert-std-to-llvm              \
+// RUN: | mlir-cpu-runner                                                      \
+// RUN:     -e main1 -entry-point-result=void -O0                              \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext  \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_async_runtime%shlibext   \
+// RUN: | FileCheck %s -check-prefix=CHECK1
+
+// RUN:   mlir-opt %s -convert-async-to-llvm -convert-std-to-llvm              \
+// RUN: | mlir-cpu-runner                                                      \
+// RUN:     -e main2 -entry-point-result=void -O0                              \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext  \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_async_runtime%shlibext   \
+// RUN: | FileCheck %s -check-prefix=CHECK2 --dump-input=always
+
+func @print_i32(%n : i32) -> ()
+
+func @print_123() {
+  %c123 = constant 123 : i32
+  call @print_i32(%c123): (i32) -> ()
+  return
+}
+
+// ===--------------------------------------------------------------------=== //
+// Create and emplace handles synchronously.
+// ===--------------------------------------------------------------------=== //
+
+func @main0() {
+  %runtime = async.default_runtime : !async.runtime
+  %handle = async.create_handle %runtime : (!async.runtime) -> !async.handle
+  async.emplace_handle %handle : !async.handle
+  async.sync_await %handle : !async.handle
+
+  // CHECK0: 123
+  %c123 = constant 123 : i32
+  call @print_i32(%c123): (i32) -> ()
+
+  return
+}
+
+// ===--------------------------------------------------------------------=== //
+// Wait for an async function synchronously.
+// ===--------------------------------------------------------------------=== //
+
+func @main1() {
+  %runtime = async.default_runtime : !async.runtime
+  %handle = async.call %runtime : !async.runtime
+            @print_123() : () -> !async.handle
+  async.sync_await %handle : !async.handle
+
+  // CHECK1: 123456
+  %c456 = constant 456 : i32
+  call @print_i32(%c456): (i32) -> ()
+
+  return
+}
+
+// ===--------------------------------------------------------------------=== //
+// Convert `async_function` to LLVM coroutine and call it asynchronously.
+// ===--------------------------------------------------------------------=== //
+
+func @async_function(%arg0: !async.runtime) -> !async.handle {
+  %c789 = constant 789 : i32
+  %ret_handle = async.create_handle %arg0 : (!async.runtime) -> !async.handle
+
+  %handle = async.call %arg0 : !async.runtime @print_123() : () -> !async.handle
+
+  // This will be printed before 123 because it is running in the caller thread.
+  call @print_i32(%c789): (i32) -> ()
+
+  async.await %handle : !async.handle
+
+  // This line will wait for the async task completion.
+  call @print_i32(%c789): (i32) -> ()
+
+  async.emplace_handle %ret_handle : !async.handle
+  return %ret_handle : !async.handle
+}
+
+func @main2() {
+  %runtime = async.default_runtime : !async.runtime
+  %handle = call @async_function(%runtime): (!async.runtime) -> !async.handle
+  async.sync_await %handle : !async.handle
+
+  // CHECK2: 789123789456
+  %c456 = constant 456 : i32
+  call @print_i32(%c456): (i32) -> ()
+
+  return
+}

mlir/test/mlir-cpu-runner/coroutines.mlir

+// RUN:   mlir-opt %s -convert-std-to-llvm                                     \
+// RUN: | mlir-cpu-runner                                                      \
+// RUN:     -e main -entry-point-result=void -O0                               \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext  \
+// RUN:     -shared-libs=%linalg_test_lib_dir/libmlir_async_runtime%shlibext   \
+// RUN: | FileCheck %s
+
+
+// A basic coroutines example in LLVM IR:
+//   (1) Call a coroutine and wait for the suspension
+//   (2) Resume coroutine from a suspension point.
+//
+// Printed numbers:
+//    (1) 123 - in main function
+//    (2) 456 - in main after coroutine suspension
+//    (3) 789 - in coroutine resume function
+
+func @print_i32(%n : !llvm.i32) -> ()
+
+func @f(%n : !llvm.i32) -> !llvm<"i8*"> {
+  %0 = llvm.mlir.constant(0 : i32) : !llvm.i32
+
+  %false = llvm.mlir.constant(0 : i1) : !llvm.i1
+  %null = llvm.mlir.null : !llvm<"i8*">
+
+  %id = llvm.call @llvm.coro.id(%0, %null, %null, %null): (!llvm.i32, !llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">)-> !llvm<"token">
+  %size = llvm.call @llvm.coro.size.i64(): () -> !llvm.i64
+  %alloc = llvm.call @malloc(%size): (!llvm.i64) -> !llvm<"i8*">
+  %hdl = llvm.call @llvm.coro.begin(%id, %alloc): (!llvm<"token">, !llvm<"i8*">) -> !llvm<"i8*">
+
+  call @print_i32(%n): (!llvm.i32) -> ()
+
+  %save = llvm.call @llvm.coro.save(%hdl): (!llvm<"i8*">) -> !llvm<"token">
+  %suspend = llvm.call @llvm.coro.suspend(%save, %false): (!llvm<"token">, !llvm.i1) -> !llvm.i8
+
+  %00 = llvm.mlir.constant(0 : i8) : !llvm.i8
+  %n1 = llvm.mlir.constant(-1 : i8) : !llvm.i8
+
+  %is_zero = llvm.icmp "eq" %suspend, %00 : !llvm.i8
+  %is_none = llvm.icmp "eq" %suspend, %n1 : !llvm.i8
+
+  llvm.cond_br %is_none, ^suspend, ^resume_or_cleanup
+^resume_or_cleanup:
+  llvm.cond_br %is_zero, ^resume, ^cleanup
+
+^resume:
+  %c789 = llvm.mlir.constant(789 : i32) : !llvm.i32
+  call @print_i32(%c789): (!llvm.i32)-> ()
+  br ^cleanup
+
+^cleanup:
+  %mem = llvm.call @llvm.coro.free(%id, %hdl): (!llvm<"token">, !llvm<"i8*">) -> !llvm<"i8*">
+  llvm.call @free(%mem): (!llvm<"i8*">) -> ()
+  br ^suspend
+
+^suspend:
+  llvm.call @llvm.coro.end(%hdl, %false): (!llvm<"i8*">, !llvm.i1) -> !llvm.i1
+  return %hdl : !llvm<"i8*">
+}
+
+func @main() {
+  %c123 = llvm.mlir.constant(123 : i32) : !llvm.i32
+  %c456 = llvm.mlir.constant(456 : i32) : !llvm.i32
+
+  %hdl = call @f(%c123): (!llvm.i32) -> !llvm<"i8*">
+  call @print_i32(%c456): (!llvm.i32)-> ()
+  llvm.call @llvm.coro.resume(%hdl): (!llvm<"i8*">) -> ()
+
+  return
+}
+
+// CHECK: 123456789
+
+
+// Declarations of C library functions.
+llvm.func @malloc(!llvm.i64) -> !llvm<"i8*">
+llvm.func @free(!llvm<"i8*">)
+
+// Declaration of Coroutines LLVM intrinsics.
+llvm.func @llvm.coro.id(!llvm.i32, !llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> !llvm<"token">
+llvm.func @llvm.coro.size.i64() -> !llvm.i64
+llvm.func @llvm.coro.size.i32() -> !llvm.i32
+llvm.func @llvm.coro.begin(!llvm<"token">, !llvm<"i8*">) -> !llvm<"i8*">
+llvm.func @llvm.coro.save(!llvm<"i8*">) -> !llvm<"token">
+llvm.func @llvm.coro.suspend(!llvm<"token">, !llvm.i1) -> !llvm.i8
+llvm.func @llvm.coro.end(!llvm<"i8*">, !llvm.i1) -> !llvm.i1
+llvm.func @llvm.coro.free(!llvm<"token">, !llvm<"i8*">) -> !llvm<"i8*">
+llvm.func @llvm.coro.resume(!llvm<"i8*">) -> ()
+llvm.func @llvm.coro.destroy(!llvm<"i8*">) -> ()