[mlir] Split intrinsics out of LLVMOps.td

[ftynse]

Summary

Move the operations that correspond to LLVM IR intrinsics in a separate .td
file. This makes it easier to maintain the intrinsics and decreases the compile
time of LLVMDialect.cpp by ~25%.

Depends On D123310

[ftynse]

Rebase.

[wsmoses]

mlir/docs/Dialects/LLVM.md

479

This reads weirdly, perhaps:

The MLIR operation system is open, making it....

[ftynse]

Fix cmake.

mlir/docs/Dialects/LLVM.md

 MLIR, blocks are not values and don't need a type.
 
 ## Operations
 
 All operations in the LLVM IR dialect have a custom form in MLIR. The mnemonic
 of an operation is that used in LLVM IR prefixed with "`llvm.`".
 
 [include "Dialects/LLVMOps.md"]
+
+## Operations for LLVM IR Intrinsics
+
+MLIR operation system is open making it unnecessary to introduce a hard bound
+between "core" operations and "intrinsics". General LLVM IR intrinsics are
+modeled as first-class operations in the LLVM dialect. Target-specific LLVM IR
+intrinsics, e.g., NVVM or ROCDL, are modeled as separate dialects.
+
+[include "Dialects/LLVMIntrinsicOps.md"]

mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt

 add_subdirectory(Transforms)
 
 set(LLVM_TARGET_DEFINITIONS LLVMAttrDefs.td)
 mlir_tablegen(LLVMOpsAttrDefs.h.inc -gen-attrdef-decls)
 mlir_tablegen(LLVMOpsAttrDefs.cpp.inc -gen-attrdef-defs)
 add_public_tablegen_target(MLIRLLVMAttrsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS LLVMOps.td)
 mlir_tablegen(LLVMOps.h.inc -gen-op-decls)
 mlir_tablegen(LLVMOps.cpp.inc -gen-op-defs)
 mlir_tablegen(LLVMOpsDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(LLVMOpsDialect.cpp.inc -gen-dialect-defs)
 mlir_tablegen(LLVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRLLVMOpsIncGen)
 
+set(LLVM_TARGET_DEFINITIONS LLVMIntrinsicOps.td)
+mlir_tablegen(LLVMIntrinsicOps.h.inc -gen-op-decls)
+mlir_tablegen(LLVMIntrinsicOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRLLVMIntrinsicOpsIncGen)
+
 add_mlir_doc(LLVMOps LLVMOps Dialects/ -gen-op-doc)
+add_mlir_doc(LLVMIntrinsicOps LLVMIntrinsicOps Dialects/ -gen-op-doc)
 
 set(LLVM_TARGET_DEFINITIONS LLVMOpsInterfaces.td)
 mlir_tablegen(LLVMOpsInterfaces.h.inc -gen-op-interface-decls)
 mlir_tablegen(LLVMOpsInterfaces.cpp.inc -gen-op-interface-defs)
 mlir_tablegen(LLVMTypeInterfaces.h.inc -gen-type-interface-decls)
 mlir_tablegen(LLVMTypeInterfaces.cpp.inc -gen-type-interface-defs)
 add_public_tablegen_target(MLIRLLVMOpsInterfacesIncGen)
 
 set(LLVM_TARGET_DEFINITIONS LLVMOps.td)
 mlir_tablegen(LLVMConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(LLVMConversionEnumsToLLVM.inc -gen-enum-to-llvmir-conversions)
 mlir_tablegen(LLVMConversionEnumsFromLLVM.inc -gen-enum-from-llvmir-conversions)
 add_public_tablegen_target(MLIRLLVMConversionsIncGen)
 
+set(LLVM_TARGET_DEFINITIONS LLVMIntrinsicOps.td)
+mlir_tablegen(LLVMIntrinsicConversions.inc -gen-llvmir-conversions)
+add_public_tablegen_target(MLIRLLVMIntrinsicConversionsIncGen)
+
 add_mlir_dialect(NVVMOps nvvm)
-add_mlir_doc(NVVMOps NVVMDialect Dialects/ -gen-dialect-doc)
+add_mlir_doc(NVVMOps NVVMDialect Dialects/ -gen-dialect-doc -dialect=nvvm)
 set(LLVM_TARGET_DEFINITIONS NVVMOps.td)
 mlir_tablegen(NVVMConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(NVVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(NVVMOpsEnums.cpp.inc -gen-enum-defs)
 mlir_tablegen(NVVMOpsStructs.h.inc -gen-struct-attr-decls)
 mlir_tablegen(NVVMOpsStructs.cpp.inc -gen-struct-attr-defs)
 mlir_tablegen(NVVMOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=nvvm)
 mlir_tablegen(NVVMOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=nvvm)
 add_public_tablegen_target(MLIRNVVMConversionsIncGen)
 
 add_mlir_dialect(ROCDLOps rocdl)
-add_mlir_doc(ROCDLOps ROCDLDialect Dialects/ -gen-dialect-doc)
+add_mlir_doc(ROCDLOps ROCDLDialect Dialects/ -gen-dialect-doc -dialect=rocdl)
 set(LLVM_TARGET_DEFINITIONS ROCDLOps.td)
 mlir_tablegen(ROCDLConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRROCDLConversionsIncGen)
 

mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h

 } // namespace mlir
 
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc"
 
 ///// Ops /////
 #define GET_OP_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOps.h.inc"
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.h.inc"
 
 #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.h.inc"
 
 namespace mlir {
 namespace LLVM {
 /// Create an LLVM global containing the string "value" at the module containing
 /// surrounding the insertion point of builder. Obtain the address of that
 /// global and use it to compute the address of the first character in the

mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td

+#ifndef LLVM_INTRINSIC_OPS
+#define LLVM_INTRINSIC_OPS
+
+include "mlir/IR/OpBase.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
+
+// Operations that correspond to LLVM intrinsics. With MLIR operation set being
+// extendable, there is no reason to introduce a hard boundary between "core"
+// operations and intrinsics. However, we systematically prefix them with
+// "intr." to avoid potential name clashes.
+
+class LLVM_UnaryIntrinsicOp<string func, list<Trait> traits = []> :
+    LLVM_OneResultIntrOp<func, [], [0],
+           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
+  let arguments = (ins LLVM_Type:$in);
+}
+
+class LLVM_BinarySameArgsIntrinsicOp<string func, list<Trait> traits = []> :
+    LLVM_OneResultIntrOp<func, [], [0],
+           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
+  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b);
+}
+
+class LLVM_BinaryIntrinsicOp<string func, list<Trait> traits = []> :
+    LLVM_OneResultIntrOp<func, [], [0,1],
+           !listconcat([NoSideEffect], traits)> {
+  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b);
+}
+
+class LLVM_TernarySameArgsIntrinsicOp<string func, list<Trait> traits = []> :
+    LLVM_OneResultIntrOp<func, [], [0],
+           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
+  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b, LLVM_Type:$c);
+}
+
+class LLVM_CountZerosIntrinsicOp<string func, list<Trait> traits = []> :
+    LLVM_OneResultIntrOp<func, [], [0],
+           !listconcat([NoSideEffect], traits)> {
+  let arguments = (ins LLVM_Type:$in, I<1>:$zero_undefined);
+}
+
+def LLVM_CopySignOp : LLVM_BinarySameArgsIntrinsicOp<"copysign">;
+def LLVM_CosOp : LLVM_UnaryIntrinsicOp<"cos">;
+def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">;
+def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">;
+def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">;
+def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">;
+def LLVM_FFloorOp : LLVM_UnaryIntrinsicOp<"floor">;
+def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">;
+def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrinsicOp<"fmuladd">;
+def LLVM_Log10Op : LLVM_UnaryIntrinsicOp<"log10">;
+def LLVM_Log2Op : LLVM_UnaryIntrinsicOp<"log2">;
+def LLVM_LogOp : LLVM_UnaryIntrinsicOp<"log">;
+def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0]> {
+  let arguments = (ins LLVM_Type:$addr, LLVM_Type:$rw, LLVM_Type:$hint,
+                   LLVM_Type:$cache);
+}
+def LLVM_SinOp : LLVM_UnaryIntrinsicOp<"sin">;
+def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">;
+def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">;
+def LLVM_PowIOp : LLVM_BinaryIntrinsicOp<"powi">;
+def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">;
+def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrinsicOp<"ctlz">;
+def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrinsicOp<"cttz">;
+def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">;
+def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrinsicOp<"maxnum">;
+def LLVM_MinNumOp : LLVM_BinarySameArgsIntrinsicOp<"minnum">;
+def LLVM_MaximumOp : LLVM_BinarySameArgsIntrinsicOp<"maximum">;
+def LLVM_MinimumOp : LLVM_BinarySameArgsIntrinsicOp<"minimum">;
+def LLVM_SMaxOp : LLVM_BinarySameArgsIntrinsicOp<"smax">;
+def LLVM_SMinOp : LLVM_BinarySameArgsIntrinsicOp<"smin">;
+def LLVM_UMaxOp : LLVM_BinarySameArgsIntrinsicOp<"umax">;
+def LLVM_UMinOp : LLVM_BinarySameArgsIntrinsicOp<"umin">;
+
+def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]> {
+  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
+                   LLVM_Type:$isVolatile);
+}
+def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]> {
+  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
+                   LLVM_Type:$isVolatile);
+}
+def LLVM_MemmoveOp : LLVM_ZeroResultIntrOp<"memmove", [0, 1, 2]> {
+  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
+                   LLVM_Type:$isVolatile);
+}
+
+def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2]> {
+  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, LLVM_Type:$val, LLVM_Type:$len,
+                   LLVM_Type:$isVolatile);
+}
+
+// Intrinsics with multiple returns.
+
+def LLVM_SAddWithOverflowOp
+    : LLVM_IntrOp<"sadd.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+def LLVM_UAddWithOverflowOp
+    : LLVM_IntrOp<"uadd.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+def LLVM_SSubWithOverflowOp
+    : LLVM_IntrOp<"ssub.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+def LLVM_USubWithOverflowOp
+    : LLVM_IntrOp<"usub.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+def LLVM_SMulWithOverflowOp
+    : LLVM_IntrOp<"smul.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+def LLVM_UMulWithOverflowOp
+    : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> {
+  let arguments = (ins LLVM_Type, LLVM_Type);
+}
+
+
+def LLVM_AssumeOp : LLVM_Op<"intr.assume", []> {
+  let arguments = (ins LLVM_Type:$cond);
+  let llvmBuilder = [{
+    llvm::Module *module = builder.GetInsertBlock()->getModule();
+    llvm::Function *fn =
+        llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::assume, {});
+    builder.CreateCall(fn, {$cond});
+  }];
+}
+
+
+//
+// Coroutine intrinsics.
+//
+
+def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> {
+  let arguments = (ins I32:$align,
+                       LLVM_i8Ptr:$promise,
+                       LLVM_i8Ptr:$coroaddr,
+                       LLVM_i8Ptr:$fnaddrs);
+  let assemblyFormat = "$align `,` $promise `,` $coroaddr `,` $fnaddrs"
+    " attr-dict `:` type($res)";
+}
+
+def LLVM_CoroBeginOp : LLVM_IntrOp<"coro.begin", [], [], [], 1> {
+  let arguments = (ins LLVM_TokenType:$token,
+                       LLVM_i8Ptr:$mem);
+  let assemblyFormat = "$token `,` $mem attr-dict `:` type($res)";
+}
+
+def LLVM_CoroSizeOp : LLVM_IntrOp<"coro.size", [0], [], [], 1> {
+  let assemblyFormat = "attr-dict `:` type($res)";
+}
+
+def LLVM_CoroAlignOp : LLVM_IntrOp<"coro.align", [0], [], [], 1> {
+  let assemblyFormat = "attr-dict `:` type($res)";
+}
+
+def LLVM_CoroSaveOp : LLVM_IntrOp<"coro.save", [], [], [], 1> {
+  let arguments = (ins LLVM_i8Ptr:$handle);
+  let assemblyFormat = "$handle attr-dict `:` type($res)";
+}
+
+def LLVM_CoroSuspendOp : LLVM_IntrOp<"coro.suspend", [], [], [], 1> {
+  let arguments = (ins LLVM_TokenType:$save,
+                       I1:$final);
+  let assemblyFormat = "$save `,` $final attr-dict `:` type($res)";
+}
+
+def LLVM_CoroEndOp : LLVM_IntrOp<"coro.end", [], [], [], 1> {
+  let arguments = (ins LLVM_i8Ptr:$handle,
+                       I1:$unwind);
+  let assemblyFormat = "$handle `,` $unwind attr-dict `:` type($res)";
+}
+
+def LLVM_CoroFreeOp : LLVM_IntrOp<"coro.free", [], [], [], 1> {
+  let arguments = (ins LLVM_TokenType:$id,
+                       LLVM_i8Ptr:$handle);
+  let assemblyFormat = "$id `,` $handle attr-dict `:` type($res)";
+}
+
+def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> {
+  let arguments = (ins LLVM_i8Ptr:$handle);
+  let assemblyFormat = "$handle attr-dict";
+}
+
+//
+// Exception handling intrinsics.
+//
+
+def LLVM_EhTypeidForOp : LLVM_OneResultIntrOp<"eh.typeid.for"> {
+    let arguments = (ins LLVM_i8Ptr:$type_info);
+    let assemblyFormat = "$type_info attr-dict `:` type($res)";
+}
+
+//
+// Stack save/restore intrinsics.
+//
+
+def LLVM_StackSaveOp : LLVM_OneResultIntrOp<"stacksave"> {
+  let assemblyFormat = "attr-dict `:` type($res)";
+}
+
+def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore"> {
+  let arguments = (ins LLVM_i8Ptr:$ptr);
+  let assemblyFormat = "$ptr attr-dict";
+}
+
+//
+// Vector Reductions.
+//
+
+def LLVM_vector_reduce_add : LLVM_VectorReduction<"add">;
+def LLVM_vector_reduce_and : LLVM_VectorReduction<"and">;
+def LLVM_vector_reduce_mul : LLVM_VectorReduction<"mul">;
+def LLVM_vector_reduce_fmax : LLVM_VectorReduction<"fmax">;
+def LLVM_vector_reduce_fmin : LLVM_VectorReduction<"fmin">;
+def LLVM_vector_reduce_or : LLVM_VectorReduction<"or">;
+def LLVM_vector_reduce_smax : LLVM_VectorReduction<"smax">;
+def LLVM_vector_reduce_smin : LLVM_VectorReduction<"smin">;
+def LLVM_vector_reduce_umax : LLVM_VectorReduction<"umax">;
+def LLVM_vector_reduce_umin : LLVM_VectorReduction<"umin">;
+def LLVM_vector_reduce_xor : LLVM_VectorReduction<"xor">;
+
+def LLVM_vector_reduce_fadd : LLVM_VectorReductionAcc<"fadd">;
+def LLVM_vector_reduce_fmul : LLVM_VectorReductionAcc<"fmul">;
+
+//
+// LLVM Matrix operations.
+//
+
+/// Create a column major, strided 2-D matrix load, as specified in the LLVM
+/// MatrixBuilder.
+/// data       - Start address of the matrix read
+/// rows       - Number of rows in matrix (must be a constant)
+/// isVolatile - True if the load operation is marked as volatile.
+/// columns    - Number of columns in matrix (must be a constant)
+/// stride     - Space between columns
+def LLVM_MatrixColumnMajorLoadOp : LLVM_Op<"intr.matrix.column.major.load"> {
+  let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile,
+                   I32Attr:$rows, I32Attr:$columns);
+  let results = (outs LLVM_AnyVector:$res);
+  let builders = [LLVM_OneResultOpBuilder];
+  string llvmBuilder = [{
+    llvm::MatrixBuilder mb(builder);
+    const llvm::DataLayout &dl =
+      builder.GetInsertBlock()->getModule()->getDataLayout();
+    llvm::Type *ElemTy = moduleTranslation.convertType(
+        getVectorElementType(op.getType()));
+    llvm::Align align = dl.getABITypeAlign(ElemTy);
+    $res = mb.CreateColumnMajorLoad(
+      ElemTy, $data, align, $stride, $isVolatile, $rows,
+      $columns);
+  }];
+  let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict"
+    "`:` type($res) `from` type($data) `stride` type($stride)";
+}
+
+/// Create a column major, strided 2-D matrix store, as specified in the LLVM
+/// MatrixBuilder.
+/// matrix     - Matrix to store
+/// ptr        - Pointer to write back to
+/// isVolatile - True if the load operation is marked as volatile.
+/// rows       - Number of rows in matrix (must be a constant)
+/// columns    - Number of columns in matrix (must be a constant)
+/// stride     - Space between columns
+def LLVM_MatrixColumnMajorStoreOp : LLVM_Op<"intr.matrix.column.major.store"> {
+  let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data,
+                   LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows,
+                   I32Attr:$columns);
+  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+  string llvmBuilder = [{
+    llvm::MatrixBuilder mb(builder);
+    const llvm::DataLayout &dl =
+      builder.GetInsertBlock()->getModule()->getDataLayout();
+    Type elementType = getVectorElementType(op.getMatrix().getType());
+    llvm::Align align = dl.getABITypeAlign(
+      moduleTranslation.convertType(elementType));
+    mb.CreateColumnMajorStore(
+      $matrix, $data, align, $stride, $isVolatile,
+      $rows, $columns);
+  }];
+  let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` "
+    "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)";
+}
+
+/// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as
+/// specified in the LLVM MatrixBuilder.
+def LLVM_MatrixMultiplyOp : LLVM_Op<"intr.matrix.multiply"> {
+  let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows,
+                   I32Attr:$lhs_columns, I32Attr:$rhs_columns);
+  let results = (outs LLVM_Type:$res);
+  let builders = [LLVM_OneResultOpBuilder];
+  string llvmBuilder = [{
+    llvm::MatrixBuilder mb(builder);
+    $res = mb.CreateMatrixMultiply(
+      $lhs, $rhs, $lhs_rows, $lhs_columns,
+      $rhs_columns);
+  }];
+  let assemblyFormat = "$lhs `,` $rhs attr-dict "
+    "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)";
+}
+
+/// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D
+/// `matrix`, as specified in the LLVM MatrixBuilder.
+def LLVM_MatrixTransposeOp : LLVM_Op<"intr.matrix.transpose"> {
+  let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns);
+  let results = (outs LLVM_Type:$res);
+  let builders = [LLVM_OneResultOpBuilder];
+  string llvmBuilder = [{
+    llvm::MatrixBuilder mb(builder);
+    $res = mb.CreateMatrixTranspose(
+      $matrix, $rows, $columns);
+  }];
+  let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)";
+}
+
+//
+// LLVM masked operations.
+//
+
+/// Create a llvm.get.active.lane.mask to set a mask up to a given position.
+def LLVM_GetActiveLaneMaskOp
+    : LLVM_OneResultIntrOp<"get.active.lane.mask", [0], [0], [NoSideEffect]> {
+  let arguments = (ins LLVM_Type:$base, LLVM_Type:$n);
+  let assemblyFormat = "$base `,` $n attr-dict `:` "
+    "type($base) `,` type($n) `to` type($res)";
+}
+
+/// Create a call to Masked Load intrinsic.
+def LLVM_MaskedLoadOp : LLVM_Op<"intr.masked.load"> {
+  let arguments = (ins LLVM_Type:$data, LLVM_Type:$mask,
+                   Variadic<LLVM_Type>:$pass_thru, I32Attr:$alignment);
+  let results = (outs LLVM_AnyVector:$res);
+  string llvmBuilder = [{
+    $res = $pass_thru.empty() ? builder.CreateMaskedLoad(
+        $_resultType, $data, llvm::Align($alignment), $mask) :
+      builder.CreateMaskedLoad(
+        $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
+  }];
+  let assemblyFormat =
+    "operands attr-dict `:` functional-type(operands, results)";
+}
+
+/// Create a call to Masked Store intrinsic.
+def LLVM_MaskedStoreOp : LLVM_Op<"intr.masked.store"> {
+  let arguments = (ins LLVM_Type:$value, LLVM_Type:$data, LLVM_Type:$mask,
+                   I32Attr:$alignment);
+  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+  string llvmBuilder = [{
+    builder.CreateMaskedStore(
+      $value, $data, llvm::Align($alignment), $mask);
+  }];
+  let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` "
+    "type($value) `,` type($mask) `into` type($data)";
+}
+
+/// Create a call to Masked Gather intrinsic.
+def LLVM_masked_gather : LLVM_Op<"intr.masked.gather"> {
+  let arguments = (ins LLVM_AnyVector:$ptrs, LLVM_Type:$mask,
+                   Variadic<LLVM_Type>:$pass_thru, I32Attr:$alignment);
+  let results = (outs LLVM_Type:$res);
+  let builders = [LLVM_OneResultOpBuilder];
+  string llvmBuilder = [{
+    $res = $pass_thru.empty() ? builder.CreateMaskedGather(
+        $_resultType, $ptrs, llvm::Align($alignment), $mask) :
+      builder.CreateMaskedGather(
+        $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
+  }];
+  let assemblyFormat =
+    "operands attr-dict `:` functional-type(operands, results)";
+}
+
+/// Create a call to Masked Scatter intrinsic.
+def LLVM_masked_scatter : LLVM_Op<"intr.masked.scatter"> {
+  let arguments = (ins LLVM_Type:$value, LLVM_Type:$ptrs, LLVM_Type:$mask,
+                   I32Attr:$alignment);
+  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+  string llvmBuilder = [{
+    builder.CreateMaskedScatter(
+      $value, $ptrs, llvm::Align($alignment), $mask);
+  }];
+  let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` "
+    "type($value) `,` type($mask) `into` type($ptrs)";
+}
+
+/// Create a call to Masked Expand Load intrinsic.
+def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
+  let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type);
+}
+
+/// Create a call to Masked Compress Store intrinsic.
+def LLVM_masked_compressstore
+    : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
+  let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type);
+}
+
+/// Create a call to vscale intrinsic.
+def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>;
+
+/// Create a call to stepvector intrinsic.
+def LLVM_StepVectorOp
+    : LLVM_IntrOp<"experimental.stepvector", [0], [], [NoSideEffect], 1> {
+  let arguments = (ins);
+  let results = (outs LLVM_Type:$res);
+  let assemblyFormat = "attr-dict `:` type($res)";
+}
+
+//
+// LLVM Vector Predication operations.
+//
+
+// Integer Binary
+def LLVM_VPAddOp  : LLVM_VPBinaryI<"add">;
+def LLVM_VPSubOp  : LLVM_VPBinaryI<"sub">;
+def LLVM_VPMulOp  : LLVM_VPBinaryI<"mul">;
+def LLVM_VPSDivOp : LLVM_VPBinaryI<"sdiv">;
+def LLVM_VPUDivOp : LLVM_VPBinaryI<"udiv">;
+def LLVM_VPSRemOp : LLVM_VPBinaryI<"srem">;
+def LLVM_VPURemOp : LLVM_VPBinaryI<"urem">;
+def LLVM_VPAShrOp : LLVM_VPBinaryI<"ashr">;
+def LLVM_VPLShrOp : LLVM_VPBinaryI<"lshr">;
+def LLVM_VPShlOp  : LLVM_VPBinaryI<"shl">;
+def LLVM_VPOrOp   : LLVM_VPBinaryI<"or">;
+def LLVM_VPAndOp  : LLVM_VPBinaryI<"and">;
+def LLVM_VPXorOp  : LLVM_VPBinaryI<"xor">;
+
+// Float Binary
+def LLVM_VPFAddOp : LLVM_VPBinaryF<"fadd">;
+def LLVM_VPFSubOp : LLVM_VPBinaryF<"fsub">;
+def LLVM_VPFMulOp : LLVM_VPBinaryF<"fmul">;
+def LLVM_VPFDivOp : LLVM_VPBinaryF<"fdiv">;
+def LLVM_VPFRemOp : LLVM_VPBinaryF<"frem">;
+
+#endif // LLVM_INTRINSIC_OPS
 No newline at end of file

mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td

class LLVM_VPBinaryBase<string mnem, Type element>
     : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>,
       Arguments<(ins LLVM_VectorOf<element>:$lhs, LLVM_VectorOf<element>:$rhs,
                      LLVM_VectorOf<I1>:$mask, I32:$evl)>;
 
 class LLVM_VPBinaryI<string mnem> : LLVM_VPBinaryBase<mnem, AnyInteger>;
 
 class LLVM_VPBinaryF<string mnem> : LLVM_VPBinaryBase<mnem, AnyFloat>;
 
+def LLVM_OneResultOpBuilder :
+  OpBuilder<(ins "Type":$resultType, "ValueRange":$operands,
+    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+  [{
+    if (resultType) $_state.addTypes(resultType);
+    $_state.addOperands(operands);
+    for (auto namedAttr : attributes)
+      $_state.addAttribute(namedAttr.getName(), namedAttr.getValue());
+  }]>;
+
+def LLVM_ZeroResultOpBuilder :
+  OpBuilder<(ins "ValueRange":$operands,
+    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+  [{
+    $_state.addOperands(operands);
+    for (auto namedAttr : attributes)
+      $_state.addAttribute(namedAttr.getName(), namedAttr.getValue());
+  }]>;
+
+// Compatibility builder that takes an instance of wrapped llvm::VoidType
+// to indicate no result.
+def LLVM_VoidResultTypeOpBuilder :
+  OpBuilder<(ins "Type":$resultType, "ValueRange":$operands,
+    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+  [{
+    assert(isCompatibleType(resultType) && "result must be an LLVM type");
+    assert(resultType.isa<LLVMVoidType>() &&
+           "for zero-result operands, only 'void' is accepted as result type");
+    build($_builder, $_state, operands, attributes);
+  }]>;
+
+
+// Opaque builder used for terminator operations that contain successors.
+def LLVM_TerminatorPassthroughOpBuilder :
+  OpBuilder<(ins "ValueRange":$operands, "SuccessorRange":$destinations,
+    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
+  [{
+    $_state.addOperands(operands);
+    $_state.addSuccessors(destinations);
+    $_state.addAttributes(attributes);
+  }]>;
+
+
 #endif  // LLVMIR_OP_BASE

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

def LoopOptionCase : I32EnumAttr<
     ]> {
   let cppNamespace = "::mlir::LLVM";
 }
 
 class LLVM_Builder<string builder> {
   string llvmBuilder = builder;
 }
 
-def LLVM_OneResultOpBuilder :
-  OpBuilder<(ins "Type":$resultType, "ValueRange":$operands,
-    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-  [{
-    if (resultType) $_state.addTypes(resultType);
-    $_state.addOperands(operands);
-    for (auto namedAttr : attributes)
-      $_state.addAttribute(namedAttr.getName(), namedAttr.getValue());
-  }]>;
-
-def LLVM_ZeroResultOpBuilder :
-  OpBuilder<(ins "ValueRange":$operands,
-    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-  [{
-    $_state.addOperands(operands);
-    for (auto namedAttr : attributes)
-      $_state.addAttribute(namedAttr.getName(), namedAttr.getValue());
-  }]>;
-
-// Compatibility builder that takes an instance of wrapped llvm::VoidType
-// to indicate no result.
-def LLVM_VoidResultTypeOpBuilder :
-  OpBuilder<(ins "Type":$resultType, "ValueRange":$operands,
-    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-  [{
-    assert(isCompatibleType(resultType) && "result must be an LLVM type");
-    assert(resultType.isa<LLVMVoidType>() &&
-           "for zero-result operands, only 'void' is accepted as result type");
-    build($_builder, $_state, operands, attributes);
-  }]>;
-
-
-// Opaque builder used for terminator operations that contain successors.
-def LLVM_TerminatorPassthroughOpBuilder :
-  OpBuilder<(ins "ValueRange":$operands, "SuccessorRange":$destinations,
-    CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-  [{
-    $_state.addOperands(operands);
-    $_state.addSuccessors(destinations);
-    $_state.addAttributes(attributes);
-  }]>;
-
 // Base class for LLVM terminator operations.  All terminator operations have
 // zero results and an optional list of successors.
 class LLVM_TerminatorOp<string mnemonic, list<Trait> traits = []> :
     LLVM_Op<mnemonic, !listconcat(traits, [Terminator])>;
 
 // Class for arithmetic binary operations.
 class LLVM_ArithmeticOpBase<Type type, string mnemonic,
                             string builderFunc, list<Trait> traits = []> :
def LLVM_ConstantOp
   let arguments = (ins AnyAttr:$value);
   let results = (outs LLVM_Type:$res);
   let builders = [LLVM_OneResultOpBuilder];
   let assemblyFormat = "`(` $value `)` attr-dict `:` type($res)";
   let hasFolder = 1;
   let hasVerifier = 1;
 }
 
-// Operations that correspond to LLVM intrinsics. With MLIR operation set being
-// extendable, there is no reason to introduce a hard boundary between "core"
-// operations and intrinsics. However, we systematically prefix them with
-// "intr." to avoid potential name clashes.
-
-class LLVM_UnaryIntrinsicOp<string func, list<Trait> traits = []> :
-    LLVM_OneResultIntrOp<func, [], [0],
-           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
-  let arguments = (ins LLVM_Type:$in);
-}
-
-class LLVM_BinarySameArgsIntrinsicOp<string func, list<Trait> traits = []> :
-    LLVM_OneResultIntrOp<func, [], [0],
-           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
-  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b);
-}
-
-class LLVM_BinaryIntrinsicOp<string func, list<Trait> traits = []> :
-    LLVM_OneResultIntrOp<func, [], [0,1],
-           !listconcat([NoSideEffect], traits)> {
-  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b);
-}
-
-class LLVM_TernarySameArgsIntrinsicOp<string func, list<Trait> traits = []> :
-    LLVM_OneResultIntrOp<func, [], [0],
-           !listconcat([NoSideEffect, SameOperandsAndResultType], traits)> {
-  let arguments = (ins LLVM_Type:$a, LLVM_Type:$b, LLVM_Type:$c);
-}
-
-class LLVM_CountZerosIntrinsicOp<string func, list<Trait> traits = []> :
-    LLVM_OneResultIntrOp<func, [], [0],
-           !listconcat([NoSideEffect], traits)> {
-  let arguments = (ins LLVM_Type:$in, I<1>:$zero_undefined);
-}
-
-def LLVM_CopySignOp : LLVM_BinarySameArgsIntrinsicOp<"copysign">;
-def LLVM_CosOp : LLVM_UnaryIntrinsicOp<"cos">;
-def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">;
-def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">;
-def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">;
-def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">;
-def LLVM_FFloorOp : LLVM_UnaryIntrinsicOp<"floor">;
-def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">;
-def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrinsicOp<"fmuladd">;
-def LLVM_Log10Op : LLVM_UnaryIntrinsicOp<"log10">;
-def LLVM_Log2Op : LLVM_UnaryIntrinsicOp<"log2">;
-def LLVM_LogOp : LLVM_UnaryIntrinsicOp<"log">;
-def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0]> {
-  let arguments = (ins LLVM_Type:$addr, LLVM_Type:$rw, LLVM_Type:$hint,
-                   LLVM_Type:$cache);
-}
-def LLVM_SinOp : LLVM_UnaryIntrinsicOp<"sin">;
-def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">;
-def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">;
-def LLVM_PowIOp : LLVM_BinaryIntrinsicOp<"powi">;
-def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">;
-def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrinsicOp<"ctlz">;
-def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrinsicOp<"cttz">;
-def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">;
-def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrinsicOp<"maxnum">;
-def LLVM_MinNumOp : LLVM_BinarySameArgsIntrinsicOp<"minnum">;
-def LLVM_MaximumOp : LLVM_BinarySameArgsIntrinsicOp<"maximum">;
-def LLVM_MinimumOp : LLVM_BinarySameArgsIntrinsicOp<"minimum">;
-def LLVM_SMaxOp : LLVM_BinarySameArgsIntrinsicOp<"smax">;
-def LLVM_SMinOp : LLVM_BinarySameArgsIntrinsicOp<"smin">;
-def LLVM_UMaxOp : LLVM_BinarySameArgsIntrinsicOp<"umax">;
-def LLVM_UMinOp : LLVM_BinarySameArgsIntrinsicOp<"umin">;
-
-def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]> {
-  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
-                   LLVM_Type:$isVolatile);
-}
-def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]> {
-  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
-                   LLVM_Type:$isVolatile);
-}
-def LLVM_MemmoveOp : LLVM_ZeroResultIntrOp<"memmove", [0, 1, 2]> {
-  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, Arg<LLVM_Type,"",[MemRead]>:$src, LLVM_Type:$len,
-                   LLVM_Type:$isVolatile);
-}
-
-def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2]> {
-  let arguments = (ins Arg<LLVM_Type,"",[MemWrite]>:$dst, LLVM_Type:$val, LLVM_Type:$len,
-                   LLVM_Type:$isVolatile);
-}
-
-// Intrinsics with multiple returns.
-
-def LLVM_SAddWithOverflowOp
-    : LLVM_IntrOp<"sadd.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-def LLVM_UAddWithOverflowOp
-    : LLVM_IntrOp<"uadd.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-def LLVM_SSubWithOverflowOp
-    : LLVM_IntrOp<"ssub.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-def LLVM_USubWithOverflowOp
-    : LLVM_IntrOp<"usub.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-def LLVM_SMulWithOverflowOp
-    : LLVM_IntrOp<"smul.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-def LLVM_UMulWithOverflowOp
-    : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> {
-  let arguments = (ins LLVM_Type, LLVM_Type);
-}
-
-//
-// Coroutine intrinsics.
-//
-
-def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> {
-  let arguments = (ins I32:$align,
-                       LLVM_i8Ptr:$promise,
-                       LLVM_i8Ptr:$coroaddr,
-                       LLVM_i8Ptr:$fnaddrs);
-  let assemblyFormat = "$align `,` $promise `,` $coroaddr `,` $fnaddrs"
-    " attr-dict `:` type($res)";
-}
-
-def LLVM_CoroBeginOp : LLVM_IntrOp<"coro.begin", [], [], [], 1> {
-  let arguments = (ins LLVM_TokenType:$token,
-                       LLVM_i8Ptr:$mem);
-  let assemblyFormat = "$token `,` $mem attr-dict `:` type($res)";
-}
-
-def LLVM_CoroSizeOp : LLVM_IntrOp<"coro.size", [0], [], [], 1> {
-  let assemblyFormat = "attr-dict `:` type($res)";
-}
-
-def LLVM_CoroAlignOp : LLVM_IntrOp<"coro.align", [0], [], [], 1> {
-  let assemblyFormat = "attr-dict `:` type($res)";
-}
-
-def LLVM_CoroSaveOp : LLVM_IntrOp<"coro.save", [], [], [], 1> {
-  let arguments = (ins LLVM_i8Ptr:$handle);
-  let assemblyFormat = "$handle attr-dict `:` type($res)";
-}
-
-def LLVM_CoroSuspendOp : LLVM_IntrOp<"coro.suspend", [], [], [], 1> {
-  let arguments = (ins LLVM_TokenType:$save,
-                       I1:$final);
-  let assemblyFormat = "$save `,` $final attr-dict `:` type($res)";
-}
-
-def LLVM_CoroEndOp : LLVM_IntrOp<"coro.end", [], [], [], 1> {
-  let arguments = (ins LLVM_i8Ptr:$handle,
-                       I1:$unwind);
-  let assemblyFormat = "$handle `,` $unwind attr-dict `:` type($res)";
-}
-
-def LLVM_CoroFreeOp : LLVM_IntrOp<"coro.free", [], [], [], 1> {
-  let arguments = (ins LLVM_TokenType:$id,
-                       LLVM_i8Ptr:$handle);
-  let assemblyFormat = "$id `,` $handle attr-dict `:` type($res)";
-}
-
-def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> {
-  let arguments = (ins LLVM_i8Ptr:$handle);
-  let assemblyFormat = "$handle attr-dict";
-}
-
-//
-// Exception handling intrinsics.
-//
-
-def LLVM_EhTypeidForOp : LLVM_OneResultIntrOp<"eh.typeid.for"> {
-    let arguments = (ins LLVM_i8Ptr:$type_info);
-    let assemblyFormat = "$type_info attr-dict `:` type($res)";
-}
-
-//
-// Stack save/restore intrinsics.
-//
-
-def LLVM_StackSaveOp : LLVM_OneResultIntrOp<"stacksave"> {
-  let assemblyFormat = "attr-dict `:` type($res)";
-}
-
-def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore"> {
-  let arguments = (ins LLVM_i8Ptr:$ptr);
-  let assemblyFormat = "$ptr attr-dict";
-}
-
-//
-// Vector Reductions.
-//
-
-def LLVM_vector_reduce_add : LLVM_VectorReduction<"add">;
-def LLVM_vector_reduce_and : LLVM_VectorReduction<"and">;
-def LLVM_vector_reduce_mul : LLVM_VectorReduction<"mul">;
-def LLVM_vector_reduce_fmax : LLVM_VectorReduction<"fmax">;
-def LLVM_vector_reduce_fmin : LLVM_VectorReduction<"fmin">;
-def LLVM_vector_reduce_or : LLVM_VectorReduction<"or">;
-def LLVM_vector_reduce_smax : LLVM_VectorReduction<"smax">;
-def LLVM_vector_reduce_smin : LLVM_VectorReduction<"smin">;
-def LLVM_vector_reduce_umax : LLVM_VectorReduction<"umax">;
-def LLVM_vector_reduce_umin : LLVM_VectorReduction<"umin">;
-def LLVM_vector_reduce_xor : LLVM_VectorReduction<"xor">;
-
-def LLVM_vector_reduce_fadd : LLVM_VectorReductionAcc<"fadd">;
-def LLVM_vector_reduce_fmul : LLVM_VectorReductionAcc<"fmul">;
-
-//
-// LLVM Matrix operations.
-//
-
-/// Create a column major, strided 2-D matrix load, as specified in the LLVM
-/// MatrixBuilder.
-/// data       - Start address of the matrix read
-/// rows       - Number of rows in matrix (must be a constant)
-/// isVolatile - True if the load operation is marked as volatile.
-/// columns    - Number of columns in matrix (must be a constant)
-/// stride     - Space between columns
-def LLVM_MatrixColumnMajorLoadOp : LLVM_Op<"intr.matrix.column.major.load"> {
-  let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile,
-                   I32Attr:$rows, I32Attr:$columns);
-  let results = (outs LLVM_AnyVector:$res);
-  let builders = [LLVM_OneResultOpBuilder];
-  string llvmBuilder = [{
-    llvm::MatrixBuilder mb(builder);
-    const llvm::DataLayout &dl =
-      builder.GetInsertBlock()->getModule()->getDataLayout();
-    llvm::Type *ElemTy = moduleTranslation.convertType(
-        getVectorElementType(op.getType()));
-    llvm::Align align = dl.getABITypeAlign(ElemTy);
-    $res = mb.CreateColumnMajorLoad(
-      ElemTy, $data, align, $stride, $isVolatile, $rows,
-      $columns);
-  }];
-  let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict"
-    "`:` type($res) `from` type($data) `stride` type($stride)";
-}
-
-/// Create a column major, strided 2-D matrix store, as specified in the LLVM
-/// MatrixBuilder.
-/// matrix     - Matrix to store
-/// ptr        - Pointer to write back to
-/// isVolatile - True if the load operation is marked as volatile.
-/// rows       - Number of rows in matrix (must be a constant)
-/// columns    - Number of columns in matrix (must be a constant)
-/// stride     - Space between columns
-def LLVM_MatrixColumnMajorStoreOp : LLVM_Op<"intr.matrix.column.major.store"> {
-  let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data,
-                   LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows,
-                   I32Attr:$columns);
-  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
-  string llvmBuilder = [{
-    llvm::MatrixBuilder mb(builder);
-    const llvm::DataLayout &dl =
-      builder.GetInsertBlock()->getModule()->getDataLayout();
-    Type elementType = getVectorElementType(op.getMatrix().getType());
-    llvm::Align align = dl.getABITypeAlign(
-      moduleTranslation.convertType(elementType));
-    mb.CreateColumnMajorStore(
-      $matrix, $data, align, $stride, $isVolatile,
-      $rows, $columns);
-  }];
-  let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` "
-    "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)";
-}
-
-/// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as
-/// specified in the LLVM MatrixBuilder.
-def LLVM_MatrixMultiplyOp : LLVM_Op<"intr.matrix.multiply"> {
-  let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows,
-                   I32Attr:$lhs_columns, I32Attr:$rhs_columns);
-  let results = (outs LLVM_Type:$res);
-  let builders = [LLVM_OneResultOpBuilder];
-  string llvmBuilder = [{
-    llvm::MatrixBuilder mb(builder);
-    $res = mb.CreateMatrixMultiply(
-      $lhs, $rhs, $lhs_rows, $lhs_columns,
-      $rhs_columns);
-  }];
-  let assemblyFormat = "$lhs `,` $rhs attr-dict "
-    "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)";
-}
-
-/// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D
-/// `matrix`, as specified in the LLVM MatrixBuilder.
-def LLVM_MatrixTransposeOp : LLVM_Op<"intr.matrix.transpose"> {
-  let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns);
-  let results = (outs LLVM_Type:$res);
-  let builders = [LLVM_OneResultOpBuilder];
-  string llvmBuilder = [{
-    llvm::MatrixBuilder mb(builder);
-    $res = mb.CreateMatrixTranspose(
-      $matrix, $rows, $columns);
-  }];
-  let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)";
-}
-
-//
-// LLVM masked operations.
-//
-
-/// Create a llvm.get.active.lane.mask to set a mask up to a given position.
-def LLVM_GetActiveLaneMaskOp
-    : LLVM_OneResultIntrOp<"get.active.lane.mask", [0], [0], [NoSideEffect]> {
-  let arguments = (ins LLVM_Type:$base, LLVM_Type:$n);
-  let assemblyFormat = "$base `,` $n attr-dict `:` "
-    "type($base) `,` type($n) `to` type($res)";
-}
-
-/// Create a call to Masked Load intrinsic.
-def LLVM_MaskedLoadOp : LLVM_Op<"intr.masked.load"> {
-  let arguments = (ins LLVM_Type:$data, LLVM_Type:$mask,
-                   Variadic<LLVM_Type>:$pass_thru, I32Attr:$alignment);
-  let results = (outs LLVM_AnyVector:$res);
-  string llvmBuilder = [{
-    $res = $pass_thru.empty() ? builder.CreateMaskedLoad(
-        $_resultType, $data, llvm::Align($alignment), $mask) :
-      builder.CreateMaskedLoad(
-        $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
-  }];
-  let assemblyFormat =
-    "operands attr-dict `:` functional-type(operands, results)";
-}
-
-/// Create a call to Masked Store intrinsic.
-def LLVM_MaskedStoreOp : LLVM_Op<"intr.masked.store"> {
-  let arguments = (ins LLVM_Type:$value, LLVM_Type:$data, LLVM_Type:$mask,
-                   I32Attr:$alignment);
-  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
-  string llvmBuilder = [{
-    builder.CreateMaskedStore(
-      $value, $data, llvm::Align($alignment), $mask);
-  }];
-  let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` "
-    "type($value) `,` type($mask) `into` type($data)";
-}
-
-/// Create a call to Masked Gather intrinsic.
-def LLVM_masked_gather : LLVM_Op<"intr.masked.gather"> {
-  let arguments = (ins LLVM_AnyVector:$ptrs, LLVM_Type:$mask,
-                   Variadic<LLVM_Type>:$pass_thru, I32Attr:$alignment);
-  let results = (outs LLVM_Type:$res);
-  let builders = [LLVM_OneResultOpBuilder];
-  string llvmBuilder = [{
-    $res = $pass_thru.empty() ? builder.CreateMaskedGather(
-        $_resultType, $ptrs, llvm::Align($alignment), $mask) :
-      builder.CreateMaskedGather(
-        $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
-  }];
-  let assemblyFormat =
-    "operands attr-dict `:` functional-type(operands, results)";
-}
-
-/// Create a call to Masked Scatter intrinsic.
-def LLVM_masked_scatter : LLVM_Op<"intr.masked.scatter"> {
-  let arguments = (ins LLVM_Type:$value, LLVM_Type:$ptrs, LLVM_Type:$mask,
-                   I32Attr:$alignment);
-  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
-  string llvmBuilder = [{
-    builder.CreateMaskedScatter(
-      $value, $ptrs, llvm::Align($alignment), $mask);
-  }];
-  let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` "
-    "type($value) `,` type($mask) `into` type($ptrs)";
-}
-
-/// Create a call to Masked Expand Load intrinsic.
-def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
-  let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type);
-}
-
-/// Create a call to Masked Compress Store intrinsic.
-def LLVM_masked_compressstore
-    : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
-  let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type);
-}
-
-/// Create a call to vscale intrinsic.
-def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>;
-
-/// Create a call to stepvector intrinsic.
-def LLVM_StepVectorOp
-    : LLVM_IntrOp<"experimental.stepvector", [0], [], [NoSideEffect], 1> {
-  let arguments = (ins);
-  let results = (outs LLVM_Type:$res);
-  let assemblyFormat = "attr-dict `:` type($res)";
-}
-
 // Atomic operations.
 //
 
 def AtomicBinOpXchg : I64EnumAttrCase<"xchg", 0>;
 def AtomicBinOpAdd  : I64EnumAttrCase<"add", 1>;
 def AtomicBinOpSub  : I64EnumAttrCase<"sub", 2>;
 def AtomicBinOpAnd  : I64EnumAttrCase<"_and", 3>;
 def AtomicBinOpNand : I64EnumAttrCase<"nand", 4>;
let llvmBuilder = [{
     $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(),
                    getLLVMAtomicOrdering($success_ordering),
                    getLLVMAtomicOrdering($failure_ordering));
   }];
   let hasCustomAssemblyFormat = 1;
   let hasVerifier = 1;
 }
 
-def LLVM_AssumeOp : LLVM_Op<"intr.assume", []> {
-  let arguments = (ins LLVM_Type:$cond);
-  let llvmBuilder = [{
-    llvm::Module *module = builder.GetInsertBlock()->getModule();
-    llvm::Function *fn =
-        llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::assume, {});
-    builder.CreateCall(fn, {$cond});
-  }];
-}
-
 def LLVM_FenceOp : LLVM_Op<"fence"> {
   let arguments = (ins AtomicOrdering:$ordering, StrAttr:$syncscope);
   let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
   let llvmBuilder = [{
     llvm::LLVMContext &llvmContext = builder.getContext();
     builder.CreateFence(getLLVMAtomicOrdering($ordering),
       llvmContext.getOrInsertSyncScopeID($syncscope));
   }];
def LLVM_InlineAsmOp : LLVM_Op<"inline_asm", []> {
 
   let extraClassDeclaration = [{
     static StringRef getElementTypeAttrName() {
       return "elementtype";
     }
   }];
 }
 
-//
-// LLVM Vector Predication operations.
-//
-
-// Integer Binary
-def LLVM_VPAddOp  : LLVM_VPBinaryI<"add">;
-def LLVM_VPSubOp  : LLVM_VPBinaryI<"sub">;
-def LLVM_VPMulOp  : LLVM_VPBinaryI<"mul">;
-def LLVM_VPSDivOp : LLVM_VPBinaryI<"sdiv">;
-def LLVM_VPUDivOp : LLVM_VPBinaryI<"udiv">;
-def LLVM_VPSRemOp : LLVM_VPBinaryI<"srem">;
-def LLVM_VPURemOp : LLVM_VPBinaryI<"urem">;
-def LLVM_VPAShrOp : LLVM_VPBinaryI<"ashr">;
-def LLVM_VPLShrOp : LLVM_VPBinaryI<"lshr">;
-def LLVM_VPShlOp  : LLVM_VPBinaryI<"shl">;
-def LLVM_VPOrOp   : LLVM_VPBinaryI<"or">;
-def LLVM_VPAndOp  : LLVM_VPBinaryI<"and">;
-def LLVM_VPXorOp  : LLVM_VPBinaryI<"xor">;
-
-// Float Binary
-def LLVM_VPFAddOp : LLVM_VPBinaryF<"fadd">;
-def LLVM_VPFSubOp : LLVM_VPBinaryF<"fsub">;
-def LLVM_VPFMulOp : LLVM_VPBinaryF<"fmul">;
-def LLVM_VPFDivOp : LLVM_VPBinaryF<"fdiv">;
-def LLVM_VPFRemOp : LLVM_VPBinaryF<"frem">;
-
-
-
-
 #endif // LLVMIR_OPS

mlir/include/mlir/Dialect/OpenMP/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/OpenMP/OMP.td)
 mlir_tablegen(OmpCommon.td --gen-directive-decl --directives-dialect=OpenMP)
 add_public_tablegen_target(omp_common_td)
 
 set(LLVM_TARGET_DEFINITIONS OpenMPOps.td)
 mlir_tablegen(OpenMPOpsDialect.h.inc -gen-dialect-decls -dialect=omp)
 mlir_tablegen(OpenMPOpsDialect.cpp.inc -gen-dialect-defs -dialect=omp)
 mlir_tablegen(OpenMPOps.h.inc -gen-op-decls)
 mlir_tablegen(OpenMPOps.cpp.inc -gen-op-defs)
 mlir_tablegen(OpenMPOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(OpenMPOpsEnums.cpp.inc -gen-enum-defs)
 mlir_tablegen(OpenMPTypeInterfaces.h.inc -gen-type-interface-decls)
 mlir_tablegen(OpenMPTypeInterfaces.cpp.inc -gen-type-interface-defs)
 mlir_tablegen(OpenMPOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=omp)
 mlir_tablegen(OpenMPOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=omp)
-add_mlir_doc(OpenMPOps OpenMPDialect Dialects/ -gen-dialect-doc)
+add_mlir_doc(OpenMPOps OpenMPDialect Dialects/ -gen-dialect-doc -dialect=omp)
 add_public_tablegen_target(MLIROpenMPOpsIncGen)
 add_dependencies(OpenMPDialectDocGen omp_common_td)
 add_mlir_interface(OpenMPOpsInterfaces)

mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt

 add_mlir_dialect(X86Vector x86vector)
-add_mlir_doc(X86Vector X86Vector Dialects/ -gen-dialect-doc)
+add_mlir_doc(X86Vector X86Vector Dialects/ -gen-dialect-doc -dialect=x86vector)
 
 set(LLVM_TARGET_DEFINITIONS X86Vector.td)
 mlir_tablegen(X86VectorConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRX86VectorConversionsIncGen)

mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp

addTypes<LLVMVoidType,
            LLVMFixedVectorType,
            LLVMScalableVectorType,
            LLVMArrayType,
            LLVMStructType>();
   // clang-format on
   addOperations<
 #define GET_OP_LIST
 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
+      ,
+#define GET_OP_LIST
+#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
       >();
 
   // Support unknown operations because not all LLVM operations are registered.
   allowUnknownOperations();
 }
 
 #define GET_OP_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
if (attr.getName() != LLVM::LLVMDialect::getDataLayoutAttrName())
     return success();
   if (auto stringAttr = attr.getValue().dyn_cast<StringAttr>())
     return verifyDataLayoutString(
         stringAttr.getValue(),
         [op](const Twine &message) { op->emitOpError() << message.str(); });
 
   return op->emitOpError() << "expected '"
                            << LLVM::LLVMDialect::getDataLayoutAttrName()
-                           << "' to be a string attribute";
+                           << "' to be a string attributes";
 }
 
 LogicalResult LLVMDialect::verifyStructAttr(Operation *op, Attribute attr,
                                             Type annotatedType) {
   auto structType = annotatedType.dyn_cast<LLVMStructType>();
   if (!structType) {
     const auto emitIncorrectAnnotatedType = [&op]() {
       return op->emitError()

mlir/lib/Dialect/LLVMIR/IR/LLVMIntrinsicOps.cpp

+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
 No newline at end of file

mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp

auto extractPosition = [](ArrayAttr attr) {
     return position;
   };
 
   llvm::IRBuilder<>::FastMathFlagGuard fmfGuard(builder);
   if (auto fmf = dyn_cast<FastmathFlagsInterface>(opInst))
     builder.setFastMathFlags(getFastmathFlags(fmf));
 
 #include "mlir/Dialect/LLVMIR/LLVMConversions.inc"
+#include "mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc"
 
   // Emit function calls.  If the "callee" attribute is present, this is a
   // direct function call and we also need to look up the remapped function
   // itself.  Otherwise, this is an indirect call and the callee is the first
   // operand, look it up as a normal value.  Return the llvm::Value representing
   // the function result, which may be of llvm::VoidTy type.
   auto convertCall = [&](Operation &op) -> llvm::Value * {
     auto operands = moduleTranslation.lookupValues(op.getOperands());

utils/bazel/llvm-project-overlay/mlir/BUILD.bazel

cc_library(
     includes = ["include"],
     deps = [
         ":ControlFlowInterfaces",
         ":DataLayoutInterfaces",
         ":IR",
         ":InferTypeOpInterface",
         ":LLVMDialectAttributesIncGen",
         ":LLVMDialectInterfaceIncGen",
+        ":LLVMIntrinsicOpsIncGen",
         ":LLVMOpsIncGen",
         ":SideEffectInterfaces",
         ":Support",
         "//llvm:AsmParser",
         "//llvm:BitReader",
         "//llvm:BitWriter",
         "//llvm:Core",
         "//llvm:Support",
] + if_cuda_available([
         "@cuda//:cuda_headers",
         "@cuda//:libcuda",
     ]),
 )
 
 td_library(
     name = "LLVMOpsTdFiles",
     srcs = [
+        "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td",
         "include/mlir/Dialect/LLVMIR/LLVMOpBase.td",
         "include/mlir/Dialect/LLVMIR/LLVMOps.td",
         "include/mlir/Dialect/LLVMIR/LLVMOpsInterfaces.td",
     ],
     includes = ["include"],
     deps = [
         ":ControlFlowInterfacesTdFiles",
         ":FunctionInterfacesTdFiles",
tbl_outs = [
         ),
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/LLVMIR/LLVMOps.td",
     deps = [":LLVMOpsTdFiles"],
 )
 
 gentbl_cc_library(
+    name = "LLVMIntrinsicOpsIncGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td",
+    deps = [":LLVMOpsTdFiles"],
+)
+
+gentbl_cc_library(
     name = "LLVMConversionIncGen",
     strip_include_prefix = "include",
     tbl_outs = [
         (
             ["-gen-llvmir-conversions"],
             "include/mlir/Dialect/LLVMIR/LLVMConversions.inc",
         ),
         (
             ["-gen-enum-to-llvmir-conversions"],
             "include/mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc",
         ),
         (
             ["-gen-enum-from-llvmir-conversions"],
             "include/mlir/Dialect/LLVMIR/LLVMConversionEnumsFromLLVM.inc",
         ),
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/LLVMIR/LLVMOps.td",
     deps = [":LLVMOpsTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "LLVMIntrinsicConversionIncGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            ["-gen-llvmir-conversions"],
+            "include/mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td",
+    deps = [":LLVMOpsTdFiles"],
+)
+
 cc_library(
     name = "NVVMDialect",
     srcs = ["lib/Dialect/LLVMIR/IR/NVVMDialect.cpp"],
     hdrs = ["include/mlir/Dialect/LLVMIR/NVVMDialect.h"],
     includes = ["include"],
     deps = [
         ":IR",
         ":LLVMDialect",
cc_library(
     ],
     includes = ["include"],
     deps = [
         ":DLTIDialect",
         ":IR",
         ":LLVMConversionIncGen",
         ":LLVMDialect",
         ":LLVMIRTransforms",
+        ":LLVMIntrinsicConversionIncGen",
         ":OpenMPDialect",
         ":Support",
         "//llvm:Core",
         "//llvm:FrontendOpenMP",
         "//llvm:Support",
         "//llvm:TransformUtils",
     ],
 )
cc_library(
     name = "LLVMToLLVMIRTranslation",
     srcs = glob(["lib/Target/LLVMIR/Dialect/LLVMIR/*.cpp"]),
     hdrs = glob(["include/mlir/Target/LLVMIR/Dialect/LLVMIR/*.h"]),
     includes = ["include"],
     deps = [
         ":IR",
         ":LLVMConversionIncGen",
         ":LLVMDialect",
+        ":LLVMIntrinsicConversionIncGen",
         ":Support",
         ":ToLLVMIRTranslation",
         "//llvm:Core",
         "//llvm:Support",
     ],
 )
 
 cc_library(