diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md --- a/mlir/docs/Dialects/LLVM.md +++ b/mlir/docs/Dialects/LLVM.md @@ -473,3 +473,12 @@ of an operation is that used in LLVM IR prefixed with "`llvm.`". [include "Dialects/LLVMOps.md"] + +## Operations for LLVM IR Intrinsics + +MLIR operation system is open making it unnecessary to introduce a hard bound +between "core" operations and "intrinsics". General LLVM IR intrinsics are +modeled as first-class operations in the LLVM dialect. Target-specific LLVM IR +intrinsics, e.g., NVVM or ROCDL, are modeled as separate dialects. + +[include "Dialects/LLVMIntrinsicOps.md"] diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt --- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt @@ -14,7 +14,13 @@ mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs) add_public_tablegen_target(MLIRLLVMOpsIncGen) +set(LLVM_TARGET_DEFINITIONS LLVMIntrinsicOps.td) +mlir_tablegen(LLVMIntrinsicOps.h.inc -gen-op-decls) +mlir_tablegen(LLVMIntrinsicOps.cpp.inc -gen-op-defs) +add_public_tablegen_target(MLIRLLVMIntrinsicOpsIncGen) + add_mlir_doc(LLVMOps LLVMOps Dialects/ -gen-op-doc) +add_mlir_doc(LLVMIntrinsicOps LLVMIntrinsicOps Dialects/ -gen-op-doc) set(LLVM_TARGET_DEFINITIONS LLVMOpsInterfaces.td) mlir_tablegen(LLVMOpsInterfaces.h.inc -gen-op-interface-decls) @@ -29,6 +35,10 @@ mlir_tablegen(LLVMConversionEnumsFromLLVM.inc -gen-enum-from-llvmir-conversions) add_public_tablegen_target(MLIRLLVMConversionsIncGen) +set(LLVM_TARGET_DEFINITIONS LLVMIntrinsicOps.td) +mlir_tablegen(LLVMIntrinsicConversions.inc -gen-llvmir-conversions) +add_public_tablegen_target(MLIRLLVMIntrinsicConversionsIncGen) + add_mlir_dialect(NVVMOps nvvm) add_mlir_doc(NVVMOps NVVMDialect Dialects/ -gen-dialect-doc -dialect=nvvm) set(LLVM_TARGET_DEFINITIONS NVVMOps.td) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h @@ -72,6 +72,8 @@ ///// Ops ///// #define GET_OP_CLASSES #include "mlir/Dialect/LLVMIR/LLVMOps.h.inc" +#define GET_OP_CLASSES +#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.h.inc" #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.h.inc" diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td new file mode 100644 --- /dev/null +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -0,0 +1,492 @@ +#ifndef LLVM_INTRINSIC_OPS +#define LLVM_INTRINSIC_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Dialect/LLVMIR/LLVMOpBase.td" +include "mlir/Interfaces/InferTypeOpInterface.td" + +// Operations that correspond to LLVM intrinsics. With MLIR operation set being +// extendable, there is no reason to introduce a hard boundary between "core" +// operations and intrinsics. However, we systematically prefix them with +// "intr." to avoid potential name clashes. + +class LLVM_UnaryIntrinsicOp traits = []> : + LLVM_OneResultIntrOp { + let arguments = (ins LLVM_Type:$in); +} + +class LLVM_BinarySameArgsIntrinsicOp traits = []> : + LLVM_OneResultIntrOp { + let arguments = (ins LLVM_Type:$a, LLVM_Type:$b); +} + +class LLVM_BinaryIntrinsicOp traits = []> : + LLVM_OneResultIntrOp { + let arguments = (ins LLVM_Type:$a, LLVM_Type:$b); +} + +class LLVM_TernarySameArgsIntrinsicOp traits = []> : + LLVM_OneResultIntrOp { + let arguments = (ins LLVM_Type:$a, LLVM_Type:$b, LLVM_Type:$c); +} + +class LLVM_CountZerosIntrinsicOp traits = []> : + LLVM_OneResultIntrOp { + let arguments = (ins LLVM_Type:$in, I<1>:$zero_undefined); +} + +def LLVM_CopySignOp : LLVM_BinarySameArgsIntrinsicOp<"copysign">; +def LLVM_CosOp : LLVM_UnaryIntrinsicOp<"cos">; +def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">; +def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">; +def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">; +def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">; +def LLVM_FFloorOp : LLVM_UnaryIntrinsicOp<"floor">; +def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">; +def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrinsicOp<"fmuladd">; +def LLVM_Log10Op : LLVM_UnaryIntrinsicOp<"log10">; +def LLVM_Log2Op : LLVM_UnaryIntrinsicOp<"log2">; +def LLVM_LogOp : LLVM_UnaryIntrinsicOp<"log">; +def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0]> { + let arguments = (ins LLVM_Type:$addr, LLVM_Type:$rw, LLVM_Type:$hint, + LLVM_Type:$cache); +} +def LLVM_SinOp : LLVM_UnaryIntrinsicOp<"sin">; +def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">; +def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">; +def LLVM_PowIOp : LLVM_BinaryIntrinsicOp<"powi">; +def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">; +def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrinsicOp<"ctlz">; +def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrinsicOp<"cttz">; +def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">; +def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrinsicOp<"maxnum">; +def LLVM_MinNumOp : LLVM_BinarySameArgsIntrinsicOp<"minnum">; +def LLVM_MaximumOp : LLVM_BinarySameArgsIntrinsicOp<"maximum">; +def LLVM_MinimumOp : LLVM_BinarySameArgsIntrinsicOp<"minimum">; +def LLVM_SMaxOp : LLVM_BinarySameArgsIntrinsicOp<"smax">; +def LLVM_SMinOp : LLVM_BinarySameArgsIntrinsicOp<"smin">; +def LLVM_UMaxOp : LLVM_BinarySameArgsIntrinsicOp<"umax">; +def LLVM_UMinOp : LLVM_BinarySameArgsIntrinsicOp<"umin">; + +def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]> { + let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, + LLVM_Type:$isVolatile); +} +def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]> { + let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, + LLVM_Type:$isVolatile); +} +def LLVM_MemmoveOp : LLVM_ZeroResultIntrOp<"memmove", [0, 1, 2]> { + let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, + LLVM_Type:$isVolatile); +} + +def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2]> { + let arguments = (ins Arg:$dst, LLVM_Type:$val, LLVM_Type:$len, + LLVM_Type:$isVolatile); +} + +// Intrinsics with multiple returns. + +def LLVM_SAddWithOverflowOp + : LLVM_IntrOp<"sadd.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} +def LLVM_UAddWithOverflowOp + : LLVM_IntrOp<"uadd.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} +def LLVM_SSubWithOverflowOp + : LLVM_IntrOp<"ssub.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} +def LLVM_USubWithOverflowOp + : LLVM_IntrOp<"usub.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} +def LLVM_SMulWithOverflowOp + : LLVM_IntrOp<"smul.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} +def LLVM_UMulWithOverflowOp + : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> { + let arguments = (ins LLVM_Type, LLVM_Type); +} + + +def LLVM_AssumeOp : LLVM_Op<"intr.assume", []> { + let arguments = (ins LLVM_Type:$cond); + let llvmBuilder = [{ + llvm::Module *module = builder.GetInsertBlock()->getModule(); + llvm::Function *fn = + llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::assume, {}); + builder.CreateCall(fn, {$cond}); + }]; +} + + +// +// Coroutine intrinsics. +// + +def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> { + let arguments = (ins I32:$align, + LLVM_i8Ptr:$promise, + LLVM_i8Ptr:$coroaddr, + LLVM_i8Ptr:$fnaddrs); + let assemblyFormat = "$align `,` $promise `,` $coroaddr `,` $fnaddrs" + " attr-dict `:` type($res)"; +} + +def LLVM_CoroBeginOp : LLVM_IntrOp<"coro.begin", [], [], [], 1> { + let arguments = (ins LLVM_TokenType:$token, + LLVM_i8Ptr:$mem); + let assemblyFormat = "$token `,` $mem attr-dict `:` type($res)"; +} + +def LLVM_CoroSizeOp : LLVM_IntrOp<"coro.size", [0], [], [], 1> { + let assemblyFormat = "attr-dict `:` type($res)"; +} + +def LLVM_CoroAlignOp : LLVM_IntrOp<"coro.align", [0], [], [], 1> { + let assemblyFormat = "attr-dict `:` type($res)"; +} + +def LLVM_CoroSaveOp : LLVM_IntrOp<"coro.save", [], [], [], 1> { + let arguments = (ins LLVM_i8Ptr:$handle); + let assemblyFormat = "$handle attr-dict `:` type($res)"; +} + +def LLVM_CoroSuspendOp : LLVM_IntrOp<"coro.suspend", [], [], [], 1> { + let arguments = (ins LLVM_TokenType:$save, + I1:$final); + let assemblyFormat = "$save `,` $final attr-dict `:` type($res)"; +} + +def LLVM_CoroEndOp : LLVM_IntrOp<"coro.end", [], [], [], 1> { + let arguments = (ins LLVM_i8Ptr:$handle, + I1:$unwind); + let assemblyFormat = "$handle `,` $unwind attr-dict `:` type($res)"; +} + +def LLVM_CoroFreeOp : LLVM_IntrOp<"coro.free", [], [], [], 1> { + let arguments = (ins LLVM_TokenType:$id, + LLVM_i8Ptr:$handle); + let assemblyFormat = "$id `,` $handle attr-dict `:` type($res)"; +} + +def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> { + let arguments = (ins LLVM_i8Ptr:$handle); + let assemblyFormat = "$handle attr-dict"; +} + +// +// Exception handling intrinsics. +// + +def LLVM_EhTypeidForOp : LLVM_OneResultIntrOp<"eh.typeid.for"> { + let arguments = (ins LLVM_i8Ptr:$type_info); + let assemblyFormat = "$type_info attr-dict `:` type($res)"; +} + +// +// Stack save/restore intrinsics. +// + +def LLVM_StackSaveOp : LLVM_OneResultIntrOp<"stacksave"> { + let assemblyFormat = "attr-dict `:` type($res)"; +} + +def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore"> { + let arguments = (ins LLVM_i8Ptr:$ptr); + let assemblyFormat = "$ptr attr-dict"; +} + +// +// Vector Reductions. +// + +def LLVM_vector_reduce_add : LLVM_VectorReduction<"add">; +def LLVM_vector_reduce_and : LLVM_VectorReduction<"and">; +def LLVM_vector_reduce_mul : LLVM_VectorReduction<"mul">; +def LLVM_vector_reduce_fmax : LLVM_VectorReduction<"fmax">; +def LLVM_vector_reduce_fmin : LLVM_VectorReduction<"fmin">; +def LLVM_vector_reduce_or : LLVM_VectorReduction<"or">; +def LLVM_vector_reduce_smax : LLVM_VectorReduction<"smax">; +def LLVM_vector_reduce_smin : LLVM_VectorReduction<"smin">; +def LLVM_vector_reduce_umax : LLVM_VectorReduction<"umax">; +def LLVM_vector_reduce_umin : LLVM_VectorReduction<"umin">; +def LLVM_vector_reduce_xor : LLVM_VectorReduction<"xor">; + +def LLVM_vector_reduce_fadd : LLVM_VectorReductionAcc<"fadd">; +def LLVM_vector_reduce_fmul : LLVM_VectorReductionAcc<"fmul">; + +// +// LLVM Matrix operations. +// + +/// Create a column major, strided 2-D matrix load, as specified in the LLVM +/// MatrixBuilder. +/// data - Start address of the matrix read +/// rows - Number of rows in matrix (must be a constant) +/// isVolatile - True if the load operation is marked as volatile. +/// columns - Number of columns in matrix (must be a constant) +/// stride - Space between columns +def LLVM_MatrixColumnMajorLoadOp : LLVM_Op<"intr.matrix.column.major.load"> { + let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile, + I32Attr:$rows, I32Attr:$columns); + let results = (outs LLVM_AnyVector:$res); + let builders = [LLVM_OneResultOpBuilder]; + string llvmBuilder = [{ + llvm::MatrixBuilder mb(builder); + const llvm::DataLayout &dl = + builder.GetInsertBlock()->getModule()->getDataLayout(); + llvm::Type *ElemTy = moduleTranslation.convertType( + getVectorElementType(op.getType())); + llvm::Align align = dl.getABITypeAlign(ElemTy); + $res = mb.CreateColumnMajorLoad( + ElemTy, $data, align, $stride, $isVolatile, $rows, + $columns); + }]; + let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict" + "`:` type($res) `from` type($data) `stride` type($stride)"; +} + +/// Create a column major, strided 2-D matrix store, as specified in the LLVM +/// MatrixBuilder. +/// matrix - Matrix to store +/// ptr - Pointer to write back to +/// isVolatile - True if the load operation is marked as volatile. +/// rows - Number of rows in matrix (must be a constant) +/// columns - Number of columns in matrix (must be a constant) +/// stride - Space between columns +def LLVM_MatrixColumnMajorStoreOp : LLVM_Op<"intr.matrix.column.major.store"> { + let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data, + LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows, + I32Attr:$columns); + let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; + string llvmBuilder = [{ + llvm::MatrixBuilder mb(builder); + const llvm::DataLayout &dl = + builder.GetInsertBlock()->getModule()->getDataLayout(); + Type elementType = getVectorElementType(op.getMatrix().getType()); + llvm::Align align = dl.getABITypeAlign( + moduleTranslation.convertType(elementType)); + mb.CreateColumnMajorStore( + $matrix, $data, align, $stride, $isVolatile, + $rows, $columns); + }]; + let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` " + "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)"; +} + +/// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as +/// specified in the LLVM MatrixBuilder. +def LLVM_MatrixMultiplyOp : LLVM_Op<"intr.matrix.multiply"> { + let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows, + I32Attr:$lhs_columns, I32Attr:$rhs_columns); + let results = (outs LLVM_Type:$res); + let builders = [LLVM_OneResultOpBuilder]; + string llvmBuilder = [{ + llvm::MatrixBuilder mb(builder); + $res = mb.CreateMatrixMultiply( + $lhs, $rhs, $lhs_rows, $lhs_columns, + $rhs_columns); + }]; + let assemblyFormat = "$lhs `,` $rhs attr-dict " + "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)"; +} + +/// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D +/// `matrix`, as specified in the LLVM MatrixBuilder. +def LLVM_MatrixTransposeOp : LLVM_Op<"intr.matrix.transpose"> { + let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns); + let results = (outs LLVM_Type:$res); + let builders = [LLVM_OneResultOpBuilder]; + string llvmBuilder = [{ + llvm::MatrixBuilder mb(builder); + $res = mb.CreateMatrixTranspose( + $matrix, $rows, $columns); + }]; + let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)"; +} + +// +// LLVM masked operations. +// + +/// Create a llvm.get.active.lane.mask to set a mask up to a given position. +def LLVM_GetActiveLaneMaskOp + : LLVM_OneResultIntrOp<"get.active.lane.mask", [0], [0], [NoSideEffect]> { + let arguments = (ins LLVM_Type:$base, LLVM_Type:$n); + let assemblyFormat = "$base `,` $n attr-dict `:` " + "type($base) `,` type($n) `to` type($res)"; +} + +/// Create a call to Masked Load intrinsic. +def LLVM_MaskedLoadOp : LLVM_Op<"intr.masked.load"> { + let arguments = (ins LLVM_Type:$data, LLVM_Type:$mask, + Variadic:$pass_thru, I32Attr:$alignment); + let results = (outs LLVM_AnyVector:$res); + string llvmBuilder = [{ + $res = $pass_thru.empty() ? builder.CreateMaskedLoad( + $_resultType, $data, llvm::Align($alignment), $mask) : + builder.CreateMaskedLoad( + $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]); + }]; + let assemblyFormat = + "operands attr-dict `:` functional-type(operands, results)"; +} + +/// Create a call to Masked Store intrinsic. +def LLVM_MaskedStoreOp : LLVM_Op<"intr.masked.store"> { + let arguments = (ins LLVM_Type:$value, LLVM_Type:$data, LLVM_Type:$mask, + I32Attr:$alignment); + let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; + string llvmBuilder = [{ + builder.CreateMaskedStore( + $value, $data, llvm::Align($alignment), $mask); + }]; + let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` " + "type($value) `,` type($mask) `into` type($data)"; +} + +/// Create a call to Masked Gather intrinsic. +def LLVM_masked_gather : LLVM_Op<"intr.masked.gather"> { + let arguments = (ins LLVM_AnyVector:$ptrs, LLVM_Type:$mask, + Variadic:$pass_thru, I32Attr:$alignment); + let results = (outs LLVM_Type:$res); + let builders = [LLVM_OneResultOpBuilder]; + string llvmBuilder = [{ + $res = $pass_thru.empty() ? builder.CreateMaskedGather( + $_resultType, $ptrs, llvm::Align($alignment), $mask) : + builder.CreateMaskedGather( + $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]); + }]; + let assemblyFormat = + "operands attr-dict `:` functional-type(operands, results)"; +} + +/// Create a call to Masked Scatter intrinsic. +def LLVM_masked_scatter : LLVM_Op<"intr.masked.scatter"> { + let arguments = (ins LLVM_Type:$value, LLVM_Type:$ptrs, LLVM_Type:$mask, + I32Attr:$alignment); + let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; + string llvmBuilder = [{ + builder.CreateMaskedScatter( + $value, $ptrs, llvm::Align($alignment), $mask); + }]; + let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` " + "type($value) `,` type($mask) `into` type($ptrs)"; +} + +/// Create a call to Masked Expand Load intrinsic. +def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> { + let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type); +} + +/// Create a call to Masked Compress Store intrinsic. +def LLVM_masked_compressstore + : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> { + let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type); +} + +/// Create a call to vscale intrinsic. +def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>; + +/// Create a call to stepvector intrinsic. +def LLVM_StepVectorOp + : LLVM_IntrOp<"experimental.stepvector", [0], [], [NoSideEffect], 1> { + let arguments = (ins); + let results = (outs LLVM_Type:$res); + let assemblyFormat = "attr-dict `:` type($res)"; +} + +// +// LLVM Vector Predication operations. +// + +class LLVM_VPBinaryBase + : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, + Arguments<(ins LLVM_VectorOf:$lhs, LLVM_VectorOf:$rhs, + LLVM_VectorOf:$mask, I32:$evl)>; + +class LLVM_VPBinaryI : LLVM_VPBinaryBase; + +class LLVM_VPBinaryF : LLVM_VPBinaryBase; + +class LLVM_VPUnaryBase + : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, + Arguments<(ins LLVM_VectorOf:$op, + LLVM_VectorOf:$mask, I32:$evl)>; + +class LLVM_VPUnaryF : LLVM_VPUnaryBase; + +class LLVM_VPTernaryBase + : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, + Arguments<(ins LLVM_VectorOf:$op1, LLVM_VectorOf:$op2, + LLVM_VectorOf:$op3, LLVM_VectorOf:$mask, + I32:$evl)>; + +class LLVM_VPTernaryF : LLVM_VPTernaryBase; + +class LLVM_VPReductionBase + : LLVM_OneResultIntrOp<"vp.reduce." # mnem, [], [1], [NoSideEffect]>, + Arguments<(ins element:$satrt_value, LLVM_VectorOf:$val, + LLVM_VectorOf:$mask, I32:$evl)>; + +class LLVM_VPReductionI : LLVM_VPReductionBase; + +class LLVM_VPReductionF : LLVM_VPReductionBase; + +// Integer Binary +def LLVM_VPAddOp : LLVM_VPBinaryI<"add">; +def LLVM_VPSubOp : LLVM_VPBinaryI<"sub">; +def LLVM_VPMulOp : LLVM_VPBinaryI<"mul">; +def LLVM_VPSDivOp : LLVM_VPBinaryI<"sdiv">; +def LLVM_VPUDivOp : LLVM_VPBinaryI<"udiv">; +def LLVM_VPSRemOp : LLVM_VPBinaryI<"srem">; +def LLVM_VPURemOp : LLVM_VPBinaryI<"urem">; +def LLVM_VPAShrOp : LLVM_VPBinaryI<"ashr">; +def LLVM_VPLShrOp : LLVM_VPBinaryI<"lshr">; +def LLVM_VPShlOp : LLVM_VPBinaryI<"shl">; +def LLVM_VPOrOp : LLVM_VPBinaryI<"or">; +def LLVM_VPAndOp : LLVM_VPBinaryI<"and">; +def LLVM_VPXorOp : LLVM_VPBinaryI<"xor">; + +// Float Binary +def LLVM_VPFAddOp : LLVM_VPBinaryF<"fadd">; +def LLVM_VPFSubOp : LLVM_VPBinaryF<"fsub">; +def LLVM_VPFMulOp : LLVM_VPBinaryF<"fmul">; +def LLVM_VPFDivOp : LLVM_VPBinaryF<"fdiv">; +def LLVM_VPFRemOp : LLVM_VPBinaryF<"frem">; + +// Float Unary +def LLVM_VPFNegOp : LLVM_VPUnaryF<"fneg">; + +// Float Ternary +def LLVM_VPFmaOp : LLVM_VPTernaryF<"fma">; + +// Integer Reduction +def LLVM_VPReduceAddOp : LLVM_VPReductionI<"add">; +def LLVM_VPReduceMulOp : LLVM_VPReductionI<"mul">; +def LLVM_VPReduceAndOp : LLVM_VPReductionI<"and">; +def LLVM_VPReduceOrOp : LLVM_VPReductionI<"or">; +def LLVM_VPReduceXorOp : LLVM_VPReductionI<"xor">; +def LLVM_VPReduceSMaxOp : LLVM_VPReductionI<"smax">; +def LLVM_VPReduceSMinOp : LLVM_VPReductionI<"smin">; +def LLVM_VPReduceUMaxOp : LLVM_VPReductionI<"umax">; +def LLVM_VPReduceUMinOp : LLVM_VPReductionI<"umin">; + +// Float Reduction +def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">; +def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">; +def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">; +def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">; + +#endif // LLVM_INTRINSIC_OPS \ No newline at end of file diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -366,38 +366,47 @@ }]; } -// LLVM vector predication intrinsics. -class LLVM_VPBinaryBase - : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, - Arguments<(ins LLVM_VectorOf:$lhs, LLVM_VectorOf:$rhs, - LLVM_VectorOf:$mask, I32:$evl)>; +def LLVM_OneResultOpBuilder : + OpBuilder<(ins "Type":$resultType, "ValueRange":$operands, + CArg<"ArrayRef", "{}">:$attributes), + [{ + if (resultType) $_state.addTypes(resultType); + $_state.addOperands(operands); + for (auto namedAttr : attributes) + $_state.addAttribute(namedAttr.getName(), namedAttr.getValue()); + }]>; + +def LLVM_ZeroResultOpBuilder : + OpBuilder<(ins "ValueRange":$operands, + CArg<"ArrayRef", "{}">:$attributes), + [{ + $_state.addOperands(operands); + for (auto namedAttr : attributes) + $_state.addAttribute(namedAttr.getName(), namedAttr.getValue()); + }]>; + +// Compatibility builder that takes an instance of wrapped llvm::VoidType +// to indicate no result. +def LLVM_VoidResultTypeOpBuilder : + OpBuilder<(ins "Type":$resultType, "ValueRange":$operands, + CArg<"ArrayRef", "{}">:$attributes), + [{ + assert(isCompatibleType(resultType) && "result must be an LLVM type"); + assert(resultType.isa() && + "for zero-result operands, only 'void' is accepted as result type"); + build($_builder, $_state, operands, attributes); + }]>; + + +// Opaque builder used for terminator operations that contain successors. +def LLVM_TerminatorPassthroughOpBuilder : + OpBuilder<(ins "ValueRange":$operands, "SuccessorRange":$destinations, + CArg<"ArrayRef", "{}">:$attributes), + [{ + $_state.addOperands(operands); + $_state.addSuccessors(destinations); + $_state.addAttributes(attributes); + }]>; -class LLVM_VPBinaryI : LLVM_VPBinaryBase; - -class LLVM_VPBinaryF : LLVM_VPBinaryBase; - -class LLVM_VPUnaryBase - : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, - Arguments<(ins LLVM_VectorOf:$op, - LLVM_VectorOf:$mask, I32:$evl)>; - -class LLVM_VPUnaryF : LLVM_VPUnaryBase; - -class LLVM_VPTernaryBase - : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [NoSideEffect]>, - Arguments<(ins LLVM_VectorOf:$op1, LLVM_VectorOf:$op2, - LLVM_VectorOf:$op3, LLVM_VectorOf:$mask, - I32:$evl)>; - -class LLVM_VPTernaryF : LLVM_VPTernaryBase; - -class LLVM_VPReductionBase - : LLVM_OneResultIntrOp<"vp.reduce." # mnem, [], [1], [NoSideEffect]>, - Arguments<(ins element:$satrt_value, LLVM_VectorOf:$val, - LLVM_VectorOf:$mask, I32:$evl)>; - -class LLVM_VPReductionI : LLVM_VPReductionBase; - -class LLVM_VPReductionF : LLVM_VPReductionBase; #endif // LLVMIR_OP_BASE diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -68,48 +68,6 @@ string llvmBuilder = builder; } -def LLVM_OneResultOpBuilder : - OpBuilder<(ins "Type":$resultType, "ValueRange":$operands, - CArg<"ArrayRef", "{}">:$attributes), - [{ - if (resultType) $_state.addTypes(resultType); - $_state.addOperands(operands); - for (auto namedAttr : attributes) - $_state.addAttribute(namedAttr.getName(), namedAttr.getValue()); - }]>; - -def LLVM_ZeroResultOpBuilder : - OpBuilder<(ins "ValueRange":$operands, - CArg<"ArrayRef", "{}">:$attributes), - [{ - $_state.addOperands(operands); - for (auto namedAttr : attributes) - $_state.addAttribute(namedAttr.getName(), namedAttr.getValue()); - }]>; - -// Compatibility builder that takes an instance of wrapped llvm::VoidType -// to indicate no result. -def LLVM_VoidResultTypeOpBuilder : - OpBuilder<(ins "Type":$resultType, "ValueRange":$operands, - CArg<"ArrayRef", "{}">:$attributes), - [{ - assert(isCompatibleType(resultType) && "result must be an LLVM type"); - assert(resultType.isa() && - "for zero-result operands, only 'void' is accepted as result type"); - build($_builder, $_state, operands, attributes); - }]>; - - -// Opaque builder used for terminator operations that contain successors. -def LLVM_TerminatorPassthroughOpBuilder : - OpBuilder<(ins "ValueRange":$operands, "SuccessorRange":$destinations, - CArg<"ArrayRef", "{}">:$attributes), - [{ - $_state.addOperands(operands); - $_state.addSuccessors(destinations); - $_state.addAttributes(attributes); - }]>; - // Base class for LLVM terminator operations. All terminator operations have // zero results and an optional list of successors. class LLVM_TerminatorOp traits = []> : @@ -1379,396 +1337,6 @@ let hasVerifier = 1; } -// Operations that correspond to LLVM intrinsics. With MLIR operation set being -// extendable, there is no reason to introduce a hard boundary between "core" -// operations and intrinsics. However, we systematically prefix them with -// "intr." to avoid potential name clashes. - -class LLVM_UnaryIntrinsicOp traits = []> : - LLVM_OneResultIntrOp { - let arguments = (ins LLVM_Type:$in); -} - -class LLVM_BinarySameArgsIntrinsicOp traits = []> : - LLVM_OneResultIntrOp { - let arguments = (ins LLVM_Type:$a, LLVM_Type:$b); -} - -class LLVM_BinaryIntrinsicOp traits = []> : - LLVM_OneResultIntrOp { - let arguments = (ins LLVM_Type:$a, LLVM_Type:$b); -} - -class LLVM_TernarySameArgsIntrinsicOp traits = []> : - LLVM_OneResultIntrOp { - let arguments = (ins LLVM_Type:$a, LLVM_Type:$b, LLVM_Type:$c); -} - -class LLVM_CountZerosIntrinsicOp traits = []> : - LLVM_OneResultIntrOp { - let arguments = (ins LLVM_Type:$in, I<1>:$zero_undefined); -} - -def LLVM_CopySignOp : LLVM_BinarySameArgsIntrinsicOp<"copysign">; -def LLVM_CosOp : LLVM_UnaryIntrinsicOp<"cos">; -def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">; -def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">; -def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">; -def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">; -def LLVM_FFloorOp : LLVM_UnaryIntrinsicOp<"floor">; -def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">; -def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrinsicOp<"fmuladd">; -def LLVM_Log10Op : LLVM_UnaryIntrinsicOp<"log10">; -def LLVM_Log2Op : LLVM_UnaryIntrinsicOp<"log2">; -def LLVM_LogOp : LLVM_UnaryIntrinsicOp<"log">; -def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0]> { - let arguments = (ins LLVM_Type:$addr, LLVM_Type:$rw, LLVM_Type:$hint, - LLVM_Type:$cache); -} -def LLVM_SinOp : LLVM_UnaryIntrinsicOp<"sin">; -def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">; -def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">; -def LLVM_PowIOp : LLVM_BinaryIntrinsicOp<"powi">; -def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">; -def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrinsicOp<"ctlz">; -def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrinsicOp<"cttz">; -def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">; -def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrinsicOp<"maxnum">; -def LLVM_MinNumOp : LLVM_BinarySameArgsIntrinsicOp<"minnum">; -def LLVM_MaximumOp : LLVM_BinarySameArgsIntrinsicOp<"maximum">; -def LLVM_MinimumOp : LLVM_BinarySameArgsIntrinsicOp<"minimum">; -def LLVM_SMaxOp : LLVM_BinarySameArgsIntrinsicOp<"smax">; -def LLVM_SMinOp : LLVM_BinarySameArgsIntrinsicOp<"smin">; -def LLVM_UMaxOp : LLVM_BinarySameArgsIntrinsicOp<"umax">; -def LLVM_UMinOp : LLVM_BinarySameArgsIntrinsicOp<"umin">; - -def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]> { - let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, - LLVM_Type:$isVolatile); -} -def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]> { - let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, - LLVM_Type:$isVolatile); -} -def LLVM_MemmoveOp : LLVM_ZeroResultIntrOp<"memmove", [0, 1, 2]> { - let arguments = (ins Arg:$dst, Arg:$src, LLVM_Type:$len, - LLVM_Type:$isVolatile); -} - -def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2]> { - let arguments = (ins Arg:$dst, LLVM_Type:$val, LLVM_Type:$len, - LLVM_Type:$isVolatile); -} - -// Intrinsics with multiple returns. - -def LLVM_SAddWithOverflowOp - : LLVM_IntrOp<"sadd.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} -def LLVM_UAddWithOverflowOp - : LLVM_IntrOp<"uadd.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} -def LLVM_SSubWithOverflowOp - : LLVM_IntrOp<"ssub.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} -def LLVM_USubWithOverflowOp - : LLVM_IntrOp<"usub.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} -def LLVM_SMulWithOverflowOp - : LLVM_IntrOp<"smul.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} -def LLVM_UMulWithOverflowOp - : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> { - let arguments = (ins LLVM_Type, LLVM_Type); -} - -// -// Coroutine intrinsics. -// - -def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> { - let arguments = (ins I32:$align, - LLVM_i8Ptr:$promise, - LLVM_i8Ptr:$coroaddr, - LLVM_i8Ptr:$fnaddrs); - let assemblyFormat = "$align `,` $promise `,` $coroaddr `,` $fnaddrs" - " attr-dict `:` type($res)"; -} - -def LLVM_CoroBeginOp : LLVM_IntrOp<"coro.begin", [], [], [], 1> { - let arguments = (ins LLVM_TokenType:$token, - LLVM_i8Ptr:$mem); - let assemblyFormat = "$token `,` $mem attr-dict `:` type($res)"; -} - -def LLVM_CoroSizeOp : LLVM_IntrOp<"coro.size", [0], [], [], 1> { - let assemblyFormat = "attr-dict `:` type($res)"; -} - -def LLVM_CoroAlignOp : LLVM_IntrOp<"coro.align", [0], [], [], 1> { - let assemblyFormat = "attr-dict `:` type($res)"; -} - -def LLVM_CoroSaveOp : LLVM_IntrOp<"coro.save", [], [], [], 1> { - let arguments = (ins LLVM_i8Ptr:$handle); - let assemblyFormat = "$handle attr-dict `:` type($res)"; -} - -def LLVM_CoroSuspendOp : LLVM_IntrOp<"coro.suspend", [], [], [], 1> { - let arguments = (ins LLVM_TokenType:$save, - I1:$final); - let assemblyFormat = "$save `,` $final attr-dict `:` type($res)"; -} - -def LLVM_CoroEndOp : LLVM_IntrOp<"coro.end", [], [], [], 1> { - let arguments = (ins LLVM_i8Ptr:$handle, - I1:$unwind); - let assemblyFormat = "$handle `,` $unwind attr-dict `:` type($res)"; -} - -def LLVM_CoroFreeOp : LLVM_IntrOp<"coro.free", [], [], [], 1> { - let arguments = (ins LLVM_TokenType:$id, - LLVM_i8Ptr:$handle); - let assemblyFormat = "$id `,` $handle attr-dict `:` type($res)"; -} - -def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> { - let arguments = (ins LLVM_i8Ptr:$handle); - let assemblyFormat = "$handle attr-dict"; -} - -// -// Exception handling intrinsics. -// - -def LLVM_EhTypeidForOp : LLVM_OneResultIntrOp<"eh.typeid.for"> { - let arguments = (ins LLVM_i8Ptr:$type_info); - let assemblyFormat = "$type_info attr-dict `:` type($res)"; -} - -// -// Stack save/restore intrinsics. -// - -def LLVM_StackSaveOp : LLVM_OneResultIntrOp<"stacksave"> { - let assemblyFormat = "attr-dict `:` type($res)"; -} - -def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore"> { - let arguments = (ins LLVM_i8Ptr:$ptr); - let assemblyFormat = "$ptr attr-dict"; -} - -// -// Vector Reductions. -// - -def LLVM_vector_reduce_add : LLVM_VectorReduction<"add">; -def LLVM_vector_reduce_and : LLVM_VectorReduction<"and">; -def LLVM_vector_reduce_mul : LLVM_VectorReduction<"mul">; -def LLVM_vector_reduce_fmax : LLVM_VectorReduction<"fmax">; -def LLVM_vector_reduce_fmin : LLVM_VectorReduction<"fmin">; -def LLVM_vector_reduce_or : LLVM_VectorReduction<"or">; -def LLVM_vector_reduce_smax : LLVM_VectorReduction<"smax">; -def LLVM_vector_reduce_smin : LLVM_VectorReduction<"smin">; -def LLVM_vector_reduce_umax : LLVM_VectorReduction<"umax">; -def LLVM_vector_reduce_umin : LLVM_VectorReduction<"umin">; -def LLVM_vector_reduce_xor : LLVM_VectorReduction<"xor">; - -def LLVM_vector_reduce_fadd : LLVM_VectorReductionAcc<"fadd">; -def LLVM_vector_reduce_fmul : LLVM_VectorReductionAcc<"fmul">; - -// -// LLVM Matrix operations. -// - -/// Create a column major, strided 2-D matrix load, as specified in the LLVM -/// MatrixBuilder. -/// data - Start address of the matrix read -/// rows - Number of rows in matrix (must be a constant) -/// isVolatile - True if the load operation is marked as volatile. -/// columns - Number of columns in matrix (must be a constant) -/// stride - Space between columns -def LLVM_MatrixColumnMajorLoadOp : LLVM_Op<"intr.matrix.column.major.load"> { - let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile, - I32Attr:$rows, I32Attr:$columns); - let results = (outs LLVM_AnyVector:$res); - let builders = [LLVM_OneResultOpBuilder]; - string llvmBuilder = [{ - llvm::MatrixBuilder mb(builder); - const llvm::DataLayout &dl = - builder.GetInsertBlock()->getModule()->getDataLayout(); - llvm::Type *ElemTy = moduleTranslation.convertType( - getVectorElementType(op.getType())); - llvm::Align align = dl.getABITypeAlign(ElemTy); - $res = mb.CreateColumnMajorLoad( - ElemTy, $data, align, $stride, $isVolatile, $rows, - $columns); - }]; - let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict" - "`:` type($res) `from` type($data) `stride` type($stride)"; -} - -/// Create a column major, strided 2-D matrix store, as specified in the LLVM -/// MatrixBuilder. -/// matrix - Matrix to store -/// ptr - Pointer to write back to -/// isVolatile - True if the load operation is marked as volatile. -/// rows - Number of rows in matrix (must be a constant) -/// columns - Number of columns in matrix (must be a constant) -/// stride - Space between columns -def LLVM_MatrixColumnMajorStoreOp : LLVM_Op<"intr.matrix.column.major.store"> { - let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data, - LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows, - I32Attr:$columns); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; - string llvmBuilder = [{ - llvm::MatrixBuilder mb(builder); - const llvm::DataLayout &dl = - builder.GetInsertBlock()->getModule()->getDataLayout(); - Type elementType = getVectorElementType(op.getMatrix().getType()); - llvm::Align align = dl.getABITypeAlign( - moduleTranslation.convertType(elementType)); - mb.CreateColumnMajorStore( - $matrix, $data, align, $stride, $isVolatile, - $rows, $columns); - }]; - let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` " - "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)"; -} - -/// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as -/// specified in the LLVM MatrixBuilder. -def LLVM_MatrixMultiplyOp : LLVM_Op<"intr.matrix.multiply"> { - let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows, - I32Attr:$lhs_columns, I32Attr:$rhs_columns); - let results = (outs LLVM_Type:$res); - let builders = [LLVM_OneResultOpBuilder]; - string llvmBuilder = [{ - llvm::MatrixBuilder mb(builder); - $res = mb.CreateMatrixMultiply( - $lhs, $rhs, $lhs_rows, $lhs_columns, - $rhs_columns); - }]; - let assemblyFormat = "$lhs `,` $rhs attr-dict " - "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)"; -} - -/// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D -/// `matrix`, as specified in the LLVM MatrixBuilder. -def LLVM_MatrixTransposeOp : LLVM_Op<"intr.matrix.transpose"> { - let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns); - let results = (outs LLVM_Type:$res); - let builders = [LLVM_OneResultOpBuilder]; - string llvmBuilder = [{ - llvm::MatrixBuilder mb(builder); - $res = mb.CreateMatrixTranspose( - $matrix, $rows, $columns); - }]; - let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)"; -} - -// -// LLVM masked operations. -// - -/// Create a llvm.get.active.lane.mask to set a mask up to a given position. -def LLVM_GetActiveLaneMaskOp - : LLVM_OneResultIntrOp<"get.active.lane.mask", [0], [0], [NoSideEffect]> { - let arguments = (ins LLVM_Type:$base, LLVM_Type:$n); - let assemblyFormat = "$base `,` $n attr-dict `:` " - "type($base) `,` type($n) `to` type($res)"; -} - -/// Create a call to Masked Load intrinsic. -def LLVM_MaskedLoadOp : LLVM_Op<"intr.masked.load"> { - let arguments = (ins LLVM_Type:$data, LLVM_Type:$mask, - Variadic:$pass_thru, I32Attr:$alignment); - let results = (outs LLVM_AnyVector:$res); - string llvmBuilder = [{ - $res = $pass_thru.empty() ? builder.CreateMaskedLoad( - $_resultType, $data, llvm::Align($alignment), $mask) : - builder.CreateMaskedLoad( - $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]); - }]; - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -/// Create a call to Masked Store intrinsic. -def LLVM_MaskedStoreOp : LLVM_Op<"intr.masked.store"> { - let arguments = (ins LLVM_Type:$value, LLVM_Type:$data, LLVM_Type:$mask, - I32Attr:$alignment); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; - string llvmBuilder = [{ - builder.CreateMaskedStore( - $value, $data, llvm::Align($alignment), $mask); - }]; - let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` " - "type($value) `,` type($mask) `into` type($data)"; -} - -/// Create a call to Masked Gather intrinsic. -def LLVM_masked_gather : LLVM_Op<"intr.masked.gather"> { - let arguments = (ins LLVM_AnyVector:$ptrs, LLVM_Type:$mask, - Variadic:$pass_thru, I32Attr:$alignment); - let results = (outs LLVM_Type:$res); - let builders = [LLVM_OneResultOpBuilder]; - string llvmBuilder = [{ - $res = $pass_thru.empty() ? builder.CreateMaskedGather( - $_resultType, $ptrs, llvm::Align($alignment), $mask) : - builder.CreateMaskedGather( - $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]); - }]; - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -/// Create a call to Masked Scatter intrinsic. -def LLVM_masked_scatter : LLVM_Op<"intr.masked.scatter"> { - let arguments = (ins LLVM_Type:$value, LLVM_Type:$ptrs, LLVM_Type:$mask, - I32Attr:$alignment); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; - string llvmBuilder = [{ - builder.CreateMaskedScatter( - $value, $ptrs, llvm::Align($alignment), $mask); - }]; - let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` " - "type($value) `,` type($mask) `into` type($ptrs)"; -} - -/// Create a call to Masked Expand Load intrinsic. -def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> { - let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type); -} - -/// Create a call to Masked Compress Store intrinsic. -def LLVM_masked_compressstore - : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> { - let arguments = (ins LLVM_Type, LLVM_Type, LLVM_Type); -} - -/// Create a call to vscale intrinsic. -def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>; - -/// Create a call to stepvector intrinsic. -def LLVM_StepVectorOp - : LLVM_IntrOp<"experimental.stepvector", [0], [], [NoSideEffect], 1> { - let arguments = (ins); - let results = (outs LLVM_Type:$res); - let assemblyFormat = "attr-dict `:` type($res)"; -} - // Atomic operations. // @@ -1857,16 +1425,6 @@ let hasVerifier = 1; } -def LLVM_AssumeOp : LLVM_Op<"intr.assume", []> { - let arguments = (ins LLVM_Type:$cond); - let llvmBuilder = [{ - llvm::Module *module = builder.GetInsertBlock()->getModule(); - llvm::Function *fn = - llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::assume, {}); - builder.CreateCall(fn, {$cond}); - }]; -} - def LLVM_FenceOp : LLVM_Op<"fence"> { let arguments = (ins AtomicOrdering:$ordering, StrAttr:$syncscope); let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; @@ -1931,54 +1489,4 @@ }]; } -// -// LLVM Vector Predication operations. -// - -// Integer Binary -def LLVM_VPAddOp : LLVM_VPBinaryI<"add">; -def LLVM_VPSubOp : LLVM_VPBinaryI<"sub">; -def LLVM_VPMulOp : LLVM_VPBinaryI<"mul">; -def LLVM_VPSDivOp : LLVM_VPBinaryI<"sdiv">; -def LLVM_VPUDivOp : LLVM_VPBinaryI<"udiv">; -def LLVM_VPSRemOp : LLVM_VPBinaryI<"srem">; -def LLVM_VPURemOp : LLVM_VPBinaryI<"urem">; -def LLVM_VPAShrOp : LLVM_VPBinaryI<"ashr">; -def LLVM_VPLShrOp : LLVM_VPBinaryI<"lshr">; -def LLVM_VPShlOp : LLVM_VPBinaryI<"shl">; -def LLVM_VPOrOp : LLVM_VPBinaryI<"or">; -def LLVM_VPAndOp : LLVM_VPBinaryI<"and">; -def LLVM_VPXorOp : LLVM_VPBinaryI<"xor">; - -// Float Binary -def LLVM_VPFAddOp : LLVM_VPBinaryF<"fadd">; -def LLVM_VPFSubOp : LLVM_VPBinaryF<"fsub">; -def LLVM_VPFMulOp : LLVM_VPBinaryF<"fmul">; -def LLVM_VPFDivOp : LLVM_VPBinaryF<"fdiv">; -def LLVM_VPFRemOp : LLVM_VPBinaryF<"frem">; - -// Float Unary -def LLVM_VPFNegOp : LLVM_VPUnaryF<"fneg">; - -// Float Ternary -def LLVM_VPFmaOp : LLVM_VPTernaryF<"fma">; - -// Integer Reduction -def LLVM_VPReduceAddOp : LLVM_VPReductionI<"add">; -def LLVM_VPReduceMulOp : LLVM_VPReductionI<"mul">; -def LLVM_VPReduceAndOp : LLVM_VPReductionI<"and">; -def LLVM_VPReduceOrOp : LLVM_VPReductionI<"or">; -def LLVM_VPReduceXorOp : LLVM_VPReductionI<"xor">; -def LLVM_VPReduceSMaxOp : LLVM_VPReductionI<"smax">; -def LLVM_VPReduceSMinOp : LLVM_VPReductionI<"smin">; -def LLVM_VPReduceUMaxOp : LLVM_VPReductionI<"umax">; -def LLVM_VPReduceUMinOp : LLVM_VPReductionI<"umin">; - -// Float Reduction -def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">; -def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">; -def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">; -def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">; - - #endif // LLVMIR_OPS diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt --- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt @@ -3,6 +3,7 @@ add_mlir_dialect_library(MLIRLLVMIR IR/FunctionCallUtils.cpp IR/LLVMDialect.cpp + IR/LLVMIntrinsicOps.cpp IR/LLVMTypes.cpp IR/LLVMTypeSyntax.cpp @@ -11,6 +12,7 @@ DEPENDS MLIRLLVMOpsIncGen + MLIRLLVMIntrinsicOpsIncGen MLIRLLVMOpsInterfacesIncGen MLIROpenMPOpsIncGen intrinsics_gen diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2600,6 +2600,9 @@ addOperations< #define GET_OP_LIST #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc" + , +#define GET_OP_LIST +#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc" >(); // Support unknown operations because not all LLVM operations are registered. @@ -2700,7 +2703,7 @@ return op->emitOpError() << "expected '" << LLVM::LLVMDialect::getDataLayoutAttrName() - << "' to be a string attribute"; + << "' to be a string attributes"; } LogicalResult LLVMDialect::verifyStructAttr(Operation *op, Attribute attr, diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMIntrinsicOps.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMIntrinsicOps.cpp new file mode 100644 --- /dev/null +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMIntrinsicOps.cpp @@ -0,0 +1,7 @@ +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; +using namespace mlir::LLVM; + +#define GET_OP_CLASSES +#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc" \ No newline at end of file diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -268,6 +268,7 @@ builder.setFastMathFlags(getFastmathFlags(fmf)); #include "mlir/Dialect/LLVMIR/LLVMConversions.inc" +#include "mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc" // Emit function calls. If the "callee" attribute is present, this is a // direct function call and we also need to look up the remapped function diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3172,6 +3172,7 @@ ":InferTypeOpInterface", ":LLVMDialectAttributesIncGen", ":LLVMDialectInterfaceIncGen", + ":LLVMIntrinsicOpsIncGen", ":LLVMOpsIncGen", ":SideEffectInterfaces", ":Support", @@ -3436,6 +3437,7 @@ td_library( name = "LLVMOpsTdFiles", srcs = [ + "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td", "include/mlir/Dialect/LLVMIR/LLVMOpBase.td", "include/mlir/Dialect/LLVMIR/LLVMOps.td", "include/mlir/Dialect/LLVMIR/LLVMOpsInterfaces.td", @@ -3784,6 +3786,24 @@ deps = [":LLVMOpsTdFiles"], ) +gentbl_cc_library( + name = "LLVMIntrinsicOpsIncGen", + strip_include_prefix = "include", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.h.inc", + ), + ( + ["-gen-op-defs"], + "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td", + deps = [":LLVMOpsTdFiles"], +) + gentbl_cc_library( name = "LLVMConversionIncGen", strip_include_prefix = "include", @@ -3806,6 +3826,20 @@ deps = [":LLVMOpsTdFiles"], ) +gentbl_cc_library( + name = "LLVMIntrinsicConversionIncGen", + strip_include_prefix = "include", + tbl_outs = [ + ( + ["-gen-llvmir-conversions"], + "include/mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td", + deps = [":LLVMOpsTdFiles"], +) + cc_library( name = "NVVMDialect", srcs = ["lib/Dialect/LLVMIR/IR/NVVMDialect.cpp"], @@ -5587,6 +5621,7 @@ ":LLVMConversionIncGen", ":LLVMDialect", ":LLVMIRTransforms", + ":LLVMIntrinsicConversionIncGen", ":OpenMPDialect", ":Support", "//llvm:Core", @@ -5696,6 +5731,7 @@ ":IR", ":LLVMConversionIncGen", ":LLVMDialect", + ":LLVMIntrinsicConversionIncGen", ":Support", ":ToLLVMIRTranslation", "//llvm:Core",