Please use GitHub pull requests for new patches. Phabricator shutdown timeline
Changeset View
Changeset View
Standalone View
Standalone View
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
| Show First 20 Lines • Show All 162 Lines • ▼ Show 20 Lines | def LLVM_SMulWithOverflowOp | ||||
| let arguments = (ins LLVM_Type, LLVM_Type); | let arguments = (ins LLVM_Type, LLVM_Type); | ||||
| } | } | ||||
| def LLVM_UMulWithOverflowOp | def LLVM_UMulWithOverflowOp | ||||
| : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> { | : LLVM_IntrOp<"umul.with.overflow", [0], [], [], 2> { | ||||
| let arguments = (ins LLVM_Type, LLVM_Type); | let arguments = (ins LLVM_Type, LLVM_Type); | ||||
| } | } | ||||
| def LLVM_AssumeOp : LLVM_Op<"intr.assume", []> { | def LLVM_AssumeOp : LLVM_ZeroResultIntrOp<"assume", []> { | ||||
| let arguments = (ins LLVM_Type:$cond); | let arguments = (ins LLVM_Type:$cond); | ||||
| let llvmBuilder = [{ | |||||
| llvm::Module *module = builder.GetInsertBlock()->getModule(); | |||||
| llvm::Function *fn = | |||||
| llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::assume, {}); | |||||
| builder.CreateCall(fn, {$cond}); | |||||
| }]; | |||||
| } | } | ||||
| // | // | ||||
| // Coroutine intrinsics. | // Coroutine intrinsics. | ||||
| // | // | ||||
| def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> { | def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> { | ||||
| ▲ Show 20 Lines • Show All 116 Lines • ▼ Show 20 Lines | |||||
| /// Create a column major, strided 2-D matrix load, as specified in the LLVM | /// Create a column major, strided 2-D matrix load, as specified in the LLVM | ||||
| /// MatrixBuilder. | /// MatrixBuilder. | ||||
| /// data - Start address of the matrix read | /// data - Start address of the matrix read | ||||
| /// rows - Number of rows in matrix (must be a constant) | /// rows - Number of rows in matrix (must be a constant) | ||||
| /// isVolatile - True if the load operation is marked as volatile. | /// isVolatile - True if the load operation is marked as volatile. | ||||
| /// columns - Number of columns in matrix (must be a constant) | /// columns - Number of columns in matrix (must be a constant) | ||||
| /// stride - Space between columns | /// stride - Space between columns | ||||
| def LLVM_MatrixColumnMajorLoadOp : LLVM_Op<"intr.matrix.column.major.load"> { | def LLVM_MatrixColumnMajorLoadOp : LLVM_OneResultIntrOp<"matrix.column.major.load"> { | ||||
| let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile, | let arguments = (ins LLVM_Type:$data, LLVM_Type:$stride, I1Attr:$isVolatile, | ||||
| I32Attr:$rows, I32Attr:$columns); | I32Attr:$rows, I32Attr:$columns); | ||||
| let results = (outs LLVM_AnyVector:$res); | let results = (outs LLVM_AnyVector:$res); | ||||
| let builders = [LLVM_OneResultOpBuilder]; | let builders = [LLVM_OneResultOpBuilder]; | ||||
| let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict" | |||||
| "`:` type($res) `from` type($data) `stride` type($stride)"; | |||||
| string llvmBuilder = [{ | string llvmBuilder = [{ | ||||
| llvm::MatrixBuilder mb(builder); | llvm::MatrixBuilder mb(builder); | ||||
| const llvm::DataLayout &dl = | const llvm::DataLayout &dl = | ||||
| builder.GetInsertBlock()->getModule()->getDataLayout(); | builder.GetInsertBlock()->getModule()->getDataLayout(); | ||||
| llvm::Type *ElemTy = moduleTranslation.convertType( | llvm::Type *ElemTy = moduleTranslation.convertType( | ||||
| getVectorElementType(op.getType())); | getVectorElementType(op.getType())); | ||||
| llvm::Align align = dl.getABITypeAlign(ElemTy); | llvm::Align align = dl.getABITypeAlign(ElemTy); | ||||
| $res = mb.CreateColumnMajorLoad( | $res = mb.CreateColumnMajorLoad( | ||||
| ElemTy, $data, align, $stride, $isVolatile, $rows, | ElemTy, $data, align, $stride, $isVolatile, $rows, | ||||
| $columns); | $columns); | ||||
| }]; | }]; | ||||
| let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict" | string mlirBuilder = [{ | ||||
| "`:` type($res) `from` type($data) `stride` type($stride)"; | $res = $_builder.create<LLVM::MatrixColumnMajorLoadOp>( | ||||
| $_location, $_resultType, $data, $stride, | |||||
| $_int_attr($isVolatile), $_int_attr($rows), $_int_attr($columns)); | |||||
| }]; | |||||
| } | } | ||||
| /// Create a column major, strided 2-D matrix store, as specified in the LLVM | /// Create a column major, strided 2-D matrix store, as specified in the LLVM | ||||
| /// MatrixBuilder. | /// MatrixBuilder. | ||||
| /// matrix - Matrix to store | /// matrix - Matrix to store | ||||
| /// ptr - Pointer to write back to | /// ptr - Pointer to write back to | ||||
| /// isVolatile - True if the load operation is marked as volatile. | /// isVolatile - True if the load operation is marked as volatile. | ||||
| /// rows - Number of rows in matrix (must be a constant) | /// rows - Number of rows in matrix (must be a constant) | ||||
| /// columns - Number of columns in matrix (must be a constant) | /// columns - Number of columns in matrix (must be a constant) | ||||
| /// stride - Space between columns | /// stride - Space between columns | ||||
| def LLVM_MatrixColumnMajorStoreOp : LLVM_Op<"intr.matrix.column.major.store"> { | def LLVM_MatrixColumnMajorStoreOp : LLVM_ZeroResultIntrOp<"matrix.column.major.store"> { | ||||
| let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data, | let arguments = (ins LLVM_AnyVector:$matrix, LLVM_Type:$data, | ||||
| LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows, | LLVM_Type:$stride, I1Attr:$isVolatile, I32Attr:$rows, | ||||
| I32Attr:$columns); | I32Attr:$columns); | ||||
| let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; | let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; | ||||
| let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` " | |||||
| "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)"; | |||||
| string llvmBuilder = [{ | string llvmBuilder = [{ | ||||
| llvm::MatrixBuilder mb(builder); | llvm::MatrixBuilder mb(builder); | ||||
| const llvm::DataLayout &dl = | const llvm::DataLayout &dl = | ||||
| builder.GetInsertBlock()->getModule()->getDataLayout(); | builder.GetInsertBlock()->getModule()->getDataLayout(); | ||||
| Type elementType = getVectorElementType(op.getMatrix().getType()); | Type elementType = getVectorElementType(op.getMatrix().getType()); | ||||
| llvm::Align align = dl.getABITypeAlign( | llvm::Align align = dl.getABITypeAlign( | ||||
| moduleTranslation.convertType(elementType)); | moduleTranslation.convertType(elementType)); | ||||
| mb.CreateColumnMajorStore( | mb.CreateColumnMajorStore( | ||||
| $matrix, $data, align, $stride, $isVolatile, | $matrix, $data, align, $stride, $isVolatile, | ||||
| $rows, $columns); | $rows, $columns); | ||||
| }]; | }]; | ||||
| let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` " | string mlirBuilder = [{ | ||||
| "attr-dict`:` type($matrix) `to` type($data) `stride` type($stride)"; | $_builder.create<LLVM::MatrixColumnMajorStoreOp>( | ||||
| $_location, $matrix, $data, $stride, | |||||
| $_int_attr($isVolatile), $_int_attr($rows), $_int_attr($columns)); | |||||
| }]; | |||||
| } | } | ||||
| /// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as | /// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as | ||||
| /// specified in the LLVM MatrixBuilder. | /// specified in the LLVM MatrixBuilder. | ||||
| def LLVM_MatrixMultiplyOp : LLVM_Op<"intr.matrix.multiply"> { | def LLVM_MatrixMultiplyOp : LLVM_OneResultIntrOp<"matrix.multiply"> { | ||||
| let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows, | let arguments = (ins LLVM_Type:$lhs, LLVM_Type:$rhs, I32Attr:$lhs_rows, | ||||
| I32Attr:$lhs_columns, I32Attr:$rhs_columns); | I32Attr:$lhs_columns, I32Attr:$rhs_columns); | ||||
| let results = (outs LLVM_Type:$res); | let results = (outs LLVM_Type:$res); | ||||
| let builders = [LLVM_OneResultOpBuilder]; | let builders = [LLVM_OneResultOpBuilder]; | ||||
| let assemblyFormat = "$lhs `,` $rhs attr-dict " | |||||
| "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)"; | |||||
| string llvmBuilder = [{ | string llvmBuilder = [{ | ||||
| llvm::MatrixBuilder mb(builder); | llvm::MatrixBuilder mb(builder); | ||||
| $res = mb.CreateMatrixMultiply( | $res = mb.CreateMatrixMultiply( | ||||
| $lhs, $rhs, $lhs_rows, $lhs_columns, | $lhs, $rhs, $lhs_rows, $lhs_columns, | ||||
| $rhs_columns); | $rhs_columns); | ||||
| }]; | }]; | ||||
| let assemblyFormat = "$lhs `,` $rhs attr-dict " | string mlirBuilder = [{ | ||||
| "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)"; | $res = $_builder.create<LLVM::MatrixMultiplyOp>( | ||||
| $_location, $_resultType, $lhs, $rhs, | |||||
| $_int_attr($lhs_rows), $_int_attr($lhs_columns), $_int_attr($rhs_columns)); | |||||
| }]; | |||||
| } | } | ||||
| /// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D | /// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D | ||||
| /// `matrix`, as specified in the LLVM MatrixBuilder. | /// `matrix`, as specified in the LLVM MatrixBuilder. | ||||
| def LLVM_MatrixTransposeOp : LLVM_Op<"intr.matrix.transpose"> { | def LLVM_MatrixTransposeOp : LLVM_OneResultIntrOp<"matrix.transpose"> { | ||||
| let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns); | let arguments = (ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns); | ||||
| let results = (outs LLVM_Type:$res); | let results = (outs LLVM_Type:$res); | ||||
| let builders = [LLVM_OneResultOpBuilder]; | let builders = [LLVM_OneResultOpBuilder]; | ||||
| let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)"; | |||||
| string llvmBuilder = [{ | string llvmBuilder = [{ | ||||
| llvm::MatrixBuilder mb(builder); | llvm::MatrixBuilder mb(builder); | ||||
| $res = mb.CreateMatrixTranspose( | $res = mb.CreateMatrixTranspose( | ||||
| $matrix, $rows, $columns); | $matrix, $rows, $columns); | ||||
| }]; | }]; | ||||
| let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)"; | string mlirBuilder = [{ | ||||
| $res = $_builder.create<LLVM::MatrixTransposeOp>( | |||||
| $_location, $_resultType, $matrix, | |||||
| $_int_attr($rows), $_int_attr($columns)); | |||||
| }]; | |||||
| } | } | ||||
| // | // | ||||
| // LLVM masked operations. | // LLVM masked operations. | ||||
| // | // | ||||
| /// Create a llvm.get.active.lane.mask to set a mask up to a given position. | /// Create a llvm.get.active.lane.mask to set a mask up to a given position. | ||||
| def LLVM_GetActiveLaneMaskOp | def LLVM_GetActiveLaneMaskOp | ||||
| ▲ Show 20 Lines • Show All 319 Lines • Show Last 20 Lines | |||||