diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td --- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td +++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td @@ -17,8 +17,7 @@ def MapNestedForallToThreads : Op, TransformEachOpTrait, TransformOpInterface]> { let description = [{ @@ -72,9 +71,7 @@ scf.forall operations with mappings other than gpu.thread are ignored. - The returned handle points to the same LaunchOp operand, consuming it and - producing a new SSA value to satisfy chaining and linearity of the IR - properties. + This operation returns nothing. #### Example: @@ -111,11 +108,11 @@ ``` }]; - let arguments = (ins PDL_Operation:$target, + let arguments = (ins TransformHandleTypeInterface:$target, DefaultValuedAttr:$block_dims, DefaultValuedOptionalAttr:$warp_dims, DefaultValuedAttr:$sync_after_distribute); - let results = (outs PDL_Operation:$result); + let results = (outs); let assemblyFormat = [{ $target @@ -123,6 +120,7 @@ (`warp_dims` `=` $warp_dims^)? (`sync_after_distribute` `=` $sync_after_distribute^)? attr-dict + `:` functional-type(operands, results) }]; let extraClassDeclaration = [{ ::mlir::DiagnosedSilenceableFailure applyToOne( diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -1651,11 +1651,13 @@ //===----------------------------------------------------------------------===// def VectorizeOp : Op { + [DeclareOpInterfaceMethods, + TransformEachOpTrait, + TransformOpInterface]> { let description = [{ Indicates that the given `target` op all the ops it contains should be vectorized with the configuration specified by the attributes of this op. + This vectorization only handles structured ops that operate on shaped types and does not vectorize loops or straight-line. Internally, it applies a set of rewrite patterns, some of which enable vectorization and some of @@ -1685,24 +1687,22 @@ This operation produces `definiteFailure` if vectorization fails for any reason. - The operation always returns the handle to the target op that is expected - to be isolated from above. + This operation returns nothing. }]; - let arguments = (ins PDL_Operation:$target, + let arguments = (ins TransformHandleTypeInterface:$target, UnitAttr:$vectorize_padding, UnitAttr:$vectorize_nd_extract, UnitAttr:$disable_multi_reduction_to_contract_patterns, UnitAttr:$disable_transfer_permutation_map_lowering_patterns); - let results = (outs PDL_Operation:$transformed); + let results = (outs); - let assemblyFormat = "$target attr-dict"; + let assemblyFormat = [{ + $target + attr-dict + `:` functional-type(operands, results) + }]; - let builders = [ - OpBuilder<(ins "Value":$target, - CArg<"bool", "false">:$vectorizePadding, - CArg<"bool", "false">:$vectorizeNDExtract)>, - ]; let extraClassDeclaration = [{ ::mlir::DiagnosedSilenceableFailure applyToOne( ::mlir::Operation *target, @@ -1711,6 +1711,10 @@ }]; } +//===----------------------------------------------------------------------===// +// MaskedVectorizeOp +//===----------------------------------------------------------------------===// + def MaskedVectorizeOp : Op, TransformOpInterface]> { @@ -1765,8 +1769,9 @@ def HoistRedundantVectorTransfersOp : Op { + [DeclareOpInterfaceMethods, + TransformEachOpTrait, + TransformOpInterface]> { let description = [{ Hoist vector.transfer_read / vector.transfer_write pairs out of immediately enclosing scf::ForOp iteratively, if the following conditions are true: @@ -1782,18 +1787,17 @@ #### Return modes: - The operation always succeeds and returns a handle to the transformed - function op. + The operation always succeeds and returns nothing. }]; let arguments = (ins TransformHandleTypeInterface:$target); - let results = (outs TransformHandleTypeInterface:$transformed); - - let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) "; + let results = (outs); + let assemblyFormat = [{ + $target + attr-dict + `:` functional-type(operands, results) + }]; - let builders = [ - OpBuilder<(ins "Value":$target)>, - ]; let extraClassDeclaration = [{ ::mlir::DiagnosedSilenceableFailure applyToOne( ::mlir::func::FuncOp target, @@ -1884,8 +1888,9 @@ def HoistRedundantTensorSubsetsOp : Op { + [DeclareOpInterfaceMethods, + TransformEachOpTrait, + TransformOpInterface]> { let description = [{ Hoists supported tensor subset extract/insert operation pairs out of immediately enclosing loop iteratively, if the following conditions @@ -1905,18 +1910,18 @@ #### Return modes: - The operation always succeeds and returns a handle to the transformed - function op. + The operation always succeeds and returns nothing. }]; let arguments = (ins TransformHandleTypeInterface:$target); - let results = (outs TransformHandleTypeInterface:$transformed); + let results = (outs); - let assemblyFormat = "$target attr-dict `:` functional-type(operands, results) "; + let assemblyFormat = [{ + $target + attr-dict + `:` functional-type(operands, results) + }]; - let builders = [ - OpBuilder<(ins "Value":$target)>, - ]; let extraClassDeclaration = [{ ::mlir::DiagnosedSilenceableFailure applyToOne( ::mlir::Operation *target, diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td --- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td +++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td @@ -17,8 +17,9 @@ include "mlir/IR/OpBase.td" def LowerVectorsOp : Op, - DeclareOpInterfaceMethods]> { + [DeclareOpInterfaceMethods, + TransformEachOpTrait, + TransformOpInterface]> { let description = [{ Indicates that the vector operations nested under the isolated from above op `target` should be lowered to finer-grained vector primitives. @@ -27,10 +28,14 @@ This is usally a late step that is run after bufferization as part of the process of lowering to e.g. LLVM or NVVM. + + #### Return modes: + + The operation returns nothing. }]; // TODO: evolve this to proper enums. - let arguments = (ins PDL_Operation:$target, + let arguments = (ins TransformHandleTypeInterface:$target, DefaultValuedAttr:$contraction_lowering, DefaultValuedAttr:$transpose_avx2_lowering, DefaultValuedAttr:$unroll_vector_transfers ); - let results = (outs PDL_Operation:$results); + let results = (outs); let builders = [ OpBuilder<(ins "Type":$resultType, "Value":$target, @@ -66,6 +71,14 @@ | `transpose_lowering` `=` $transpose_lowering ) attr-dict + `:` functional-type(operands, results) + }]; + + let extraClassDeclaration = [{ + ::mlir::DiagnosedSilenceableFailure applyToOne( + ::mlir::Operation *target, + ::mlir::transform::ApplyToEachResultList &results, + ::mlir::transform::TransformState &state); }]; } diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp --- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp +++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp @@ -848,6 +848,12 @@ return DiagnosedSilenceableFailure::success(); } +void transform::MapNestedForallToThreads::getEffects( + SmallVectorImpl &effects) { + onlyReadsHandle(getTarget(), effects); + modifiesPayload(effects); +} + DiagnosedSilenceableFailure transform::MapNestedForallToThreads::applyToOne( Operation *target, ApplyToEachResultList &results, TransformState &state) { LaunchOp gpuLaunch = dyn_cast(target); @@ -880,7 +886,6 @@ mapNestedForallToThreadsImpl(rewriter, transformOp, gpuLaunch, blockDims, getWarpDims(), getSyncAfterDistribute()); - results.push_back(gpuLaunch.getOperation()); return diag; } diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -1786,7 +1786,7 @@ } //===---------------------------------------------------------------------===// -// HoistPadOp +// PadOp //===---------------------------------------------------------------------===// DiagnosedSilenceableFailure @@ -2977,21 +2977,6 @@ // VectorizeOp //===----------------------------------------------------------------------===// -void transform::VectorizeOp::build(OpBuilder &builder, OperationState &result, - Value target, bool vectorizePadding, - bool vectorizeExtract) { - result.addOperands(target); - if (vectorizePadding) { - result.addAttribute(VectorizeOp::getVectorizePaddingAttrName(result.name), - builder.getUnitAttr()); - } - if (vectorizeExtract) { - result.addAttribute(VectorizeOp::getVectorizeNdExtractAttrName(result.name), - builder.getUnitAttr()); - } - result.addTypes(pdl::OperationType::get(builder.getContext())); -} - namespace { /// This is an helper only to call vectorize via a pattern inside of /// VectorizeOp::applyToOne. @@ -3050,10 +3035,15 @@ if (failed(applyPatternsAndFoldGreedily(target, std::move(patterns)))) return emitDefaultDefiniteFailure(target); - results.push_back(target); return DiagnosedSilenceableFailure::success(); } +void transform::VectorizeOp::getEffects( + SmallVectorImpl &effects) { + transform::onlyReadsHandle(getTarget(), effects); + transform::modifiesPayload(effects); +} + //===----------------------------------------------------------------------===// // MaskedVectorizeOp //===----------------------------------------------------------------------===// @@ -3133,22 +3123,6 @@ return getMixedValues(getStaticVectorSizes(), getVectorSizes(), b); } -//===----------------------------------------------------------------------===// -// HoistRedundantVectorTransfersOp -//===----------------------------------------------------------------------===// - -DiagnosedSilenceableFailure -transform::HoistRedundantVectorTransfersOp::applyToOne( - func::FuncOp target, transform::ApplyToEachResultList &results, - transform::TransformState &state) { - // WARNING: This hoisting does not model parallelism and is generally - // incorrect when used on distributed loops with memref semantics! - // TODO: obsolete and should be retired. - linalg::hoistRedundantVectorTransfers(target); - results.push_back(target); - return DiagnosedSilenceableFailure::success(); -} - //===----------------------------------------------------------------------===// // ConvertConv2DToImg2ColOp. //===----------------------------------------------------------------------===// @@ -3193,9 +3167,7 @@ IRRewriter rewriter(target->getContext()); auto forOp = dyn_cast(target); if (forOp) { - scf::ForOp newForOp = - linalg::hoistRedundantSubsetExtractInsert(rewriter, forOp); - results.push_back(newForOp); + linalg::hoistRedundantSubsetExtractInsert(rewriter, forOp); return DiagnosedSilenceableFailure::success(); } @@ -3204,10 +3176,36 @@ target->walk([&](scf::ForOp forOp) { hoistRedundantSubsetExtractInsert(rewriter, forOp); }); - results.push_back(target); return DiagnosedSilenceableFailure::success(); } +void transform::HoistRedundantTensorSubsetsOp::getEffects( + SmallVectorImpl &effects) { + transform::onlyReadsHandle(getTarget(), effects); + transform::modifiesPayload(effects); +} + +//===----------------------------------------------------------------------===// +// HoistRedundantVectorTransfersOp +//===----------------------------------------------------------------------===// + +DiagnosedSilenceableFailure +transform::HoistRedundantVectorTransfersOp::applyToOne( + func::FuncOp target, transform::ApplyToEachResultList &results, + transform::TransformState &state) { + // WARNING: This hoisting does not model parallelism and is generally + // incorrect when used on distributed loops with memref semantics! + // TODO: obsolete and should be retired. + linalg::hoistRedundantVectorTransfers(target); + return DiagnosedSilenceableFailure::success(); +} + +void transform::HoistRedundantVectorTransfersOp::getEffects( + SmallVectorImpl &effects) { + transform::onlyReadsHandle(getTarget(), effects); + transform::modifiesPayload(effects); +} + //===----------------------------------------------------------------------===// // Transform op registration //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp --- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp +++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp @@ -29,98 +29,90 @@ void transform::LowerVectorsOp::getEffects( SmallVectorImpl &effects) { - consumesHandle(getTarget(), effects); - producesHandle(getResults(), effects); + onlyReadsHandle(getTarget(), effects); modifiesPayload(effects); } -DiagnosedSilenceableFailure transform::LowerVectorsOp::apply( - mlir::transform::TransformResults &transformResults, - mlir::transform::TransformState &state) { - - SmallVector results; - ArrayRef payloadOps = state.getPayloadOps(getTarget()); - for (Operation *target : payloadOps) { - // This check can't be part of the verifier because payload IR is - // independent from transform IR and may not even exist. - if (!target->hasTrait()) { - return mlir::emitDefiniteFailure(target, - "applies only to isolated-from-above " - "targets because it needs to apply " - "patterns greedily"); - } - - MLIRContext *ctx = getContext(); - RewritePatternSet patterns(ctx); - vector::VectorTransposeLowering vectorTransposeLowering = - getTransposeLowering(); - vector::VectorMultiReductionLowering vectorMultiReductionLowering = - getMultireductionLowering(); - vector::VectorContractLowering vectorContractLowering = - getContractionLowering(); - vector::VectorTransferSplit vectorTransferSplit = getSplitTransfers(); - - vector::VectorTransformsOptions vectorTransformOptions; - vectorTransformOptions.setVectorTransformsOptions(vectorContractLowering) - .setVectorMultiReductionLowering(vectorMultiReductionLowering) - .setVectorTransposeLowering(vectorTransposeLowering) - .setVectorTransferSplit(vectorTransferSplit); - - VectorTransferToSCFOptions vectorTransferToSCFOptions = - VectorTransferToSCFOptions().enableFullUnroll( - getUnrollVectorTransfers()); - - int maxTransferRank = 1; - - auto avx2LoweringOptions = - x86vector::avx2::LoweringOptions().setTransposeOptions( - x86vector::avx2::TransposeLoweringOptions() - .lower4x8xf32(getTransposeAvx2Lowering()) - .lower8x8xf32(getTransposeAvx2Lowering())); - - vector::populateVectorToVectorCanonicalizationPatterns(patterns); - - // In the future we may want to more finely select particular stages. - // Stage 1: contraction lowerings. - patterns.add(vectorTransformOptions, - ctx); - vector::populateVectorTransferPermutationMapLoweringPatterns(patterns); - - // Stage 2: multi-reduction lowerings. - vector::populateVectorMultiReductionLoweringPatterns( - patterns, vectorTransformOptions.vectorMultiReductionLowering); - - // Stage 3: Rewrite vector.transfer into full and partial parts. - patterns.add( - ctx, vectorTransformOptions); - - // Stage 4: Lower vector transfers. - vector::populateVectorTransferLoweringPatterns(patterns, maxTransferRank); - - // Stage 5: Vector to scf patterns. - populateVectorToSCFConversionPatterns( - patterns, vectorTransferToSCFOptions.setTargetRank(maxTransferRank)); - - // Stage 6: Lower vector.shape_cast. - vector::populateVectorShapeCastLoweringPatterns(patterns); - - // Stage 7: Lower vector.transpose. - vector::populateVectorTransposeLoweringPatterns(patterns, - vectorTransformOptions); - if (getTransposeAvx2Lowering()) - x86vector::avx2::populateSpecializedTransposeLoweringPatterns( - patterns, avx2LoweringOptions, /*benefit=*/10); - - // Apply everything. - if (failed(applyPatternsAndFoldGreedily(target, std::move(patterns)))) - return DiagnosedSilenceableFailure::definiteFailure(); - - results.push_back(target); +DiagnosedSilenceableFailure transform::LowerVectorsOp::applyToOne( + ::mlir::Operation *target, + ::mlir::transform::ApplyToEachResultList &results, + ::mlir::transform::TransformState &state) { + + // This check can't be part of the verifier because payload IR is + // independent from transform IR and may not even exist. + if (!target->hasTrait()) { + return mlir::emitDefiniteFailure(target, + "applies only to isolated-from-above " + "targets because it needs to apply " + "patterns greedily"); } - transformResults.set(getResults().cast(), results); + MLIRContext *ctx = getContext(); + RewritePatternSet patterns(ctx); + vector::VectorTransposeLowering vectorTransposeLowering = + getTransposeLowering(); + vector::VectorMultiReductionLowering vectorMultiReductionLowering = + getMultireductionLowering(); + vector::VectorContractLowering vectorContractLowering = + getContractionLowering(); + vector::VectorTransferSplit vectorTransferSplit = getSplitTransfers(); + + vector::VectorTransformsOptions vectorTransformOptions; + vectorTransformOptions.setVectorTransformsOptions(vectorContractLowering) + .setVectorMultiReductionLowering(vectorMultiReductionLowering) + .setVectorTransposeLowering(vectorTransposeLowering) + .setVectorTransferSplit(vectorTransferSplit); + + VectorTransferToSCFOptions vectorTransferToSCFOptions = + VectorTransferToSCFOptions().enableFullUnroll(getUnrollVectorTransfers()); + + int maxTransferRank = 1; + + auto avx2LoweringOptions = + x86vector::avx2::LoweringOptions().setTransposeOptions( + x86vector::avx2::TransposeLoweringOptions() + .lower4x8xf32(getTransposeAvx2Lowering()) + .lower8x8xf32(getTransposeAvx2Lowering())); + + vector::populateVectorToVectorCanonicalizationPatterns(patterns); + + // In the future we may want to more finely select particular stages. + // Stage 1: contraction lowerings. + patterns.add(vectorTransformOptions, + ctx); + vector::populateVectorTransferPermutationMapLoweringPatterns(patterns); + + // Stage 2: multi-reduction lowerings. + vector::populateVectorMultiReductionLoweringPatterns( + patterns, vectorTransformOptions.vectorMultiReductionLowering); + + // Stage 3: Rewrite vector.transfer into full and partial parts. + patterns.add( + ctx, vectorTransformOptions); + + // Stage 4: Lower vector transfers. + vector::populateVectorTransferLoweringPatterns(patterns, maxTransferRank); + + // Stage 5: Vector to scf patterns. + populateVectorToSCFConversionPatterns( + patterns, vectorTransferToSCFOptions.setTargetRank(maxTransferRank)); + + // Stage 6: Lower vector.shape_cast. + vector::populateVectorShapeCastLoweringPatterns(patterns); + + // Stage 7: Lower vector.transpose. + vector::populateVectorTransposeLoweringPatterns(patterns, + vectorTransformOptions); + if (getTransposeAvx2Lowering()) + x86vector::avx2::populateSpecializedTransposeLoweringPatterns( + patterns, avx2LoweringOptions, /*benefit=*/10); + + // Apply everything. + if (failed(applyPatternsAndFoldGreedily(target, std::move(patterns)))) + return DiagnosedSilenceableFailure::definiteFailure(); + return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py b/mlir/python/mlir/dialects/_structured_transform_ops_ext.py --- a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py +++ b/mlir/python/mlir/dialects/_structured_transform_ops_ext.py @@ -325,11 +325,9 @@ vectorize_padding: Union[bool, BoolAttr] = False, loc=None, ip=None): - pdl_operation_type = pdl.OperationType.get() if isinstance(vectorize_padding, bool): vectorize_padding = UnitAttr.get() super().__init__( - pdl_operation_type, _get_op_result_or_value(target), vectorize_padding=vectorize_padding, loc=loc, diff --git a/mlir/test/Dialect/GPU/transform-gpu-failing.mlir b/mlir/test/Dialect/GPU/transform-gpu-failing.mlir --- a/mlir/test/Dialect/GPU/transform-gpu-failing.mlir +++ b/mlir/test/Dialect/GPU/transform-gpu-failing.mlir @@ -8,7 +8,8 @@ ^bb0(%arg0: !pdl.operation): %funcop = transform.structured.match ops{["tensor.empty"]} in %arg0 : (!pdl.operation) -> !pdl.operation // expected-error @below {{Given target is not a gpu.launch}} - %1 = transform.gpu.map_nested_forall_to_threads %funcop block_dims = [1, 1, 1] + transform.gpu.map_nested_forall_to_threads %funcop block_dims = [1, 1, 1] + : (!pdl.operation) -> () } // ----- @@ -50,6 +51,7 @@ // expected-error @below {{Trying to launch a GPU kernel with grid_dims = (1, 1, 1) block_dims = (1200, 9, 1). It is larger than the limits.}} // expected-note @below {{"block_dims" is too large}} transform.gpu.map_nested_forall_to_threads %funcop block_dims = [1200, 9, 1] + : (!pdl.operation) -> () } // ----- @@ -91,6 +93,7 @@ %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation // expected-error @below {{Trying to map to fewer GPU threads than loop iterations but overprovisioning is not yet supported. Try additional tiling of the before mapping or map to more threads.}} transform.gpu.map_nested_forall_to_threads %funcop block_dims = [128, 4, 1] + : (!pdl.operation) -> () } // ----- @@ -117,6 +120,7 @@ %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation // expected-error @below {{unsupported dynamic sizes}} transform.gpu.map_nested_forall_to_threads %funcop block_dims = [128, 4, 1] + : (!pdl.operation) -> () } // ----- @@ -139,6 +143,7 @@ %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation // expected-error @below {{only bufferized scf.forall can be mapped}} transform.gpu.map_nested_forall_to_threads %funcop block_dims = [128, 4, 1] + : (!pdl.operation) -> () } // ----- @@ -272,6 +277,7 @@ %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation // expected-error @below {{duplicated attribute, cannot map different loops to the same processor}} transform.gpu.map_nested_forall_to_threads %funcop block_dims = [32, 32, 1] + : (!pdl.operation) -> () } // ----- diff --git a/mlir/test/Dialect/GPU/transform-gpu.mlir b/mlir/test/Dialect/GPU/transform-gpu.mlir --- a/mlir/test/Dialect/GPU/transform-gpu.mlir +++ b/mlir/test/Dialect/GPU/transform-gpu.mlir @@ -88,6 +88,7 @@ ^bb1(%arg0: !pdl.operation): %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation transform.gpu.map_nested_forall_to_threads %funcop block_dims = [12, 9, 1] + : (!pdl.operation) -> () } // ----- @@ -128,6 +129,7 @@ %funcop = transform.structured.match ops{["func.func"]} in %arg0 : (!pdl.operation) -> !pdl.operation %gpuLaunch = transform.gpu.map_forall_to_blocks %funcop { generate_gpu_launch } transform.gpu.map_nested_forall_to_threads %gpuLaunch block_dims = [32, 4, 1] + : (!pdl.operation) -> () } // ----- @@ -161,6 +163,7 @@ ^bb1(%arg0: !pdl.operation): %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation transform.gpu.map_nested_forall_to_threads %funcop block_dims = [12, 9, 1] sync_after_distribute = false + : (!pdl.operation) -> () } // ----- @@ -193,6 +196,7 @@ ^bb1(%arg0: !pdl.operation): %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation transform.gpu.map_nested_forall_to_threads %funcop block_dims = [32, 1, 1] + : (!pdl.operation) -> () } // ----- @@ -229,6 +233,7 @@ ^bb1(%arg0: !pdl.operation): %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation transform.gpu.map_nested_forall_to_threads %funcop block_dims = [12, 9, 1] sync_after_distribute = false + : (!pdl.operation) -> () } // ----- @@ -304,4 +309,5 @@ %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!pdl.operation) -> !pdl.operation transform.gpu.map_nested_forall_to_threads %funcop block_dims = [12, 11, 1] warp_dims = [2, 2, 1] + : (!pdl.operation) -> () } diff --git a/mlir/test/Dialect/LLVM/transform-e2e.mlir b/mlir/test/Dialect/LLVM/transform-e2e.mlir --- a/mlir/test/Dialect/LLVM/transform-e2e.mlir +++ b/mlir/test/Dialect/LLVM/transform-e2e.mlir @@ -17,9 +17,10 @@ %0 = transform.structured.match ops{["linalg.matmul"]} in %module_op : (!pdl.operation) -> !pdl.operation %1, %loops:3 = transform.structured.tile %0 [2, 2, 2] : (!pdl.operation) -> (!pdl.operation, !pdl.operation, !pdl.operation, !pdl.operation) %2 = get_closest_isolated_parent %1 : (!pdl.operation) -> !pdl.operation - transform.structured.vectorize %2 + transform.structured.vectorize %2 : (!pdl.operation) -> () transform.bufferization.one_shot_bufferize layout{IdentityLayoutMap} %module_op {bufferize_function_boundaries = true} %func = transform.structured.match ops{["func.func"]} in %module_op : (!pdl.operation) -> !pdl.operation - transform.vector.lower_vectors %func multireduction_lowering = "innerreduction" + transform.vector.lower_vectors %func multireduction_lowering = "innerreduction" + : (!pdl.operation) -> () } diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir --- a/mlir/test/Dialect/Linalg/hoisting.mlir +++ b/mlir/test/Dialect/Linalg/hoisting.mlir @@ -79,7 +79,10 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_vector_transfers %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () + // Test we can call the op twice without consuming the handle. + transform.structured.hoist_redundant_vector_transfers %0 + : (!pdl.operation) -> () } // ----- @@ -168,7 +171,7 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_vector_transfers %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } // ----- @@ -213,7 +216,7 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_vector_transfers %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } // ----- @@ -302,7 +305,10 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_tensor_subsets %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () + // Test we can call the op twice without consuming the handle. + transform.structured.hoist_redundant_tensor_subsets %0 + : (!pdl.operation) -> () } // ----- @@ -397,7 +403,7 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_tensor_subsets %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } // ----- @@ -514,7 +520,7 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_tensor_subsets %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } // ----- @@ -561,7 +567,7 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_tensor_subsets %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } // ----- @@ -674,5 +680,5 @@ %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!pdl.operation) -> !pdl.operation transform.structured.hoist_redundant_tensor_subsets %0 - : (!pdl.operation) -> !pdl.operation + : (!pdl.operation) -> () } diff --git a/mlir/test/Dialect/Linalg/transform-op-vectorize.mlir b/mlir/test/Dialect/Linalg/transform-op-vectorize.mlir --- a/mlir/test/Dialect/Linalg/transform-op-vectorize.mlir +++ b/mlir/test/Dialect/Linalg/transform-op-vectorize.mlir @@ -20,7 +20,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -66,7 +66,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -114,7 +114,9 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 {vectorize_padding} + transform.structured.vectorize %1 {vectorize_padding} : (!pdl.operation) -> () + // Apply transform twice to ensure %1 is not consumed. + transform.structured.vectorize %1 {vectorize_padding} : (!pdl.operation) -> () } // ----- @@ -131,5 +133,5 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation // expected-error @below {{op requires isolated-from-above targets}} - %2 = transform.structured.vectorize %0 + transform.structured.vectorize %0 : (!pdl.operation) -> () } diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir --- a/mlir/test/Dialect/Linalg/vectorization.mlir +++ b/mlir/test/Dialect/Linalg/vectorization.mlir @@ -14,7 +14,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.dot"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + : (!pdl.operation) -> () } // ----- @@ -33,7 +34,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + : (!pdl.operation) -> () } // ----- @@ -51,7 +53,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + : (!pdl.operation) -> () } // ----- @@ -70,7 +73,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.batch_matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + : (!pdl.operation) -> () } // ----- @@ -110,7 +114,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -150,7 +155,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -177,7 +183,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -217,7 +224,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -237,7 +245,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns } + : (!pdl.operation) -> () } // ----- @@ -261,7 +270,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -285,7 +294,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -330,7 +339,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } + : (!pdl.operation) -> () } // ----- @@ -347,7 +357,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -365,7 +375,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -382,7 +392,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -402,7 +412,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -418,7 +428,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -446,7 +456,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -475,7 +485,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -560,7 +570,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -651,7 +662,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -695,7 +707,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -738,7 +751,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -770,7 +784,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -799,7 +814,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_padding } + transform.structured.vectorize %1 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -828,7 +843,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_padding } + transform.structured.vectorize %1 { vectorize_padding } : (!pdl.operation) -> () } @@ -865,7 +880,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_padding } + transform.structured.vectorize %1 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -885,7 +900,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_padding } + transform.structured.vectorize %1 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -915,7 +930,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_padding } + transform.structured.vectorize %1 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -948,7 +963,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { vectorize_padding } + transform.structured.vectorize %4 { vectorize_padding } : (!pdl.operation) -> () } @@ -985,7 +1000,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { vectorize_padding } + transform.structured.vectorize %4 { vectorize_padding } : (!pdl.operation) -> () } @@ -1019,7 +1034,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { vectorize_padding } + transform.structured.vectorize %4 { vectorize_padding } : (!pdl.operation) -> () } @@ -1047,7 +1062,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1084,7 +1099,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { vectorize_padding } + transform.structured.vectorize %4 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -1119,7 +1134,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1164,7 +1179,8 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %4 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -1194,7 +1210,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 { vectorize_padding } + transform.structured.vectorize %4 { vectorize_padding } : (!pdl.operation) -> () } // ----- @@ -1225,7 +1241,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1255,7 +1271,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1285,7 +1301,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1315,7 +1331,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1345,7 +1361,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1379,7 +1395,7 @@ ^bb1(%arg1: !pdl.operation): %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1417,11 +1433,11 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %4 = get_closest_isolated_parent %3 : (!pdl.operation) -> !pdl.operation - %5 = transform.structured.vectorize %4 + transform.structured.vectorize %4 : (!pdl.operation) -> () } // ----- @@ -1464,7 +1480,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } @@ -1495,7 +1511,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -1534,7 +1550,8 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + transform.structured.vectorize %1 { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } + : (!pdl.operation) -> () } // ----- @@ -1570,7 +1587,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -1606,7 +1623,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1645,7 +1662,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1695,7 +1712,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1743,7 +1760,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1787,7 +1804,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1829,7 +1846,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1873,7 +1890,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1913,7 +1930,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1953,7 +1970,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } // ----- @@ -1992,7 +2009,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 { vectorize_nd_extract } + transform.structured.vectorize %1 { vectorize_nd_extract } : (!pdl.operation) -> () } @@ -2017,7 +2034,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.map"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -2036,7 +2053,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.transpose"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -2059,7 +2076,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- @@ -2310,7 +2327,7 @@ transform.sequence failures(propagate) { ^bb0(%arg0: !pdl.operation): %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!pdl.operation) -> !pdl.operation - %1 = transform.structured.vectorize %0 + transform.structured.vectorize %0 : (!pdl.operation) -> () } // ----- @@ -2345,7 +2362,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // CHECK-LABEL: @wrong_reduction_detection @@ -2374,7 +2391,7 @@ ^bb1(%arg1: !pdl.operation): %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - %2 = transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } // ----- diff --git a/mlir/test/Dialect/Transform/selective-targeting.mlir b/mlir/test/Dialect/Transform/selective-targeting.mlir --- a/mlir/test/Dialect/Transform/selective-targeting.mlir +++ b/mlir/test/Dialect/Transform/selective-targeting.mlir @@ -80,7 +80,7 @@ transform.structured.tile %0 [4, 4, 4] : (!pdl.operation) -> (!pdl.operation, !pdl.operation, !pdl.operation, !pdl.operation) %1 = pdl_match @pdl_target_attrC in %arg1 : (!pdl.operation) -> !pdl.operation %2 = transform.get_closest_isolated_parent %1 : (!pdl.operation) -> !pdl.operation - transform.structured.vectorize %2 + transform.structured.vectorize %2 : (!pdl.operation) -> () } } @@ -125,7 +125,7 @@ ^bb1(%arg1: !pdl.operation): %0 = pdl_match @pdl_target in %arg1 : (!pdl.operation) -> !pdl.operation %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation - transform.structured.vectorize %1 + transform.structured.vectorize %1 : (!pdl.operation) -> () } } @@ -150,5 +150,5 @@ transform.sequence failures(propagate) { ^bb0(%arg0: !pdl.operation): - transform.structured.vectorize %arg0 + transform.structured.vectorize %arg0 : (!pdl.operation) -> () } diff --git a/mlir/test/Dialect/Vector/transform-vector.mlir b/mlir/test/Dialect/Vector/transform-vector.mlir --- a/mlir/test/Dialect/Vector/transform-vector.mlir +++ b/mlir/test/Dialect/Vector/transform-vector.mlir @@ -18,9 +18,10 @@ %0 = transform.structured.match ops{["linalg.matmul"]} in %module_op : (!pdl.operation) -> !pdl.operation %1, %loops:3 = transform.structured.tile %0 [8, 4, 2] : (!pdl.operation) -> (!pdl.operation, !pdl.operation, !pdl.operation, !pdl.operation) %2 = get_closest_isolated_parent %1 : (!pdl.operation) -> !pdl.operation - transform.structured.vectorize %2 + transform.structured.vectorize %2 : (!pdl.operation) -> () transform.bufferization.one_shot_bufferize %module_op %func = transform.structured.match ops{["func.func"]} in %module_op : (!pdl.operation) -> !pdl.operation transform.vector.lower_vectors %func multireduction_lowering = "innerreduction" + : (!pdl.operation) -> () } diff --git a/mlir/test/python/dialects/transform_structured_ext.py b/mlir/test/python/dialects/transform_structured_ext.py --- a/mlir/test/python/dialects/transform_structured_ext.py +++ b/mlir/test/python/dialects/transform_structured_ext.py @@ -206,5 +206,5 @@ transform.YieldOp() # CHECK-LABEL: TEST: testVectorize # CHECK: transform.sequence - # CHECK: = transform.structured.vectorize + # CHECK: transform.structured.vectorize # CHECK: {vectorize_padding}