diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h --- a/mlir/include/mlir/Dialect/Vector/VectorOps.h +++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h @@ -13,6 +13,7 @@ #ifndef MLIR_DIALECT_VECTOR_VECTOROPS_H #define MLIR_DIALECT_VECTOR_VECTOROPS_H +#include "mlir/IR/AffineMap.h" #include "mlir/IR/Attributes.h" #include "mlir/IR/Dialect.h" #include "mlir/IR/OpDefinition.h" @@ -71,6 +72,14 @@ /// the integer type required for subscripts in the vector dialect. ArrayAttr getVectorSubscriptAttr(Builder &b, ArrayRef values); +namespace impl { +/// Build the default minor identity map suitable for a vector transfer. This +/// also handles the case memref<... x vector<...>> -> vector<...> in which the +/// rank of the identity map must take the vector element type into account. +AffineMap getTransferMinorIdentityMap(MemRefType memRefType, + VectorType vectorType); +} // namespace impl + #define GET_OP_CLASSES #include "mlir/Dialect/Vector/VectorOps.h.inc" diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td --- a/mlir/include/mlir/Dialect/Vector/VectorOps.td +++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td @@ -863,6 +863,18 @@ let assemblyFormat = "$vector attr-dict `:` type($vector) `to` type(results)"; } +def Vector_TransferOpUtils { + code extraTransferDeclaration = [{ + static StringRef getPermutationMapAttrName() { return "permutation_map"; } + MemRefType getMemRefType() { + return memref().getType().cast(); + } + VectorType getVectorType() { + return vector().getType().cast(); + } + }]; +} + def Vector_TransferReadOp : Vector_Op<"transfer_read">, Arguments<(ins AnyMemRef:$memref, Variadic:$indices, @@ -884,15 +896,21 @@ supplied as the operands `2 .. 1 + rank(memref)`. The permutation_map [attribute](../LangRef.md#attributes) is an [affine-map](Affine.md#affine-maps) which specifies the transposition on the - slice to match the vector shape. The size of the slice is specified by the - size of the vector, given as the return type. An `ssa-value` of the same - elemental type as the MemRef is provided as the last operand to specify - padding in the case of out-of-bounds accesses. This operation is called - 'read' by opposition to 'load' because the super-vector granularity is - generally not representable with a single hardware register. - A `vector.transfer_read` is thus a mid-level - abstraction that supports super-vectorization with non-effecting padding for - full-tile-only code. + slice to match the vector shape. The permutation map may be implicit and + ommitted from parsing and printing if it is the canonical minor identity map + (i.e. if it does not permute or broadcast any dimension). + + The size of the slice is specified by the size of the vector, given as the + return type. + + An `ssa-value` of the same elemental type as the MemRef is provided as the + last operand to specify padding in the case of out-of-bounds accesses. + + This operation is called 'read' by opposition to 'load' because the + super-vector granularity is generally not representable with a single + hardware register. A `vector.transfer_read` is thus a mid-level abstraction + that supports super-vectorization with non-effecting padding for full-tile + only operations. More precisely, let's dive deeper into the permutation_map for the following MLIR: @@ -995,19 +1013,25 @@ }]; let builders = [ - // Builder that sets permutation map and padding to 'getMinorIdentityMap' - // and zero, respectively, by default. + // Builder that sets padding to zero. + OpBuilder<"OpBuilder &builder, OperationState &result, VectorType vector, " + "Value memref, ValueRange indices, AffineMap permutationMap">, + // Builder that sets permutation map (resp. padding) to + // 'getMinorIdentityMap' (resp. zero). OpBuilder<"OpBuilder &builder, OperationState &result, VectorType vector, " "Value memref, ValueRange indices"> ]; - let extraClassDeclaration = [{ - MemRefType getMemRefType() { - return memref().getType().cast(); - } - VectorType getVectorType() { - return vector().getType().cast(); - } + let extraClassDeclaration = Vector_TransferOpUtils.extraTransferDeclaration # + [{ + /// Build the default minor identity map suitable for a vector transfer. + /// This also handles the case memref<... x vector<...>> -> vector<...> in + /// which the rank of the identity map must take the vector element type + /// into account. + static AffineMap getTransferMinorIdentityMap( + MemRefType memRefType, VectorType vectorType) { + return impl::getTransferMinorIdentityMap(memRefType, vectorType); + } }]; } @@ -1033,10 +1057,15 @@ supplied as the operands `3 .. 2 + rank(memref)`. The permutation_map [attribute](../LangRef.md#attributes) is an [affine-map](Affine.md#affine-maps) which specifies the transposition on the - slice to match the vector shape. The size of the slice is specified by the - size of the vector. This operation is called 'write' by opposition to - 'store' because the super-vector granularity is generally not representable - with a single hardware register. A `vector.transfer_write` is thus a + slice to match the vector shape. The permutation map may be implicit and + ommitted from parsing and printing if it is the canonical minor identity map + (i.e. if it does not permute or broadcast any dimension). + + The size of the slice is specified by the size of the vector. + + This operation is called 'write' by opposition to 'store' because the + super-vector granularity is generally not representable with a single + hardware register. A `vector.transfer_write` is thus a mid-level abstraction that supports super-vectorization with non-effecting padding for full-tile-only code. It is the responsibility of `vector.transfer_write`'s implementation to ensure the memory writes are @@ -1066,23 +1095,21 @@ }]; let builders = [ - // Builder that sets permutation map and padding to 'getMinorIdentityMap' - // by default. + // Builder that sets permutation map to 'getMinorIdentityMap'. OpBuilder<"OpBuilder &builder, OperationState &result, Value vector, " "Value memref, ValueRange indices"> ]; - let extraClassDeclaration = [{ - VectorType getVectorType() { - return vector().getType().cast(); - } - MemRefType getMemRefType() { - return memref().getType().cast(); - } - }]; - let assemblyFormat = [{ - $vector `,` $memref `[` $indices `]` attr-dict `:` type($vector) `,` - type($memref) + let extraClassDeclaration = Vector_TransferOpUtils.extraTransferDeclaration # + [{ + /// Build the default minor identity map suitable for a vector transfer. + /// This also handles the case memref<... x vector<...>> -> vector<...> in + /// which the rank of the identity map must take the vector element type + /// into account. + static AffineMap getTransferMinorIdentityMap( + MemRefType memRefType, VectorType vectorType) { + return impl::getTransferMinorIdentityMap(memRefType, vectorType); + } }]; } diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp --- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp +++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp @@ -187,15 +187,15 @@ MemRefBoundsCapture &memrefBounds) { // If in-bounds, index into memref and lower to 1-D transfer read. auto thenBlockBuilder = [&](ValueRange majorIvsPlusOffsets) { - auto map = AffineMap::getMinorIdentityMap( - xferOp.getMemRefType().getRank(), minorRank, xferOp.getContext()); - // Lower to 1-D vector_transfer_read and let recursion handle it. - Value memref = xferOp.memref(); SmallVector indexing; indexing.reserve(leadingRank + majorRank + minorRank); indexing.append(leadingOffsets.begin(), leadingOffsets.end()); indexing.append(majorIvsPlusOffsets.begin(), majorIvsPlusOffsets.end()); indexing.append(minorOffsets.begin(), minorOffsets.end()); + // Lower to 1-D vector_transfer_read and let recursion handle it. + Value memref = xferOp.memref(); + auto map = TransferReadOp::getTransferMinorIdentityMap( + xferOp.getMemRefType(), minorVectorType); auto loaded1D = vector_transfer_read(minorVectorType, memref, indexing, AffineMapAttr::get(map), xferOp.padding()); @@ -230,14 +230,15 @@ MemRefBoundsCapture &memrefBounds) { auto thenBlockBuilder = [&](ValueRange majorIvsPlusOffsets) { // Lower to 1-D vector_transfer_write and let recursion handle it. - Value loaded1D = std_load(alloc, majorIvs); - auto map = AffineMap::getMinorIdentityMap( - xferOp.getMemRefType().getRank(), minorRank, xferOp.getContext()); SmallVector indexing; indexing.reserve(leadingRank + majorRank + minorRank); indexing.append(leadingOffsets.begin(), leadingOffsets.end()); indexing.append(majorIvsPlusOffsets.begin(), majorIvsPlusOffsets.end()); indexing.append(minorOffsets.begin(), minorOffsets.end()); + // Lower to 1-D vector_transfer_write and let recursion handle it. + Value loaded1D = std_load(alloc, majorIvs); + auto map = TransferWriteOp::getTransferMinorIdentityMap( + xferOp.getMemRefType(), minorVectorType); vector_transfer_write(loaded1D, xferOp.memref(), indexing, AffineMapAttr::get(map)); }; diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp --- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp +++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp @@ -793,10 +793,7 @@ LLVM_DEBUG(permutationMap.print(dbgs())); auto transfer = b.create( opInst->getLoc(), vectorType, memoryOp.getMemRef(), indices, - AffineMapAttr::get(permutationMap), - // TODO(b/144455320) add a proper padding value, not just 0.0 : f32 - state->folder->create(b, opInst->getLoc(), - APFloat(0.0f), b.getF32Type())); + permutationMap); state->registerReplacement(opInst, transfer.getOperation()); } else { state->registerTerminal(opInst); diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp --- a/mlir/lib/Dialect/Vector/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/VectorOps.cpp @@ -1281,28 +1281,60 @@ if (permutationMap.getNumInputs() != memrefType.getRank()) return op->emitOpError("requires a permutation_map with input dims of the " "same rank as the memref type"); + return success(); } +/// Build the default minor identity map suitable for a vector transfer. This +/// also handles the case memref<... x vector<...>> -> vector<...> in which the +/// rank of the identity map must take the vector element type into account. +AffineMap +mlir::vector::impl::getTransferMinorIdentityMap(MemRefType memRefType, + VectorType vectorType) { + int64_t elementVectorRank = 0; + VectorType elementVectorType = + memRefType.getElementType().dyn_cast(); + if (elementVectorType) + elementVectorRank += elementVectorType.getRank(); + return AffineMap::getMinorIdentityMap( + memRefType.getRank(), vectorType.getRank() - elementVectorRank, + memRefType.getContext()); +} + /// Builder that sets permutation map and padding to 'getMinorIdentityMap' and /// zero, respectively, by default. void TransferReadOp::build(OpBuilder &builder, OperationState &result, - VectorType vector, Value memref, - ValueRange indices) { - auto permMap = AffineMap::getMinorIdentityMap( - memref.getType().cast().getRank(), vector.getRank(), - builder.getContext()); + VectorType vector, Value memref, ValueRange indices, + AffineMap permutationMap) { Type elemType = vector.cast().getElementType(); Value padding = builder.create(result.location, elemType, builder.getZeroAttr(elemType)); + build(builder, result, vector, memref, indices, permutationMap, padding); +} - build(builder, result, vector, memref, indices, permMap, padding); +/// Builder that sets permutation map (resp. padding) to 'getMinorIdentityMap' +/// (resp. zero). +void TransferReadOp::build(OpBuilder &builder, OperationState &result, + VectorType vectorType, Value memref, + ValueRange indices) { + build(builder, result, vectorType, memref, indices, + getTransferMinorIdentityMap(memref.getType().cast(), + vectorType)); +} + +template +void printTransferAttrs(OpAsmPrinter &p, TransferOp op) { + SmallVector elidedAttrs; + if (op.permutation_map() == TransferOp::getTransferMinorIdentityMap( + op.getMemRefType(), op.getVectorType())) + elidedAttrs.push_back(op.getPermutationMapAttrName()); + p.printOptionalAttrDict(op.getAttrs(), elidedAttrs); } static void print(OpAsmPrinter &p, TransferReadOp op) { p << op.getOperationName() << " " << op.memref() << "[" << op.indices() - << "], " << op.padding() << " "; - p.printOptionalAttrDict(op.getAttrs()); + << "], " << op.padding(); + printTransferAttrs(p, op); p << " : " << op.getMemRefType() << ", " << op.getVectorType(); } @@ -1313,7 +1345,7 @@ SmallVector indexInfo; OpAsmParser::OperandType paddingInfo; SmallVector types; - // Parsing with support for optional paddingValue. + // Parsing with support for paddingValue. if (parser.parseOperand(memrefInfo) || parser.parseOperandList(indexInfo, OpAsmParser::Delimiter::Square) || parser.parseComma() || parser.parseOperand(paddingInfo) || @@ -1321,12 +1353,21 @@ parser.getCurrentLocation(&typesLoc) || parser.parseColonTypeList(types)) return failure(); if (types.size() != 2) - return parser.emitError(typesLoc, "two types required"); + return parser.emitError(typesLoc, "requires two types"); auto indexType = parser.getBuilder().getIndexType(); MemRefType memRefType = types[0].dyn_cast(); if (!memRefType) - return parser.emitError(typesLoc, "memref type required"), failure(); - Type vectorType = types[1]; + return parser.emitError(typesLoc, "requires memref type"); + VectorType vectorType = types[1].dyn_cast(); + if (!vectorType) + return parser.emitError(typesLoc, "requires vector type"); + auto permutationAttrName = TransferReadOp::getPermutationMapAttrName(); + auto attr = result.attributes.get(permutationAttrName); + if (!attr) { + auto permMap = + TransferReadOp::getTransferMinorIdentityMap(memRefType, vectorType); + result.attributes.set(permutationAttrName, AffineMapAttr::get(permMap)); + } return failure( parser.resolveOperand(memrefInfo, memRefType, result.operands) || parser.resolveOperands(indexInfo, indexType, result.operands) || @@ -1376,17 +1417,56 @@ // TransferWriteOp //===----------------------------------------------------------------------===// -/// Builder that sets permutation map and padding to 'getMinorIdentityMap' by -/// default. +/// Builder that sets permutation map to 'getMinorIdentityMap'. void TransferWriteOp::build(OpBuilder &builder, OperationState &result, Value vector, Value memref, ValueRange indices) { auto vectorType = vector.getType().cast(); - auto permMap = AffineMap::getMinorIdentityMap( - memref.getType().cast().getRank(), vectorType.getRank(), - builder.getContext()); + auto permMap = getTransferMinorIdentityMap( + memref.getType().cast(), vectorType); build(builder, result, vector, memref, indices, permMap); } +static ParseResult parseTransferWriteOp(OpAsmParser &parser, + OperationState &result) { + llvm::SMLoc typesLoc; + OpAsmParser::OperandType vectorInfo, memrefInfo; + SmallVector indexInfo; + SmallVector types; + if (parser.parseOperand(vectorInfo) || parser.parseComma() || + parser.parseOperand(memrefInfo) || + parser.parseOperandList(indexInfo, OpAsmParser::Delimiter::Square) || + parser.parseOptionalAttrDict(result.attributes) || + parser.getCurrentLocation(&typesLoc) || parser.parseColonTypeList(types)) + return failure(); + if (types.size() != 2) + return parser.emitError(typesLoc, "requires two types"); + auto indexType = parser.getBuilder().getIndexType(); + VectorType vectorType = types[0].dyn_cast(); + if (!vectorType) + return parser.emitError(typesLoc, "requires vector type"); + MemRefType memRefType = types[1].dyn_cast(); + if (!memRefType) + return parser.emitError(typesLoc, "requires memref type"); + auto permutationAttrName = TransferWriteOp::getPermutationMapAttrName(); + auto attr = result.attributes.get(permutationAttrName); + if (!attr) { + auto permMap = + TransferWriteOp::getTransferMinorIdentityMap(memRefType, vectorType); + result.attributes.set(permutationAttrName, AffineMapAttr::get(permMap)); + } + return failure( + parser.resolveOperand(vectorInfo, vectorType, result.operands) || + parser.resolveOperand(memrefInfo, memRefType, result.operands) || + parser.resolveOperands(indexInfo, indexType, result.operands)); +} + +static void print(OpAsmPrinter &p, TransferWriteOp op) { + p << op.getOperationName() << " " << op.vector() << ", " << op.memref() << "[" + << op.indices() << "]"; + printTransferAttrs(p, op); + p << " : " << op.getVectorType() << ", " << op.getMemRefType(); +} + static LogicalResult verify(TransferWriteOp op) { // Consistency of elemental types in memref and vector. MemRefType memrefType = op.getMemRefType(); diff --git a/mlir/test/Conversion/AffineToStandard/lower-affine-to-vector.mlir b/mlir/test/Conversion/AffineToStandard/lower-affine-to-vector.mlir --- a/mlir/test/Conversion/AffineToStandard/lower-affine-to-vector.mlir +++ b/mlir/test/Conversion/AffineToStandard/lower-affine-to-vector.mlir @@ -1,6 +1,5 @@ // RUN: mlir-opt -lower-affine --split-input-file %s | FileCheck %s -// CHECK: #[[perm_map:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func @affine_vector_load func @affine_vector_load(%arg0 : index) { %0 = alloc() : memref<100xf32> @@ -12,13 +11,12 @@ // CHECK-NEXT: %[[c7:.*]] = constant 7 : index // CHECK-NEXT: %[[b:.*]] = addi %[[a]], %[[c7]] : index // CHECK-NEXT: %[[pad:.*]] = constant 0.0 -// CHECK-NEXT: vector.transfer_read %[[buf]][%[[b]]], %[[pad]] {permutation_map = #[[perm_map]]} : memref<100xf32>, vector<8xf32> +// CHECK-NEXT: vector.transfer_read %[[buf]][%[[b]]], %[[pad]] : memref<100xf32>, vector<8xf32> return } // ----- -// CHECK: #[[perm_map:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func @affine_vector_store func @affine_vector_store(%arg0 : index) { %0 = alloc() : memref<100xf32> @@ -33,13 +31,12 @@ // CHECK-NEXT: %[[b:.*]] = addi %{{.*}}, %[[a]] : index // CHECK-NEXT: %[[c7:.*]] = constant 7 : index // CHECK-NEXT: %[[c:.*]] = addi %[[b]], %[[c7]] : index -// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[c]]] {permutation_map = #[[perm_map]]} : vector<4xf32>, memref<100xf32> +// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[c]]] : vector<4xf32>, memref<100xf32> return } // ----- -// CHECK: #[[perm_map:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func @affine_vector_load func @affine_vector_load(%arg0 : index) { %0 = alloc() : memref<100xf32> @@ -51,13 +48,12 @@ // CHECK-NEXT: %[[c7:.*]] = constant 7 : index // CHECK-NEXT: %[[b:.*]] = addi %[[a]], %[[c7]] : index // CHECK-NEXT: %[[pad:.*]] = constant 0.0 -// CHECK-NEXT: vector.transfer_read %[[buf]][%[[b]]], %[[pad]] {permutation_map = #[[perm_map]]} : memref<100xf32>, vector<8xf32> +// CHECK-NEXT: vector.transfer_read %[[buf]][%[[b]]], %[[pad]] : memref<100xf32>, vector<8xf32> return } // ----- -// CHECK: #[[perm_map:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func @affine_vector_store func @affine_vector_store(%arg0 : index) { %0 = alloc() : memref<100xf32> @@ -72,13 +68,12 @@ // CHECK-NEXT: %[[b:.*]] = addi %{{.*}}, %[[a]] : index // CHECK-NEXT: %[[c7:.*]] = constant 7 : index // CHECK-NEXT: %[[c:.*]] = addi %[[b]], %[[c7]] : index -// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[c]]] {permutation_map = #[[perm_map]]} : vector<4xf32>, memref<100xf32> +// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[c]]] : vector<4xf32>, memref<100xf32> return } // ----- -// CHECK: #[[perm_map:.*]] = affine_map<(d0, d1) -> (d0, d1)> // CHECK-LABEL: func @vector_load_2d func @vector_load_2d() { %0 = alloc() : memref<100x100xf32> @@ -89,7 +84,7 @@ // CHECK: scf.for %[[i0:.*]] = // CHECK: scf.for %[[i1:.*]] = // CHECK-NEXT: %[[pad:.*]] = constant 0.0 -// CHECK-NEXT: vector.transfer_read %[[buf]][%[[i0]], %[[i1]]], %[[pad]] {permutation_map = #[[perm_map]]} : memref<100x100xf32>, vector<2x8xf32> +// CHECK-NEXT: vector.transfer_read %[[buf]][%[[i0]], %[[i1]]], %[[pad]] : memref<100x100xf32>, vector<2x8xf32> } } return @@ -97,7 +92,6 @@ // ----- -// CHECK: #[[perm_map:.*]] = affine_map<(d0, d1) -> (d0, d1)> // CHECK-LABEL: func @vector_store_2d func @vector_store_2d() { %0 = alloc() : memref<100x100xf32> @@ -109,7 +103,7 @@ // CHECK: %[[val:.*]] = constant dense // CHECK: scf.for %[[i0:.*]] = // CHECK: scf.for %[[i1:.*]] = -// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[i0]], %[[i1]]] {permutation_map = #[[perm_map]]} : vector<2x8xf32>, memref<100x100xf32> +// CHECK-NEXT: vector.transfer_write %[[val]], %[[buf]][%[[i0]], %[[i1]]] : vector<2x8xf32>, memref<100x100xf32> } } return diff --git a/mlir/test/Conversion/VectorToLoops/vector-to-loops.mlir b/mlir/test/Conversion/VectorToLoops/vector-to-loops.mlir --- a/mlir/test/Conversion/VectorToLoops/vector-to-loops.mlir +++ b/mlir/test/Conversion/VectorToLoops/vector-to-loops.mlir @@ -229,7 +229,7 @@ // CHECK: %[[cmp:.*]] = cmpi "slt", %[[add]], %[[dim]] : index // CHECK: %[[cond1:.*]] = and %[[cmp]], %[[cond0]] : i1 // CHECK: scf.if %[[cond1]] { - // CHECK: %[[vec_1d:.*]] = vector.transfer_read %[[A]][%[[add]], %[[base]]], %[[cst]] {permutation_map = #[[MAP1]]} : memref, vector<15xf32> + // CHECK: %[[vec_1d:.*]] = vector.transfer_read %[[A]][%[[add]], %[[base]]], %[[cst]] : memref, vector<15xf32> // CHECK: store %[[vec_1d]], %[[alloc]][%[[I]]] : memref<17xvector<15xf32>> // CHECK: } else { // CHECK: store %[[splat]], %[[alloc]][%[[I]]] : memref<17xvector<15xf32>> @@ -264,7 +264,7 @@ // CHECK: %[[cond1:.*]] = and %[[cmp]], %[[cond0]] : i1 // CHECK: scf.if %[[cond1]] { // CHECK: %[[vec_1d:.*]] = load %0[%[[I]]] : memref<17xvector<15xf32>> - // CHECK: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] {permutation_map = #[[MAP1]]} : vector<15xf32>, memref + // CHECK: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] : vector<15xf32>, memref // CHECK: } vector.transfer_write %vec, %A[%base, %base] {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} : diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir --- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir @@ -2,7 +2,6 @@ // Permutation maps used in vectorization. // CHECK: #[[map_proj_d0d1_0:map[0-9]+]] = affine_map<(d0, d1) -> (0)> -// CHECK: #[[map_proj_d0d1_d1:map[0-9]+]] = affine_map<(d0, d1) -> (d1)> #map0 = affine_map<(d0) -> (d0)> #mapadd1 = affine_map<(d0) -> (d0 + 1)> @@ -13,7 +12,6 @@ // Maps introduced to vectorize fastest varying memory index. // CHECK-LABEL: func @vec1d_1 func @vec1d_1(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: %[[C0:[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, 1 : memref @@ -22,10 +20,11 @@ %N = dim %A, 1 : memref %P = dim %B, 2 : memref %cst0 = constant 0 : index -// + // CHECK: for {{.*}} step 128 // CHECK-NEXT: %{{.*}} = affine.apply #map0(%[[C0]]) // CHECK-NEXT: %{{.*}} = affine.apply #map0(%[[C0]]) +// CHECK-NEXT: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_0]]} : memref, vector<128xf32> affine.for %i0 = 0 to %M { // vectorized due to scalar -> vector %a0 = affine.load %A[%cst0, %cst0] : memref @@ -35,7 +34,6 @@ // CHECK-LABEL: func @vec1d_2 func @vec1d_2(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: %[[C0:[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, 1 : memref @@ -46,7 +44,8 @@ %cst0 = constant 0 : index // // CHECK:for [[IV3:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128 -// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : memref, vector<128xf32> +// CHECK-NEXT: %[[CST:.*]] = constant 0.0{{.*}}: f32 +// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %[[CST]] : memref, vector<128xf32> affine.for %i3 = 0 to %M { // vectorized %a3 = affine.load %A[%cst0, %i3] : memref } @@ -55,7 +54,6 @@ // CHECK-LABEL: func @vec1d_3 func @vec1d_3(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: %[[C0:[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %arg0, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %arg0, 1 : memref @@ -69,7 +67,8 @@ // CHECK-NEXT: for [[IV9:%[arg0-9]*]] = 0 to [[ARG_N]] { // CHECK-NEXT: %[[APP9_0:[0-9]+]] = affine.apply {{.*}}([[IV9]], [[IV8]]) // CHECK-NEXT: %[[APP9_1:[0-9]+]] = affine.apply {{.*}}([[IV9]], [[IV8]]) -// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%[[APP9_0]], %[[APP9_1]]], %{{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : memref, vector<128xf32> +// CHECK-NEXT: %[[CST:.*]] = constant 0.0{{.*}}: f32 +// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%[[APP9_0]], %[[APP9_1]]], %[[CST]] : memref, vector<128xf32> affine.for %i8 = 0 to %M { // vectorized affine.for %i9 = 0 to %N { %a9 = affine.load %A[%i9, %i8 + %i9] : memref @@ -87,31 +86,31 @@ %f2 = constant 2.0 : f32 affine.for %i0 = 0 to %M { affine.for %i1 = 0 to %N { - // CHECK: [[C1:%.*]] = constant dense<1.000000e+00> : vector<128xf32> - // CHECK: vector.transfer_write [[C1]], {{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : vector<128xf32>, memref + // CHECK: %[[C1:.*]] = constant dense<1.000000e+00> : vector<128xf32> + // CHECK: vector.transfer_write %[[C1]], {{.*}} : vector<128xf32>, memref // non-scoped %f1 affine.store %f1, %A[%i0, %i1] : memref } } affine.for %i2 = 0 to %M { affine.for %i3 = 0 to %N { - // CHECK: [[C3:%.*]] = constant dense<2.000000e+00> : vector<128xf32> - // CHECK: vector.transfer_write [[C3]], {{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : vector<128xf32>, memref + // CHECK: %[[C3:.*]] = constant dense<2.000000e+00> : vector<128xf32> + // CHECK: vector.transfer_write %[[C3]], {{.*}} : vector<128xf32>, memref // non-scoped %f2 affine.store %f2, %B[%i2, %i3] : memref } } affine.for %i4 = 0 to %M { affine.for %i5 = 0 to %N { - // CHECK: [[A5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : memref, vector<128xf32> - // CHECK: [[B5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : memref, vector<128xf32> - // CHECK: [[S5:%.*]] = addf [[A5]], [[B5]] : vector<128xf32> - // CHECK: [[SPLAT1:%.*]] = constant dense<1.000000e+00> : vector<128xf32> - // CHECK: [[S6:%.*]] = addf [[S5]], [[SPLAT1]] : vector<128xf32> - // CHECK: [[SPLAT2:%.*]] = constant dense<2.000000e+00> : vector<128xf32> - // CHECK: [[S7:%.*]] = addf [[S5]], [[SPLAT2]] : vector<128xf32> - // CHECK: [[S8:%.*]] = addf [[S7]], [[S6]] : vector<128xf32> - // CHECK: vector.transfer_write [[S8]], {{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : vector<128xf32>, memref + // CHECK: %[[A5:.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{[a-zA-Z0-9_]*}} : memref, vector<128xf32> + // CHECK: %[[B5:.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{[a-zA-Z0-9_]*}} : memref, vector<128xf32> + // CHECK: %[[S5:.*]] = addf %[[A5]], %[[B5]] : vector<128xf32> + // CHECK: %[[SPLAT1:.*]] = constant dense<1.000000e+00> : vector<128xf32> + // CHECK: %[[S6:.*]] = addf %[[S5]], %[[SPLAT1]] : vector<128xf32> + // CHECK: %[[SPLAT2:.*]] = constant dense<2.000000e+00> : vector<128xf32> + // CHECK: %[[S7:.*]] = addf %[[S5]], %[[SPLAT2]] : vector<128xf32> + // CHECK: %[[S8:.*]] = addf %[[S7]], %[[S6]] : vector<128xf32> + // CHECK: vector.transfer_write %[[S8]], {{.*}} : vector<128xf32>, memref %a5 = affine.load %A[%i4, %i5] : memref %b5 = affine.load %B[%i4, %i5] : memref %s5 = addf %a5, %b5 : f32 @@ -168,7 +167,6 @@ // CHECK-LABEL: func @vec_rejected_3 func @vec_rejected_3(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: [[C0:%[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, 1 : memref @@ -180,7 +178,8 @@ // // CHECK:for [[IV4:%[arg0-9]+]] = 0 to [[ARG_M]] step 128 { // CHECK-NEXT: for [[IV5:%[arg0-9]*]] = 0 to [[ARG_N]] { -// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_d1]]} : memref, vector<128xf32> +// CHECK-NEXT: %{{.*}} = constant 0.0{{.*}}: f32 +// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{[a-zA-Z0-9_]*}} : memref, vector<128xf32> affine.for %i4 = 0 to %M { // vectorized affine.for %i5 = 0 to %N { // not vectorized, would vectorize with --test-fastest-varying=1 %a5 = affine.load %A[%i5, %i4] : memref @@ -277,7 +276,6 @@ // CHECK-LABEL: func @vec_rejected_8 func @vec_rejected_8(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: %[[C0:[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, 1 : memref @@ -291,6 +289,7 @@ // CHECK: for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128 // CHECK: %{{.*}} = affine.apply #map0(%{{.*}}) // CHECK: %{{.*}} = affine.apply #map0(%{{.*}}) +// CHECK: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_0]]} : memref, vector<128xf32> affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %{{.*}} in DFS post-order prevents vectorizing %{{.*}} affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector @@ -302,7 +301,6 @@ // CHECK-LABEL: func @vec_rejected_9 func @vec_rejected_9(%A : memref, %B : memref) { -// CHECK-DAG: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-DAG: %[[C0:[a-z0-9_]+]] = constant 0 : index // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, 0 : memref // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, 1 : memref @@ -316,6 +314,7 @@ // CHECK: for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128 // CHECK: %{{.*}} = affine.apply #map0(%{{.*}}) // CHECK-NEXT: %{{.*}} = affine.apply #map0(%{{.*}}) +// CHECK-NEXT: %{{.*}} = constant 0.0{{.*}}: f32 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1_0]]} : memref, vector<128xf32> affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %i18 in DFS post-order prevents vectorizing %{{.*}} affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir --- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir @@ -54,7 +54,7 @@ affine.for %i0 = 0 to %M { affine.for %i1 = 0 to %N { // CHECK: [[C1:%.*]] = constant dense<1.000000e+00> : vector<32x256xf32> - // CHECK: vector.transfer_write [[C1]], {{.*}} {permutation_map = #[[map_id2]]} : vector<32x256xf32>, memref + // CHECK: vector.transfer_write [[C1]], {{.*}} : vector<32x256xf32>, memref // non-scoped %f1 affine.store %f1, %A[%i0, %i1] : memref } @@ -62,22 +62,22 @@ affine.for %i2 = 0 to %M { affine.for %i3 = 0 to %N { // CHECK: [[C3:%.*]] = constant dense<2.000000e+00> : vector<32x256xf32> - // CHECK: vector.transfer_write [[C3]], {{.*}} {permutation_map = #[[map_id2]]} : vector<32x256xf32>, memref + // CHECK: vector.transfer_write [[C3]], {{.*}} : vector<32x256xf32>, memref // non-scoped %f2 affine.store %f2, %B[%i2, %i3] : memref } } affine.for %i4 = 0 to %M { affine.for %i5 = 0 to %N { - // CHECK: [[A5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} {permutation_map = #[[map_id2]]} : memref, vector<32x256xf32> - // CHECK: [[B5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} {permutation_map = #[[map_id2]]} : memref, vector<32x256xf32> + // CHECK: [[A5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} : memref, vector<32x256xf32> + // CHECK: [[B5:%.*]] = vector.transfer_read %{{.*}}[{{.*}}], %{{.*}} : memref, vector<32x256xf32> // CHECK: [[S5:%.*]] = addf [[A5]], [[B5]] : vector<32x256xf32> // CHECK: [[SPLAT1:%.*]] = constant dense<1.000000e+00> : vector<32x256xf32> // CHECK: [[S6:%.*]] = addf [[S5]], [[SPLAT1]] : vector<32x256xf32> // CHECK: [[SPLAT2:%.*]] = constant dense<2.000000e+00> : vector<32x256xf32> // CHECK: [[S7:%.*]] = addf [[S5]], [[SPLAT2]] : vector<32x256xf32> // CHECK: [[S8:%.*]] = addf [[S7]], [[S6]] : vector<32x256xf32> - // CHECK: vector.transfer_write [[S8]], {{.*}} {permutation_map = #[[map_id2]]} : vector<32x256xf32>, memref + // CHECK: vector.transfer_write [[S8]], {{.*}} : vector<32x256xf32>, memref // %a5 = affine.load %A[%i4, %i5] : memref %b5 = affine.load %B[%i4, %i5] : memref @@ -110,7 +110,7 @@ // VECT: {{.*}} #[[map_id1]](%[[M]]) step 4 { // VECT-NEXT: {{.*}} #[[map_id1]](%[[N]]) step 8 { // VECT: %[[VC0:.*]] = constant dense<0.000000e+00> : vector<4x8xf32> - // VECT-NEXT: vector.transfer_write %[[VC0]], %{{.*}}[%{{.*}}, %{{.*}}] {permutation_map = #[[map_id2]]} : vector<4x8xf32>, memref + // VECT-NEXT: vector.transfer_write %[[VC0]], %{{.*}}[%{{.*}}, %{{.*}}] : vector<4x8xf32>, memref affine.for %i0 = affine_map<(d0) -> (d0)>(%c0) to affine_map<(d0) -> (d0)>(%M) { affine.for %i1 = affine_map<(d0) -> (d0)>(%c0) to affine_map<(d0) -> (d0)>(%N) { %cst = constant 0.000000e+00 : f32 @@ -120,12 +120,12 @@ // VECT: affine.for %[[I2:.*]] = #[[map_id1]](%[[C0]]) to #[[map_id1]](%[[M]]) step 4 { // VECT-NEXT: affine.for %[[I3:.*]] = #[[map_id1]](%[[C0]]) to #[[map_id1]](%[[N]]) step 8 { // VECT-NEXT: affine.for %[[I4:.*]] = #map5(%[[C0]]) to #[[map_id1]](%[[K]]) { - // VECT-NEXT: %[[A:.*]] = vector.transfer_read %{{.*}}[%[[I4]], %[[I3]]], %{{.*}} {permutation_map = #[[map_proj_d0d1_zerod1]]} : memref, vector<4x8xf32> - // VECT-NEXT: %[[B:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I4]]], %{{.*}} {permutation_map = #[[map_proj_d0d1_d0zero]]} : memref, vector<4x8xf32> + // VECT: %[[A:.*]] = vector.transfer_read %{{.*}}[%[[I4]], %[[I3]]], %{{.*}} {permutation_map = #[[map_proj_d0d1_zerod1]]} : memref, vector<4x8xf32> + // VECT: %[[B:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I4]]], %{{.*}} {permutation_map = #[[map_proj_d0d1_d0zero]]} : memref, vector<4x8xf32> // VECT-NEXT: %[[C:.*]] = mulf %[[B]], %[[A]] : vector<4x8xf32> - // VECT-NEXT: %[[D:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I3]]], %{{.*}} {permutation_map = #[[map_id2]]} : memref, vector<4x8xf32> + // VECT: %[[D:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I3]]], %{{.*}} : memref, vector<4x8xf32> // VECT-NEXT: %[[E:.*]] = addf %[[D]], %[[C]] : vector<4x8xf32> - // VECT-NEXT: vector.transfer_write %[[E]], %{{.*}}[%[[I2]], %[[I3]]] {permutation_map = #[[map_id2]]} : vector<4x8xf32>, memref + // VECT: vector.transfer_write %[[E]], %{{.*}}[%[[I2]], %[[I3]]] : vector<4x8xf32>, memref affine.for %i2 = affine_map<(d0) -> (d0)>(%c0) to affine_map<(d0) -> (d0)>(%M) { affine.for %i3 = affine_map<(d0) -> (d0)>(%c0) to affine_map<(d0) -> (d0)>(%N) { affine.for %i4 = affine_map<(d0) -> (d0)>(%c0) to affine_map<(d0) -> (d0)>(%K) { diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_3d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_3d.mlir --- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_3d.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_3d.mlir @@ -12,7 +12,7 @@ // CHECK: affine.for %{{.*}} = 0 to %{{.*}} step 32 { // CHECK: affine.for %{{.*}} = 0 to %{{.*}} step 64 { // CHECK: affine.for %{{.*}} = 0 to %{{.*}} step 256 { - // CHECK: %{{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[map_proj_d0d1d2_d0d1d2]]} : memref, vector<32x64x256xf32> + // CHECK: %{{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}], %{{.*}} : memref, vector<32x64x256xf32> affine.for %t0 = 0 to %0 { affine.for %t1 = 0 to %0 { affine.for %i0 = 0 to %0 { diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir --- a/mlir/test/Dialect/Vector/invalid.mlir +++ b/mlir/test/Dialect/Vector/invalid.mlir @@ -238,17 +238,28 @@ func @test_vector.transfer_read(%arg0: memref) { %c3 = constant 3 : index %cst = constant 3.0 : f32 - // expected-error@+1 {{two types required}} + // expected-error@+1 {{requires two types}} %0 = vector.transfer_read %arg0[%c3, %c3], %cst { permutation_map = affine_map<()->(0)> } : memref } // ----- -func @test_vector.transfer_read(%arg0: memref) { +func @test_vector.transfer_read(%arg0: vector<4x3xf32>) { %c3 = constant 3 : index - %cst = constant 3.0 : f32 - // expected-error@+1 {{requires 2 indices}} - %0 = vector.transfer_read %arg0[%c3, %c3, %c3], %cst { permutation_map = affine_map<()->(0)> } : memref, vector<128xf32> + %f0 = constant 0.0 : f32 + %vf0 = splat %f0 : vector<4x3xf32> + // expected-error@+1 {{ requires memref type}} + %0 = vector.transfer_read %arg0[%c3, %c3], %vf0 : vector<4x3xf32>, vector<1x1x2x3xf32> +} + +// ----- + +func @test_vector.transfer_read(%arg0: memref<4x3xf32>) { + %c3 = constant 3 : index + %f0 = constant 0.0 : f32 + %vf0 = splat %f0 : vector<4x3xf32> + // expected-error@+1 {{ requires vector type}} + %0 = vector.transfer_read %arg0[%c3, %c3], %vf0 : memref<4x3xf32>, f32 } // ----- @@ -256,8 +267,8 @@ func @test_vector.transfer_read(%arg0: memref) { %c3 = constant 3 : index %cst = constant 3.0 : f32 - // expected-error@+1 {{requires attribute 'permutation_map'}} - %0 = vector.transfer_read %arg0[%c3, %c3], %cst {perm = affine_map<(d0)->(d0)>} : memref, vector<128xf32> + // expected-error@+1 {{requires 2 indices}} + %0 = vector.transfer_read %arg0[%c3, %c3, %c3], %cst { permutation_map = affine_map<()->(0)> } : memref, vector<128xf32> } // ----- @@ -339,9 +350,29 @@ func @test_vector.transfer_write(%arg0: memref) { %c3 = constant 3 : index - %cst = constant dense<3.0> : vector<128 x f32> - // expected-error@+1 {{expected 5 operand types but had 4}} - %0 = "vector.transfer_write"(%cst, %arg0, %c3, %c3, %c3) {permutation_map = affine_map<()->(0)>} : (vector<128xf32>, memref, index, index) -> () + %cst = constant 3.0 : f32 + // expected-error@+1 {{requires two types}} + vector.transfer_write %arg0, %arg0[%c3, %c3] : memref +} + +// ----- + +func @test_vector.transfer_write(%arg0: memref>) { + %c3 = constant 3 : index + %f0 = constant 0.0 : f32 + %vf0 = splat %f0 : vector<4x3xf32> + // expected-error@+1 {{ requires vector type}} + vector.transfer_write %arg0, %arg0[%c3, %c3] : memref>, vector<4x3xf32> +} + +// ----- + +func @test_vector.transfer_write(%arg0: vector<4x3xf32>) { + %c3 = constant 3 : index + %f0 = constant 0.0 : f32 + %vf0 = splat %f0 : vector<4x3xf32> + // expected-error@+1 {{ requires memref type}} + vector.transfer_write %arg0, %arg0[%c3, %c3] : vector<4x3xf32>, f32 } // ----- @@ -349,8 +380,8 @@ func @test_vector.transfer_write(%arg0: memref) { %c3 = constant 3 : index %cst = constant dense<3.0> : vector<128 x f32> - // expected-error@+1 {{requires 2 indices}} - vector.transfer_write %cst, %arg0[%c3, %c3, %c3] {permutation_map = affine_map<()->(0)>} : vector<128xf32>, memref + // expected-error@+1 {{expected 5 operand types but had 4}} + %0 = "vector.transfer_write"(%cst, %arg0, %c3, %c3, %c3) {permutation_map = affine_map<()->(0)>} : (vector<128xf32>, memref, index, index) -> () } // ----- @@ -358,8 +389,8 @@ func @test_vector.transfer_write(%arg0: memref) { %c3 = constant 3 : index %cst = constant dense<3.0> : vector<128 x f32> - // expected-error@+1 {{requires attribute 'permutation_map'}} - vector.transfer_write %cst, %arg0[%c3, %c3] {perm = affine_map<(d0)->(d0)>} : vector<128xf32>, memref + // expected-error@+1 {{requires 2 indices}} + vector.transfer_write %cst, %arg0[%c3, %c3, %c3] {permutation_map = affine_map<()->(0)>} : vector<128xf32>, memref } // ----- diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir --- a/mlir/test/Dialect/Vector/ops.mlir +++ b/mlir/test/Dialect/Vector/ops.mlir @@ -20,14 +20,14 @@ %2 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d0)>} : memref, vector<128xf32> // CHECK: vector.transfer_read %3 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d1)>} : memref, vector<128xf32> - // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref>, vector<1x1x4x3xf32> + // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : memref>, vector<1x1x4x3xf32> %4 = vector.transfer_read %arg1[%c3, %c3], %vf0 {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref>, vector<1x1x4x3xf32> // CHECK: vector.transfer_write vector.transfer_write %0, %arg0[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0)>} : vector<128xf32>, memref // CHECK: vector.transfer_write vector.transfer_write %1, %arg0[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d1, d0)>} : vector<3x7xf32>, memref - // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] {permutation_map = #[[MAP0]]} : vector<1x1x4x3xf32>, memref> + // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, memref> vector.transfer_write %4, %arg1[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : vector<1x1x4x3xf32>, memref> return diff --git a/mlir/test/Dialect/Vector/vector-transforms.mlir b/mlir/test/Dialect/Vector/vector-transforms.mlir --- a/mlir/test/Dialect/Vector/vector-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-transforms.mlir @@ -231,26 +231,26 @@ // Check LHS vector.transfer read is split for each user. -// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x2xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x2xf32>, vector<2x2xf32> +// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} : memref<4x2xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C0]]], %{{.*}} : memref<4x2xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR2:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<2x4xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C2]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<2x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR2:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} : memref<2x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C2]]], %{{.*}} : memref<2x4xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR4:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x4xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR5:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C2]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x4xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR6:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x4xf32>, vector<2x2xf32> -// CHECK-NEXT: %[[VTR7:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C2]]], %{{.*}} {permutation_map = #[[MAP0]]} : memref<4x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR4:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR5:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C2]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR6:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32> +// CHECK-NEXT: %[[VTR7:.*]] = vector.transfer_read %{{.*}}[%[[C2]], %[[C2]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32> // CHECK-NEXT: %[[R0:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR0]], %[[VTR2]], %[[VTR4]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> // CHECK-NEXT: %[[R1:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR0]], %[[VTR3]], %[[VTR5]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> // CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> // CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"]} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> -// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {permutation_map = #[[MAP0]]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {permutation_map = #[[MAP0]]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {permutation_map = #[[MAP0]]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {permutation_map = #[[MAP0]]} : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] : vector<2x2xf32>, memref<4x4xf32> // CHECK-NEXT: return func @contraction4x4_ikj_xfer_read(%arg0 : memref<4x2xf32>, @@ -425,10 +425,10 @@ // CHECK-LABEL: func @vector_transfers_vector_element_type // CHECK: %[[C0:.*]] = constant 0 : index // CHECK: %[[C1:.*]] = constant 1 : index -// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP1]]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> -// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {permutation_map = #[[MAP1]]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> -// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {permutation_map = #[[MAP1]]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> -// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {permutation_map = #[[MAP1]]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> +// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> +// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> +// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> +// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> func @vector_transfers_vector_element_type() { %c0 = constant 0 : index