diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h --- a/mlir/include/mlir/Dialect/Vector/VectorOps.h +++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h @@ -121,18 +121,18 @@ /// intrinsics. Flat = 1, }; -/// Enum to control the splitting of `vector.transfer` operations into masked -/// and unmasked variants. +/// Enum to control the splitting of `vector.transfer` operations into +/// in-bounds and out-of-bounds variants. enum class VectorTransferSplit { /// Do not split vector transfer operations. None = 0, - /// Split using masked + unmasked vector.transfer operations. + /// Split using in-bounds + out-of-bounds vector.transfer operations. VectorTransfer = 1, - /// Split using a unmasked vector.transfer + linalg.fill + linalg.copy + /// Split using an in-bounds vector.transfer + linalg.fill + linalg.copy /// operations. LinalgCopy = 2, - /// Do not split vector transfer operation but instead mark it as "unmasked". - ForceUnmasked = 3 + /// Do not split vector transfer operation but instead mark it as "in-bounds". + ForceInBounds = 3 }; /// Structure to control the behavior of vector transform patterns. struct VectorTransformsOptions { diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td --- a/mlir/include/mlir/Dialect/Vector/VectorOps.td +++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td @@ -1139,7 +1139,7 @@ ]>, Arguments<(ins AnyShaped:$source, Variadic:$indices, AffineMapAttr:$permutation_map, AnyType:$padding, - OptionalAttr:$masked)>, + OptionalAttr:$in_bounds)>, Results<(outs AnyVector:$vector)> { let summary = "Reads a supervector from memory into an SSA vector value."; @@ -1166,15 +1166,14 @@ The size of the slice is specified by the size of the vector, given as the return type. - An `ssa-value` of the same elemental type as the MemRef/Tensor is provided - as the last operand to specify padding in the case of out-of-bounds - accesses. + An SSA value `padding` of the same elemental type as the MemRef/Tensor is + provided to specify a fallback value in the case of out-of-bounds accesses. An optional boolean array attribute is provided to specify which dimensions - of the transfer need masking. When a dimension is specified as not requiring - masking, the `vector.transfer_read` may be lowered to simple loads. The - absence of this `masked` attribute signifies that all dimensions of the - transfer need to be masked. + of the transfer are guaranteed to be within bounds. The absence of this + `in_bounds` attribute signifies that any dimension of the transfer may be + out-of-bounds. A `vector.transfer_read` can be lowered to a simple load if + all dimensions are specified to be within bounds. This operation is called 'read' by opposition to 'load' because the super-vector granularity is generally not representable with a single @@ -1291,15 +1290,15 @@ // Builder that sets padding to zero. OpBuilder<(ins "VectorType":$vector, "Value":$source, "ValueRange":$indices, "AffineMap":$permutationMap, - CArg<"ArrayRef", "{}">:$maybeMasked)>, + CArg<"ArrayRef", "{}">:$inBounds)>, // Builder that sets padding to 'getMinorIdentityMap'. OpBuilder<(ins "VectorType":$vector, "Value":$source, "ValueRange":$indices, "Value":$padding, - CArg<"ArrayRef", "{}">:$maybeMasked)>, + CArg<"ArrayRef", "{}">:$inBounds)>, // Builder that sets permutation map (resp. padding) to // 'getMinorIdentityMap' (resp. zero). OpBuilder<(ins "VectorType":$vector, "Value":$source, - "ValueRange":$indices, CArg<"ArrayRef", "{}">:$maybeMasked)> + "ValueRange":$indices, CArg<"ArrayRef", "{}">:$inBounds)>, ]; let hasFolder = 1; @@ -1314,7 +1313,7 @@ Arguments<(ins AnyVector:$vector, AnyShaped:$source, Variadic:$indices, AffineMapAttr:$permutation_map, - OptionalAttr:$masked)>, + OptionalAttr:$in_bounds)>, Results<(outs Optional:$result)> { let summary = "The vector.transfer_write op writes a supervector to memory."; @@ -1343,10 +1342,10 @@ The size of the slice is specified by the size of the vector. An optional boolean array attribute is provided to specify which dimensions - of the transfer need masking. When a dimension is specified as not requiring - masking, the `vector.transfer_write` may be lowered to simple stores. The - absence of this `mask` attribute signifies that all dimensions of the - transfer need to be masked. + of the transfer are guaranteed to be within bounds. The absence of this + `in_bounds` attribute signifies that any dimension of the transfer may be + out-of-bounds. A `vector.transfer_write` can be lowered to a simple store + if all dimensions are specified to be within bounds. This operation is called 'write' by opposition to 'store' because the super-vector granularity is generally not representable with a single @@ -1387,13 +1386,13 @@ let builders = [ // Builder that sets permutation map to 'getMinorIdentityMap'. OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices, - CArg<"ArrayRef", "{}">:$maybeMasked)>, + CArg<"ArrayRef", "{}">:$inBounds)>, OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices, "AffineMap":$permutationMap)>, OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices, - "AffineMapAttr":$permutationMap, "ArrayAttr":$masked)>, + "AffineMapAttr":$permutationMap, "ArrayAttr":$inBounds)>, OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices, - "AffineMap":$permutationMap, "ArrayAttr":$masked)>, + "AffineMap":$permutationMap, "ArrayAttr":$inBounds)>, ]; let hasFolder = 1; diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h --- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h @@ -166,7 +166,8 @@ UnrollVectorOptions options; }; -/// Split a vector.transfer operation into an unmasked fastpath and a slowpath. +/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds +/// masking) fastpath and a slowpath. /// If `ifOp` is not null and the result is `success, the `ifOp` points to the /// newly created conditional upon function return. /// To accomodate for the fact that the original vector.transfer indexing may be @@ -185,11 +186,11 @@ /// memref.cast %A: memref to compatibleMemRefType /// scf.yield %view : compatibleMemRefType, index, index /// } else { -/// // slowpath, masked vector.transfer or linalg.copy. +/// // slowpath, not in-bounds vector.transfer or linalg.copy. /// memref.cast %alloc: memref to compatibleMemRefType /// scf.yield %4 : compatibleMemRefType, index, index // } -/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]} +/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} /// ``` /// where `alloc` is a top of the function alloca'ed buffer of one vector. /// diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td --- a/mlir/include/mlir/Interfaces/VectorInterfaces.td +++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td @@ -53,12 +53,12 @@ let methods = [ StaticInterfaceMethod< - /*desc=*/"Return the `masked` attribute name.", + /*desc=*/"Return the `in_bounds` attribute name.", /*retTy=*/"StringRef", - /*methodName=*/"getMaskedAttrName", + /*methodName=*/"getInBoundsAttrName", /*args=*/(ins), /*methodBody=*/"", - /*defaultImplementation=*/ [{ return "masked"; }] + /*defaultImplementation=*/ [{ return "in_bounds"; }] >, StaticInterfaceMethod< /*desc=*/"Return the `permutation_map` attribute name.", @@ -70,16 +70,16 @@ >, InterfaceMethod< /*desc=*/[{ - Return `false` when the `masked` attribute at dimension - `dim` is set to `false`. Return `true` otherwise.}], + Return `true` when the `in_bounds` attribute at dimension + `dim` is set to `true`. Return `false` otherwise.}], /*retTy=*/"bool", - /*methodName=*/"isMaskedDim", + /*methodName=*/"isDimInBounds", /*args=*/(ins "unsigned":$dim), /*methodBody=*/"", /*defaultImplementation=*/[{ - return !$_op.masked() || - $_op.masked()->template cast()[dim] - .template cast().getValue(); + return $_op.in_bounds() && + $_op.in_bounds()->template cast()[dim] + .template cast().getValue(); }] >, InterfaceMethod< @@ -115,11 +115,11 @@ /*defaultImplementation=*/ >, InterfaceMethod< - /*desc=*/"Return the `masked` boolean ArrayAttr.", + /*desc=*/"Return the `in_bounds` boolean ArrayAttr.", /*retTy=*/"Optional", - /*methodName=*/"masked", + /*methodName=*/"in_bounds", /*args=*/(ins), - /*methodBody=*/"return $_op.masked();" + /*methodBody=*/"return $_op.in_bounds();" /*defaultImplementation=*/ >, InterfaceMethod< @@ -162,14 +162,15 @@ "return $_op.getShapedType().getRank() - $_op.getTransferRank();" >, InterfaceMethod< - /*desc=*/[{ Returns true if at least one of the dimensions is masked.}], + /*desc=*/[{ Returns true if at least one of the dimensions may be + out-of-bounds.}], /*retTy=*/"bool", - /*methodName=*/"hasMaskedDim", + /*methodName=*/"hasOutOfBoundsDim", /*args=*/(ins), /*methodBody=*/"", /*defaultImplementation=*/[{ for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx) - if ($_op.isMaskedDim(idx)) + if (!$_op.isDimInBounds(idx)) return true; return false; }] diff --git a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp --- a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp +++ b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp @@ -79,7 +79,7 @@ ArrayRef sourceIndices, PatternRewriter &rewriter) const { rewriter.replaceOpWithNewOp( loadOp, loadOp.getVectorType(), subViewOp.source(), sourceIndices, - loadOp.permutation_map(), loadOp.padding(), loadOp.maskedAttr()); + loadOp.permutation_map(), loadOp.padding(), loadOp.in_boundsAttr()); } template <> @@ -92,12 +92,12 @@ template <> void StoreOpOfSubViewFolder::replaceOp( - vector::TransferWriteOp tranferWriteOp, memref::SubViewOp subViewOp, + vector::TransferWriteOp transferWriteOp, memref::SubViewOp subViewOp, ArrayRef sourceIndices, PatternRewriter &rewriter) const { rewriter.replaceOpWithNewOp( - tranferWriteOp, tranferWriteOp.vector(), subViewOp.source(), - sourceIndices, tranferWriteOp.permutation_map(), - tranferWriteOp.maskedAttr()); + transferWriteOp, transferWriteOp.vector(), subViewOp.source(), + sourceIndices, transferWriteOp.permutation_map(), + transferWriteOp.in_boundsAttr()); } } // namespace diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp @@ -1248,14 +1248,15 @@ Value vectorDataPtr = castDataPtr(rewriter, loc, dataPtr, memRefType, toLLVMTy(vtp)); - if (!xferOp.isMaskedDim(0)) + if (xferOp.isDimInBounds(0)) return replaceTransferOpWithLoadOrStore(rewriter, *this->getTypeConverter(), loc, xferOp, operands, vectorDataPtr); // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ]. // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ]. - // 4. Let dim the memref dimension, compute the vector comparison mask: + // 4. Let dim the memref dimension, compute the vector comparison mask + // (in-bounds mask): // [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ] // // TODO: when the leaf transfer rank is k > 1, we need the last `k` diff --git a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp --- a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp +++ b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp @@ -72,7 +72,7 @@ return failure(); // Have it handled in vector->llvm conversion pass. - if (!xferOp.isMaskedDim(0)) + if (xferOp.isDimInBounds(0)) return failure(); auto toLLVMTy = [&](Type t) { diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp --- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp +++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp @@ -230,7 +230,7 @@ Value iv = std::get<0>(it), off = std::get<1>(it), ub = std::get<2>(it); using namespace mlir::edsc::op; majorIvsPlusOffsets.push_back(iv + off); - if (xferOp.isMaskedDim(leadingRank + idx)) { + if (!xferOp.isDimInBounds(leadingRank + idx)) { Value inBoundsCond = onTheFlyFoldSLT(majorIvsPlusOffsets.back(), ub); if (inBoundsCond) inBoundsCondition = (inBoundsCondition) @@ -276,14 +276,14 @@ Value memref = xferOp.source(); auto map = getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType); - ArrayAttr masked; - if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) { + ArrayAttr inBounds; + if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) { OpBuilder &b = ScopedContext::getBuilderRef(); - masked = b.getBoolArrayAttr({false}); + inBounds = b.getBoolArrayAttr({true}); } return vector_transfer_read(minorVectorType, memref, indexing, AffineMapAttr::get(map), xferOp.padding(), - masked); + inBounds); }; // 1. Compute the inBoundsCondition in the current loops ivs + offset @@ -382,13 +382,13 @@ result = memref_load(alloc, majorIvs); auto map = getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType); - ArrayAttr masked; - if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) { + ArrayAttr inBounds; + if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) { OpBuilder &b = ScopedContext::getBuilderRef(); - masked = b.getBoolArrayAttr({false}); + inBounds = b.getBoolArrayAttr({true}); } vector_transfer_write(result, xferOp.source(), indexing, - AffineMapAttr::get(map), masked); + AffineMapAttr::get(map), inBounds); }; // 1. Compute the inBoundsCondition in the current loops ivs + offset diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp --- a/mlir/lib/Dialect/Vector/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/VectorOps.cpp @@ -2192,7 +2192,12 @@ static LogicalResult verifyTransferOp(Operation *op, ShapedType shapedType, VectorType vectorType, AffineMap permutationMap, - ArrayAttr optionalMasked) { + ArrayAttr inBounds) { + if (op->hasAttr("masked")) { + return op->emitOpError("masked attribute has been removed. " + "Use in_bounds instead."); + } + if (!shapedType.isa()) return op->emitOpError( "requires source to be a memref or ranked tensor type"); @@ -2239,11 +2244,10 @@ return op->emitOpError("requires a permutation_map with input dims of the " "same rank as the source type"); - if (optionalMasked) { - if (permutationMap.getNumResults() != - static_cast(optionalMasked.size())) - return op->emitOpError("expects the optional masked attr of same rank as " - "permutation_map results: ") + if (inBounds) { + if (permutationMap.getNumResults() != static_cast(inBounds.size())) + return op->emitOpError("expects the optional in_bounds attr of same rank " + "as permutation_map results: ") << AffineMapAttr::get(permutationMap); } @@ -2254,58 +2258,58 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result, VectorType vectorType, Value source, ValueRange indices, AffineMap permutationMap, - ArrayRef maybeMasked) { + ArrayRef inBounds) { Type elemType = source.getType().cast().getElementType(); Value padding = builder.create(result.location, elemType, builder.getZeroAttr(elemType)); - if (maybeMasked.empty()) + if (inBounds.empty()) return build(builder, result, vectorType, source, indices, permutationMap, padding, ArrayAttr()); - ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked); + ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds); build(builder, result, vectorType, source, indices, permutationMap, padding, - maskedArrayAttr); + inBoundsArrayAttr); } /// Builder that sets permutation map to 'getMinorIdentityMap'. void TransferReadOp::build(OpBuilder &builder, OperationState &result, VectorType vectorType, Value source, ValueRange indices, Value padding, - ArrayRef maybeMasked) { + ArrayRef inBounds) { auto permMap = getTransferMinorIdentityMap( source.getType().cast(), vectorType); - if (maybeMasked.empty()) + if (inBounds.empty()) return build(builder, result, vectorType, source, indices, permMap, padding, ArrayAttr()); - ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked); + ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds); build(builder, result, vectorType, source, indices, permMap, padding, - maskedArrayAttr); + inBoundsArrayAttr); } /// Builder that sets permutation map (resp. padding) to 'getMinorIdentityMap' /// (resp. zero). void TransferReadOp::build(OpBuilder &builder, OperationState &result, VectorType vectorType, Value source, - ValueRange indices, ArrayRef maybeMasked) { + ValueRange indices, ArrayRef inBounds) { auto permMap = getTransferMinorIdentityMap( source.getType().cast(), vectorType); - build(builder, result, vectorType, source, indices, permMap, maybeMasked); + build(builder, result, vectorType, source, indices, permMap, inBounds); } static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) { SmallVector elidedAttrs; if (op.permutation_map().isMinorIdentity()) elidedAttrs.push_back(op.getPermutationMapAttrName()); - bool elideMasked = true; - if (auto maybeMasked = op.masked()) { - for (auto attr : *maybeMasked) { - if (!attr.template cast().getValue()) { - elideMasked = false; + bool elideInBounds = true; + if (auto inBounds = op.in_bounds()) { + for (auto attr : *inBounds) { + if (attr.template cast().getValue()) { + elideInBounds = false; break; } } } - if (elideMasked) - elidedAttrs.push_back(op.getMaskedAttrName()); + if (elideInBounds) + elidedAttrs.push_back(op.getInBoundsAttrName()); p.printOptionalAttrDict(op->getAttrs(), elidedAttrs); } @@ -2366,7 +2370,7 @@ if (failed(verifyTransferOp(op.getOperation(), shapedType, vectorType, permutationMap, - op.masked() ? *op.masked() : ArrayAttr()))) + op.in_bounds() ? *op.in_bounds() : ArrayAttr()))) return failure(); if (auto sourceVectorElementType = sourceElementType.dyn_cast()) { @@ -2438,23 +2442,23 @@ } template -static LogicalResult foldTransferMaskAttribute(TransferOp op) { +static LogicalResult foldTransferInBoundsAttribute(TransferOp op) { AffineMap permutationMap = op.permutation_map(); if (!permutationMap.isMinorIdentity()) return failure(); bool changed = false; - SmallVector isMasked; - isMasked.reserve(op.getTransferRank()); + SmallVector newInBounds; + newInBounds.reserve(op.getTransferRank()); op.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) { - // Already marked unmasked, nothing to see here. - if (!op.isMaskedDim(resultIdx)) { - isMasked.push_back(false); + // Already marked as in-bounds, nothing to see here. + if (op.isDimInBounds(resultIdx)) { + newInBounds.push_back(true); return; } - // Currently masked, check whether we can statically determine it is + // Currently out-of-bounds, check whether we can statically determine it is // inBounds. auto inBounds = isInBounds(op, resultIdx, indicesIdx); - isMasked.push_back(!inBounds); + newInBounds.push_back(inBounds); // We commit the pattern if it is "more inbounds". changed |= inBounds; }); @@ -2462,13 +2466,14 @@ return failure(); // OpBuilder is only used as a helper to build an I64ArrayAttr. OpBuilder b(op.getContext()); - op->setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked)); + op->setAttr(TransferOp::getInBoundsAttrName(), + b.getBoolArrayAttr(newInBounds)); return success(); } OpFoldResult TransferReadOp::fold(ArrayRef) { /// transfer_read(memrefcast) -> transfer_read - if (succeeded(foldTransferMaskAttribute(*this))) + if (succeeded(foldTransferInBoundsAttribute(*this))) return getResult(); if (succeeded(foldMemRefCast(*this))) return getResult(); @@ -2496,40 +2501,40 @@ /// Builder that sets permutation map to 'getMinorIdentityMap'. void TransferWriteOp::build(OpBuilder &builder, OperationState &result, Value vector, Value source, ValueRange indices, - ArrayRef maybeMasked) { + ArrayRef inBounds) { auto vectorType = vector.getType().cast(); auto permMap = getTransferMinorIdentityMap( source.getType().cast(), vectorType); - if (maybeMasked.empty()) + if (inBounds.empty()) return build(builder, result, vector, source, indices, permMap, ArrayAttr()); - ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked); - build(builder, result, vector, source, indices, permMap, maskedArrayAttr); + ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds); + build(builder, result, vector, source, indices, permMap, inBoundsArrayAttr); } void TransferWriteOp::build(OpBuilder &builder, OperationState &result, Value vector, Value source, ValueRange indices, AffineMap permutationMap) { build(builder, result, vector, source, indices, permutationMap, - /*maybeMasked=*/ArrayAttr()); + /*inBounds=*/ArrayAttr()); } void TransferWriteOp::build(OpBuilder &builder, OperationState &result, Value vector, Value source, ValueRange indices, AffineMapAttr permutationMap, - /*optional*/ ArrayAttr masked) { + /*optional*/ ArrayAttr inBounds) { Type resultType = source.getType().dyn_cast(); build(builder, result, resultType, vector, source, indices, permutationMap, - masked); + inBounds); } void TransferWriteOp::build(OpBuilder &builder, OperationState &result, Value vector, Value source, ValueRange indices, AffineMap permutationMap, - /*optional*/ ArrayAttr masked) { + /*optional*/ ArrayAttr inBounds) { Type resultType = source.getType().dyn_cast(); build(builder, result, resultType, vector, source, indices, permutationMap, - masked); + inBounds); } static ParseResult parseTransferWriteOp(OpAsmParser &parser, @@ -2585,7 +2590,7 @@ if (failed(verifyTransferOp(op.getOperation(), shapedType, vectorType, permutationMap, - op.masked() ? *op.masked() : ArrayAttr()))) + op.in_bounds() ? *op.in_bounds() : ArrayAttr()))) return failure(); return verifyPermutationMap(permutationMap, @@ -2595,9 +2600,9 @@ /// Fold: /// ``` /// %t1 = ... -/// %v = vector.transfer_read %t0[%c0...], {masked = [false...]} : +/// %v = vector.transfer_read %t0[%c0...], {in_bounds = [true...]} : /// tensor, vector -/// %t2 = vector.transfer_write %v, %t1[%c0...] {masked = [false...]} : +/// %t2 = vector.transfer_write %v, %t1[%c0...] {in_bounds = [true...]} : /// vector, tensor /// ``` /// @@ -2627,8 +2632,8 @@ // Bail on mismatching ranks. if (read.getTransferRank() != write.getTransferRank()) return failure(); - // Bail on masked. - if (read.hasMaskedDim() || write.hasMaskedDim()) + // Bail on potential out-of-bounds accesses. + if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim()) return failure(); // Tensor types must be the same. if (read.source().getType() != rankedTensorType) @@ -2656,7 +2661,7 @@ SmallVectorImpl &results) { if (succeeded(foldReadInitWrite(*this, operands, results))) return success(); - if (succeeded(foldTransferMaskAttribute(*this))) + if (succeeded(foldTransferInBoundsAttribute(*this))) return success(); return foldMemRefCast(*this); } diff --git a/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp --- a/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp +++ b/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp @@ -38,7 +38,8 @@ /// transfer_read. static bool transferEncompasses(vector::TransferWriteOp defWrite, vector::TransferReadOp read) { - return !defWrite.hasMaskedDim() && defWrite.indices() == read.indices() && + return !defWrite.hasOutOfBoundsDim() && + defWrite.indices() == read.indices() && defWrite.getVectorType() == read.getVectorType() && defWrite.permutation_map() == read.permutation_map(); } @@ -175,7 +176,7 @@ /// potentially aliasing ops that may reach the transfer_read are post-dominated /// by the transfer_write. void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) { - if (read.hasMaskedDim()) + if (read.hasOutOfBoundsDim()) return; LLVM_DEBUG(DBGS() << "Candidate for Forwarding: " << *read.getOperation() << "\n"); diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp --- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp +++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp @@ -613,12 +613,12 @@ // Get VectorType for slice 'i'. auto sliceVectorType = tupleType.getType(index); // Create split TransferReadOp for 'sliceUser'. - // `masked` attribute propagates conservatively: if the coarse op didn't - // need masking, the fine op doesn't either. + // `in_bounds` attribute propagates conservatively: if the coarse op didn't + // need out-of-bounds masking, the fine op doesn't either. vectorTupleValues[index] = builder.create( loc, sliceVectorType, readOp.source(), sliceIndices, readOp.permutation_map(), readOp.padding(), - readOp.masked() ? *readOp.masked() : ArrayAttr()); + readOp.in_bounds() ? *readOp.in_bounds() : ArrayAttr()); }; generateTransferOpSlices(shapedElementType, sourceVectorType, tupleType, targetShape, strides, indices, builder, createSlice); @@ -662,7 +662,7 @@ loc, element.getResult(), resultTensor ? resultTensor : writeOp.source(), sliceIndices, writeOp.permutation_map(), - writeOp.masked() ? *writeOp.masked() : ArrayAttr()); + writeOp.in_bounds() ? *writeOp.in_bounds() : ArrayAttr()); if (!write->getResults().empty()) resultTensor = write->getResult(0); }; @@ -800,13 +800,13 @@ Value resultTensor; auto createSlice = [&](unsigned index, ArrayRef sliceIndices) { // Create split TransferWriteOp for source vector 'tupleOp.operand[i]'. - // 'masked' attribute propagates conservatively: if the coarse op didn't - // need masking, the fine op doesn't either. + // 'in_bounds' attribute propagates conservatively: if the coarse op + // didn't need out-of-bounds masking, the fine op doesn't either. Operation *write = rewriter.create( loc, tupleOp.getOperand(index), resultTensor ? resultTensor : writeOp.source(), sliceIndices, writeOp.permutation_map(), - writeOp.masked() ? *writeOp.masked() : ArrayAttr()); + writeOp.in_bounds() ? *writeOp.in_bounds() : ArrayAttr()); if (!write->getResults().empty()) resultTensor = write->getResult(0); }; @@ -2267,16 +2267,16 @@ } // Operates under a scoped context to build the condition to ensure that a -// particular VectorTransferOpInterface is unmasked. +// particular VectorTransferOpInterface is in-bounds. static Value createScopedInBoundsCond(VectorTransferOpInterface xferOp) { assert(xferOp.permutation_map().isMinorIdentity() && "Expected minor identity map"); Value inBoundsCond; xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) { // Zip over the resulting vector shape and memref indices. - // If the dimension is known to be unmasked, it does not participate in the - // construction of `inBoundsCond`. - if (!xferOp.isMaskedDim(resultIdx)) + // If the dimension is known to be in-bounds, it does not participate in + // the construction of `inBoundsCond`. + if (xferOp.isDimInBounds(resultIdx)) return; int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx); using namespace edsc::op; @@ -2298,8 +2298,8 @@ // TODO: expand support to these 2 cases. if (!xferOp.permutation_map().isMinorIdentity()) return failure(); - // Must have some masked dimension to be a candidate for splitting. - if (!xferOp.hasMaskedDim()) + // Must have some out-of-bounds dimension to be a candidate for splitting. + if (!xferOp.hasOutOfBoundsDim()) return failure(); // Don't split transfer operations directly under IfOp, this avoids applying // the pattern recursively. @@ -2492,7 +2492,8 @@ return fullPartialIfOp; } -/// Split a vector.transfer operation into an unmasked fastpath and a slowpath. +/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds +/// masking) fastpath and a slowpath. /// If `ifOp` is not null and the result is `success, the `ifOp` points to the /// newly created conditional upon function return. /// To accomodate for the fact that the original vector.transfer indexing may be @@ -2511,11 +2512,11 @@ /// memref.cast %A: memref to compatibleMemRefType /// scf.yield %view : compatibleMemRefType, index, index /// } else { -/// // slowpath, masked vector.transfer or linalg.copy. +/// // slowpath, not in-bounds vector.transfer or linalg.copy. /// memref.cast %alloc: memref to compatibleMemRefType /// scf.yield %4 : compatibleMemRefType, index, index // } -/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]} +/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} /// ``` /// where `alloc` is a top of the function alloca'ed buffer of one vector. /// @@ -2533,10 +2534,11 @@ if (options.vectorTransferSplit == VectorTransferSplit::None) return failure(); - SmallVector bools(xferOp.getTransferRank(), false); - auto unmaskedAttr = b.getBoolArrayAttr(bools); - if (options.vectorTransferSplit == VectorTransferSplit::ForceUnmasked) { - xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr); + SmallVector bools(xferOp.getTransferRank(), true); + auto inBoundsAttr = b.getBoolArrayAttr(bools); + if (options.vectorTransferSplit == VectorTransferSplit::ForceInBounds) { + xferOp->setAttr(vector::TransferReadOp::getInBoundsAttrName(), + inBoundsAttr); return success(); } @@ -2575,7 +2577,7 @@ getCastCompatibleMemRefType(xferOp.getShapedType().cast(), alloc.getType().cast()); - // Read case: full fill + partial copy -> unmasked vector.xfer_read. + // Read case: full fill + partial copy -> in-bounds vector.xfer_read. SmallVector returnTypes(1 + xferOp.getTransferRank(), b.getIndexType()); returnTypes[0] = compatibleMemRefType; @@ -2590,10 +2592,10 @@ if (ifOp) *ifOp = fullPartialIfOp; - // Unmask the existing read op, it always reads from a full buffer. + // Set existing read op to in-bounds, it always reads from a full buffer. for (unsigned i = 0, e = returnTypes.size(); i != e; ++i) xferReadOp.setOperand(i, fullPartialIfOp.getResult(i)); - xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr); + xferOp->setAttr(vector::TransferReadOp::getInBoundsAttrName(), inBoundsAttr); return success(); } @@ -2691,7 +2693,7 @@ } Value newRead = vector_transfer_read(extract.getType(), read.source(), indices, read.permutation_map(), - read.padding(), read.maskedAttr()); + read.padding(), read.in_boundsAttr()); Value dest = rewriter.create( read.getLoc(), read.getType(), rewriter.getZeroAttr(read.getType())); newRead = rewriter.create(read.getLoc(), newRead, dest, @@ -2726,7 +2728,7 @@ std_constant_index(insert.getSourceVectorType().getDimSize(pos)); } vector_transfer_write(insert.vector(), write.source(), indices, - write.permutation_map(), write.maskedAttr()); + write.permutation_map(), write.in_boundsAttr()); rewriter.eraseOp(write); return success(); } @@ -2736,7 +2738,7 @@ /// `vector.transfer_read` to a combination of `vector.load` and /// `vector.broadcast` if all of the following hold: /// - The op reads from a memref with the default layout. -/// - Masking is not required. +/// - Out-of-bounds masking is not required. /// - If the memref's element type is a vector type then it coincides with the /// result type. /// - The permutation map doesn't perform permutation (broadcasting is allowed). @@ -2774,8 +2776,9 @@ // TODO: Support non-default layouts. if (!memRefType.getAffineMaps().empty()) return failure(); - // TODO: When masking is required, we can create a MaskedLoadOp - if (read.hasMaskedDim()) + // TODO: When out-of-bounds masking is required, we can create a + // MaskedLoadOp. + if (read.hasOutOfBoundsDim()) return failure(); Operation *loadOp; @@ -2807,7 +2810,7 @@ /// Progressive lowering of transfer_write. This pattern supports lowering of /// `vector.transfer_write` to `vector.store` if all of the following hold: /// - The op writes to a memref with the default layout. -/// - Masking is not required. +/// - Out-of-bounds masking is not required. /// - If the memref's element type is a vector type then it coincides with the /// type of the written value. /// - The permutation map is the minor identity map (neither permutation nor @@ -2833,8 +2836,9 @@ // TODO: Support non-default layouts. if (!memRefType.getAffineMaps().empty()) return failure(); - // TODO: When masking is required, we can create a MaskedStoreOp - if (write.hasMaskedDim()) + // TODO: When out-of-bounds masking is required, we can create a + // MaskedStoreOp. + if (write.hasOutOfBoundsDim()) return failure(); rewriter.replaceOpWithNewOp( write, write.vector(), write.source(), write.indices()); @@ -2889,7 +2893,7 @@ VectorType::get(newVectorShape, op.getVectorType().getElementType()); Value newRead = rewriter.create( op.getLoc(), newReadType, op.source(), op.indices(), newMap, - op.padding(), op.masked() ? *op.masked() : ArrayAttr()); + op.padding(), op.in_bounds() ? *op.in_bounds() : ArrayAttr()); SmallVector transposePerm(permutation.begin(), permutation.end()); rewriter.replaceOpWithNewOp(op, newRead, transposePerm); @@ -2935,14 +2939,14 @@ originalVecType.getShape().take_back(reducedShapeRank)); VectorType newReadType = VectorType::get(newShape, originalVecType.getElementType()); - ArrayAttr newMask = - op.masked() + ArrayAttr newInBounds = + op.in_bounds() ? rewriter.getArrayAttr( - op.maskedAttr().getValue().take_back(reducedShapeRank)) + op.in_boundsAttr().getValue().take_back(reducedShapeRank)) : ArrayAttr(); Value newRead = rewriter.create( op.getLoc(), newReadType, op.source(), op.indices(), newMap, - op.padding(), newMask); + op.padding(), newInBounds); rewriter.replaceOpWithNewOp(op, originalVecType, newRead); return success(); @@ -3075,14 +3079,14 @@ AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(), newResults, rewriter.getContext()); - ArrayAttr mask; - if (read.masked()) - mask = rewriter.getArrayAttr( - read.maskedAttr().getValue().take_back(newType.getRank())); + ArrayAttr inBounds; + if (read.in_bounds()) + inBounds = rewriter.getArrayAttr( + read.in_boundsAttr().getValue().take_back(newType.getRank())); auto newRead = rewriter.create( read.getLoc(), newType, read.source(), read.indices(), newMap, - read.padding(), mask); + read.padding(), inBounds); rewriter.replaceOpWithNewOp(read, oldType, newRead); return success(); @@ -3115,15 +3119,15 @@ AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(), newResults, rewriter.getContext()); - ArrayAttr mask; - if (write.masked()) - mask = rewriter.getArrayAttr( - write.maskedAttr().getValue().take_back(newType.getRank())); + ArrayAttr inBounds; + if (write.in_bounds()) + inBounds = rewriter.getArrayAttr( + write.in_boundsAttr().getValue().take_back(newType.getRank())); auto newVector = rewriter.create( write.getLoc(), newType, write.vector()); rewriter.replaceOpWithNewOp( - write, newVector, write.source(), write.indices(), newMap, mask); + write, newVector, write.source(), write.indices(), newMap, inBounds); return success(); } diff --git a/mlir/test/Conversion/LinalgToVector/linalg-to-vector.mlir b/mlir/test/Conversion/LinalgToVector/linalg-to-vector.mlir --- a/mlir/test/Conversion/LinalgToVector/linalg-to-vector.mlir +++ b/mlir/test/Conversion/LinalgToVector/linalg-to-vector.mlir @@ -38,8 +38,8 @@ // CHECK: %[[v15:.*]] = affine.min #[[$map4]](%arg4)[%0] // CHECK: %[[v16:.*]] = subview %[[arg1]][%[[arg4]]] [%[[v15]]] [1] : memref to memref // CHECK: %[[v17:.*]] = subview %[[v6]][0] [%[[v13]]] [1] : memref<3xf32> to memref -// CHECK: %[[v19:.*]] = vector.transfer_read %[[v6]][%[[c0]]], %[[cst]] {masked = [false]} : memref<3xf32>, vector<3xf32> -// CHECK: %[[v20:.*]] = vector.transfer_read %[[v7]][%[[c0]]], %[[cst]] {masked = [false]} : memref<3xf32>, vector<3xf32> +// CHECK: %[[v19:.*]] = vector.transfer_read %[[v6]][%[[c0]]], %[[cst]] {in_bounds = [true]} : memref<3xf32>, vector<3xf32> +// CHECK: %[[v20:.*]] = vector.transfer_read %[[v7]][%[[c0]]], %[[cst]] {in_bounds = [true]} : memref<3xf32>, vector<3xf32> // CHECK: %[[v21:.*]] = mulf %[[v19]], %[[v20]] : vector<3xf32> // CHECK: %[[v22:.*]] = vector.reduction "add", %[[v21]], %[[cst]] : vector<3xf32> into f32 // CHECK: store %[[v22]], %[[v8]][%[[c0]]] : memref<1xf32> diff --git a/mlir/test/Conversion/StandardToSPIRV/legalization.mlir b/mlir/test/Conversion/StandardToSPIRV/legalization.mlir --- a/mlir/test/Conversion/StandardToSPIRV/legalization.mlir +++ b/mlir/test/Conversion/StandardToSPIRV/legalization.mlir @@ -74,10 +74,10 @@ // CHECK: [[INDEX1:%.*]] = addi [[ARG1]], [[STRIDE1]] : index // CHECK: [[STRIDE2:%.*]] = muli [[ARG4]], [[C3]] : index // CHECK: [[INDEX2:%.*]] = addi [[ARG2]], [[STRIDE2]] : index - // CHECK: vector.transfer_read [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}}, [[F1]] {masked = [false]} + // CHECK: vector.transfer_read [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}}, [[F1]] {in_bounds = [true]} %f1 = constant 1.0 : f32 %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : memref<12x32xf32> to memref<4x4xf32, offset:?, strides: [64, 3]> - %1 = vector.transfer_read %0[%arg3, %arg4], %f1 {masked = [false]} : memref<4x4xf32, offset:?, strides: [64, 3]>, vector<4xf32> + %1 = vector.transfer_read %0[%arg3, %arg4], %f1 {in_bounds = [true]} : memref<4x4xf32, offset:?, strides: [64, 3]>, vector<4xf32> return %1 : vector<4xf32> } @@ -91,9 +91,9 @@ // CHECK: [[INDEX1:%.*]] = addi [[ARG1]], [[STRIDE1]] : index // CHECK: [[STRIDE2:%.*]] = muli [[ARG4]], [[C3]] : index // CHECK: [[INDEX2:%.*]] = addi [[ARG2]], [[STRIDE2]] : index - // CHECK: vector.transfer_write [[ARG5]], [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}} {masked = [false]} + // CHECK: vector.transfer_write [[ARG5]], [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}} {in_bounds = [true]} %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : memref<12x32xf32> to memref<4x4xf32, offset:?, strides: [64, 3]> - vector.transfer_write %arg5, %0[%arg3, %arg4] {masked = [false]} : vector<4xf32>, memref<4x4xf32, offset:?, strides: [64, 3]> + vector.transfer_write %arg5, %0[%arg3, %arg4] {in_bounds = [true]} : vector<4xf32>, memref<4x4xf32, offset:?, strides: [64, 3]> return } diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir @@ -1162,13 +1162,13 @@ // ----- -func @transfer_read_1d_not_masked(%A : memref, %base: index) -> vector<17xf32> { +func @transfer_read_1d_inbounds(%A : memref, %base: index) -> vector<17xf32> { %f7 = constant 7.0: f32 - %f = vector.transfer_read %A[%base], %f7 {masked = [false]} : + %f = vector.transfer_read %A[%base], %f7 {in_bounds = [true]} : memref, vector<17xf32> return %f: vector<17xf32> } -// CHECK-LABEL: func @transfer_read_1d_not_masked +// CHECK-LABEL: func @transfer_read_1d_inbounds // CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32> // // 1. Bitcast to vector form. @@ -1184,7 +1184,7 @@ func @transfer_read_1d_cast(%A : memref, %base: index) -> vector<12xi8> { %c0 = constant 0: i32 - %v = vector.transfer_read %A[%base], %c0 {masked = [false]} : + %v = vector.transfer_read %A[%base], %c0 {in_bounds = [true]} : memref, vector<12xi8> return %v: vector<12xi8> } diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir --- a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir +++ b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir @@ -318,15 +318,15 @@ // FULL-UNROLL-DAG: #[[$MAP1:.*]] = affine_map<()[s0] -> (s0 + 1)> // FULL-UNROLL-DAG: #[[$MAP2:.*]] = affine_map<()[s0] -> (s0 + 2)> -// CHECK-LABEL: transfer_write_progressive_unmasked( +// CHECK-LABEL: transfer_write_progressive_inbounds( // CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: memref, // CHECK-SAME: %[[base:[a-zA-Z0-9]+]]: index, // CHECK-SAME: %[[vec:[a-zA-Z0-9]+]]: vector<3x15xf32> -// FULL-UNROLL-LABEL: transfer_write_progressive_unmasked( +// FULL-UNROLL-LABEL: transfer_write_progressive_inbounds( // FULL-UNROLL-SAME: %[[A:[a-zA-Z0-9]+]]: memref, // FULL-UNROLL-SAME: %[[base:[a-zA-Z0-9]+]]: index, // FULL-UNROLL-SAME: %[[vec:[a-zA-Z0-9]+]]: vector<3x15xf32> -func @transfer_write_progressive_unmasked(%A : memref, %base: index, %vec: vector<3x15xf32>) { +func @transfer_write_progressive_inbounds(%A : memref, %base: index, %vec: vector<3x15xf32>) { // CHECK-NOT: scf.if // CHECK-NEXT: %[[alloc:.*]] = memref.alloca() : memref<3xvector<15xf32>> // CHECK-NEXT: %[[vmemref:.*]] = vector.type_cast %[[alloc]] : memref<3xvector<15xf32>> to memref> @@ -334,17 +334,17 @@ // CHECK-NEXT: affine.for %[[I:.*]] = 0 to 3 { // CHECK-NEXT: %[[add:.*]] = affine.apply #[[$MAP0]](%[[I]])[%[[base]]] // CHECK-NEXT: %[[vec_1d:.*]] = memref.load %0[%[[I]]] : memref<3xvector<15xf32>> - // CHECK-NEXT: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] {masked = [false]} : vector<15xf32>, memref + // CHECK-NEXT: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref // FULL-UNROLL: %[[VEC0:.*]] = vector.extract %[[vec]][0] : vector<3x15xf32> - // FULL-UNROLL: vector.transfer_write %[[VEC0]], %[[A]][%[[base]], %[[base]]] {masked = [false]} : vector<15xf32>, memref + // FULL-UNROLL: vector.transfer_write %[[VEC0]], %[[A]][%[[base]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref // FULL-UNROLL: %[[I1:.*]] = affine.apply #[[$MAP1]]()[%[[base]]] // FULL-UNROLL: %[[VEC1:.*]] = vector.extract %[[vec]][1] : vector<3x15xf32> - // FULL-UNROLL: vector.transfer_write %2, %[[A]][%[[I1]], %[[base]]] {masked = [false]} : vector<15xf32>, memref + // FULL-UNROLL: vector.transfer_write %2, %[[A]][%[[I1]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref // FULL-UNROLL: %[[I2:.*]] = affine.apply #[[$MAP2]]()[%[[base]]] // FULL-UNROLL: %[[VEC2:.*]] = vector.extract %[[vec]][2] : vector<3x15xf32> - // FULL-UNROLL: vector.transfer_write %[[VEC2:.*]], %[[A]][%[[I2]], %[[base]]] {masked = [false]} : vector<15xf32>, memref - vector.transfer_write %vec, %A[%base, %base] {masked = [false, false]} : + // FULL-UNROLL: vector.transfer_write %[[VEC2:.*]], %[[A]][%[[I2]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref + vector.transfer_write %vec, %A[%base, %base] {in_bounds = [true, true]} : vector<3x15xf32>, memref return } diff --git a/mlir/test/Dialect/Linalg/forward-vector-transfers.mlir b/mlir/test/Dialect/Linalg/forward-vector-transfers.mlir --- a/mlir/test/Dialect/Linalg/forward-vector-transfers.mlir +++ b/mlir/test/Dialect/Linalg/forward-vector-transfers.mlir @@ -6,14 +6,14 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_read %[[ARG0]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testAllocRead(%in: memref) -> vector<32 x f32> { %c0 = constant 0: index %f0 = constant 0.0: f32 %alloc = memref.alloc() : memref<32 x f32> %subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32> linalg.copy(%in, %subview): memref, memref<16 x f32> - %0 = vector.transfer_read %alloc[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32> + %0 = vector.transfer_read %alloc[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32> memref.dealloc %alloc : memref<32 x f32> return %0: vector<32 x f32> } @@ -24,7 +24,7 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_read %[[ARG0]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testAllocFillRead(%in: memref) -> vector<32 x f32> { %c0 = constant 0: index %f0 = constant 0.0: f32 @@ -32,7 +32,7 @@ linalg.fill(%alloc, %f0): memref<32 x f32>, f32 %subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32> linalg.copy(%in, %subview): memref, memref<16 x f32> - %0 = vector.transfer_read %alloc[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32> + %0 = vector.transfer_read %alloc[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32> memref.dealloc %alloc : memref<32 x f32> return %0: vector<32 x f32> } @@ -43,7 +43,7 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_read %[[ARG0]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testViewRead(%in: memref) -> vector<32 x f32> { %c0 = constant 0: index %f0 = constant 0.0: f32 @@ -51,7 +51,7 @@ %view = memref.view %alloc[%c0][] : memref<128 x i8> to memref<32 x f32> %subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32> linalg.copy(%in, %subview): memref, memref<16 x f32> - %0 = vector.transfer_read %view[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32> + %0 = vector.transfer_read %view[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32> memref.dealloc %alloc : memref<128 x i8> return %0: vector<32 x f32> } @@ -62,7 +62,7 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_read %[[ARG0]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testViewFillRead(%in: memref) -> vector<32 x f32> { %c0 = constant 0: index %f0 = constant 0.0: f32 @@ -71,7 +71,7 @@ %subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32> linalg.fill(%view, %f0): memref<32 x f32>, f32 linalg.copy(%in, %subview): memref, memref<16 x f32> - %0 = vector.transfer_read %view[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32> + %0 = vector.transfer_read %view[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32> memref.dealloc %alloc : memref<128 x i8> return %0: vector<32 x f32> } @@ -82,13 +82,13 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_write %[[ARG0]], %[[ARG1]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testAllocWrite(%vec: vector<32 x f32>, %out: memref) { %c0 = constant 0: index %f0 = constant 0.0: f32 %alloc = memref.alloc() : memref<32 x f32> %subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32> - vector.transfer_write %vec, %alloc[%c0] {masked = [false]} : vector<32 x f32>, memref<32 x f32> + vector.transfer_write %vec, %alloc[%c0] {in_bounds = [true]} : vector<32 x f32>, memref<32 x f32> linalg.copy(%subview, %out): memref<16 x f32>, memref memref.dealloc %alloc : memref<32 x f32> return @@ -100,14 +100,14 @@ // CHECK-NOT: linalg.copy // CHECK: %[[ALLOC:.*]] = memref.alloc // CHECK: vector.transfer_write %[[ARG0]], %[[ARG1]] -// CHECK-NOT: masked +// CHECK-NOT: in_bounds func @testViewWrite(%vec: vector<32 x f32>, %out: memref) { %c0 = constant 0: index %f0 = constant 0.0: f32 %alloc = memref.alloc() : memref<128 x i8> %view = memref.view %alloc[%c0][] : memref<128 x i8> to memref<32 x f32> %subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32> - vector.transfer_write %vec, %view[%c0] {masked = [false]} : vector<32 x f32>, memref<32 x f32> + vector.transfer_write %vec, %view[%c0] {in_bounds = [true]} : vector<32 x f32>, memref<32 x f32> linalg.copy(%subview, %out): memref<16 x f32>, memref memref.dealloc %alloc : memref<128 x i8> return diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir --- a/mlir/test/Dialect/Linalg/vectorization.mlir +++ b/mlir/test/Dialect/Linalg/vectorization.mlir @@ -393,7 +393,7 @@ // a later canonicalization fuses the add into vector.contract. // CHECK: %[[C:.*]] = vector.contract {{.*}} iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %[[V0]], %[[V1]], %[[VEC_C0]] : vector<8x4xf32>, vector<4x12xf32> into vector<8x12xf32> // CHECK: %[[C2:.*]] = addf %[[V2]], %[[C]] : vector<8x12xf32> - // CHECK: %[[W:.*]] = vector.transfer_write %[[C2]], %[[ARG2]][%[[C0]], %[[C0]]] {masked = [false, false]} : vector<8x12xf32>, tensor<8x12xf32> + // CHECK: %[[W:.*]] = vector.transfer_write %[[C2]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x12xf32>, tensor<8x12xf32> %0 = linalg.matmul ins(%arg0, %arg1: tensor<8x4xf32>, tensor<4x12xf32>) outs(%arg2: tensor<8x12xf32>) -> tensor<8x12xf32> @@ -421,7 +421,7 @@ // CHECK: %[[C:.*]] = vector.contract {{.*}} iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %[[V0_32]], %[[V1_32]], %[[VEC_C0]] // CHECK-SAME: vector<4x6xi32>, vector<6x12xi32> into vector<4x12xi32> // CHECK: %[[RES:.*]] = addi %[[V2]], %[[C]] : vector<4x12xi32> - // CHECK: vector.transfer_write %[[RES]], %[[ARG2]][%[[C0]], %[[C0]]] {masked = [false, false]} + // CHECK: vector.transfer_write %[[RES]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} // CHECK-SAME: vector<4x12xi32>, memref<4x12xi32> linalg.matmul_i8_i8_i32 ins(%a, %b : memref<4x6xi8>, memref<6x12xi8>) outs(%c: memref<4x12xi32>) @@ -438,7 +438,7 @@ // CHECK-SAME: : tensor, vector<2x3x4xf32> // CHECK: %[[INIT:.*]] = linalg.init_tensor [2, 3, 4] : tensor<2x3x4xf32> // CHECK: %[[WRITTEN:.*]] = vector.transfer_write %[[READ]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] - // CHECK-SAME: {masked = [false, false, false]} : vector<2x3x4xf32>, tensor<2x3x4xf32> + // CHECK-SAME: {in_bounds = [true, true, true]} : vector<2x3x4xf32>, tensor<2x3x4xf32> %c0 = constant 0 : index %0 = linalg.pad_tensor %arg0 low[0, %c0, 0] high[0, 0, %c0] { ^bb0(%arg1: index, %arg2: index, %arg3: index): diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir --- a/mlir/test/Dialect/Vector/canonicalize.mlir +++ b/mlir/test/Dialect/Vector/canonicalize.mlir @@ -257,10 +257,10 @@ %f0 = constant 0.0 : f32 %0 = memref.cast %A : memref<4x8xf32> to memref - // CHECK: vector.transfer_read %{{.*}} {masked = [false, false]} : memref<4x8xf32>, vector<4x8xf32> + // CHECK: vector.transfer_read %{{.*}} {in_bounds = [true, true]} : memref<4x8xf32>, vector<4x8xf32> %1 = vector.transfer_read %0[%c0, %c0], %f0 : memref, vector<4x8xf32> - // CHECK: vector.transfer_write %{{.*}} {masked = [false, false]} : vector<4x8xf32>, memref<4x8xf32> + // CHECK: vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, memref<4x8xf32> vector.transfer_write %1, %0[%c0, %c0] : vector<4x8xf32>, memref return %1 : vector<4x8xf32> } @@ -273,7 +273,7 @@ %f0 = constant 0.0 : f32 %0 = tensor.cast %A : tensor<4x8xf32> to tensor - // CHECK: vector.transfer_read %{{.*}} {masked = [false, false]} : tensor<4x8xf32>, vector<4x8xf32> + // CHECK: vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<4x8xf32>, vector<4x8xf32> %1 = vector.transfer_read %0[%c0, %c0], %f0 : tensor, vector<4x8xf32> return %1 : vector<4x8xf32> @@ -537,20 +537,20 @@ %c0 = constant 0 : index %f0 = constant 0.0 : f32 - // CHECK: vector.transfer_read %{{.*}} {masked = [true, false]} + // CHECK: vector.transfer_read %{{.*}} {in_bounds = [false, true]} %1 = vector.transfer_read %A[%c0, %c0], %f0 : memref, vector<4x8xf32> - // CHECK: vector.transfer_write %{{.*}} {masked = [true, false]} + // CHECK: vector.transfer_write %{{.*}} {in_bounds = [false, true]} vector.transfer_write %1, %A[%c0, %c0] : vector<4x8xf32>, memref - // Both dims masked, attribute is elided. + // Both dims may be out-of-bounds, attribute is elided. // CHECK: vector.transfer_read %{{.*}} - // CHECK-NOT: masked + // CHECK-NOT: in_bounds %2 = vector.transfer_read %A[%c0, %c0], %f0 : memref, vector<4x9xf32> - // Both dims masked, attribute is elided. + // Both dims may be out-of-bounds, attribute is elided. // CHECK: vector.transfer_write %{{.*}} - // CHECK-NOT: masked + // CHECK-NOT: in_bounds vector.transfer_write %2, %A[%c0, %c0] : vector<4x9xf32>, memref // CHECK: return @@ -780,20 +780,20 @@ { %c0 = constant 0 : index %pad = constant 0.0 : f32 - %v = vector.transfer_read %t0[%c0, %c0, %c0], %pad {masked = [false, false, false]} : + %v = vector.transfer_read %t0[%c0, %c0, %c0], %pad {in_bounds = [true, true, true]} : tensor<2x3x4xf32>, vector<2x3x4xf32> - %r0 = vector.transfer_write %v, %t1[%c0, %c0, %c0] {masked = [false, false, false]} : + %r0 = vector.transfer_write %v, %t1[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<2x3x4xf32>, tensor<2x3x4xf32> %t2 = "test.constant"() { value = dense<6.0> : tensor<2x3x4xf32>} : () -> (tensor<2x3x4xf32>) - %r1 = vector.transfer_write %v, %t2[%c0, %c0, %c0] {masked = [false, false, false]} : + %r1 = vector.transfer_write %v, %t2[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<2x3x4xf32>, tensor<2x3x4xf32> // CHECK-NEXT: some_op_that_may_have_side_effects %t3 = "some_op_that_may_have_side_effects"() : () -> (tensor<2x3x4xf32>) - %r2 = vector.transfer_write %v, %t0[%c0, %c0, %c0] {masked = [false, false, false]} : + %r2 = vector.transfer_write %v, %t0[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<2x3x4xf32>, tensor<2x3x4xf32> // CHECK-NEXT: return %[[T0]], %[[T0]], %[[T0]] diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir --- a/mlir/test/Dialect/Vector/invalid.mlir +++ b/mlir/test/Dialect/Vector/invalid.mlir @@ -363,8 +363,8 @@ %c3 = constant 3 : index %f0 = constant 0.0 : f32 %vf0 = splat %f0 : vector<2x3xf32> - // expected-error@+1 {{ expects the optional masked attr of same rank as permutation_map results: affine_map<(d0, d1) -> (d0, d1)>}} - %0 = vector.transfer_read %arg0[%c3, %c3], %vf0 {masked = [false], permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref>, vector<1x1x2x3xf32> + // expected-error@+1 {{ expects the optional in_bounds attr of same rank as permutation_map results: affine_map<(d0, d1) -> (d0, d1)>}} + %0 = vector.transfer_read %arg0[%c3, %c3], %vf0 {in_bounds = [true], permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref>, vector<1x1x2x3xf32> } // ----- diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir --- a/mlir/test/Dialect/Vector/ops.mlir +++ b/mlir/test/Dialect/Vector/ops.mlir @@ -23,8 +23,8 @@ %3 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d1)>} : memref, vector<128xf32> // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : memref>, vector<1x1x4x3xf32> %4 = vector.transfer_read %arg1[%c3, %c3], %vf0 {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref>, vector<1x1x4x3xf32> - // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {masked = [true, false]} : memref>, vector<1x1x4x3xf32> - %5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {masked = [true, false]} : memref>, vector<1x1x4x3xf32> + // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {in_bounds = [false, true]} : memref>, vector<1x1x4x3xf32> + %5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {in_bounds = [false, true]} : memref>, vector<1x1x4x3xf32> // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : memref>, vector<5x24xi8> %6 = vector.transfer_read %arg2[%c3, %c3], %v0 : memref>, vector<5x24xi8> @@ -36,7 +36,7 @@ // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, memref> vector.transfer_write %4, %arg1[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : vector<1x1x4x3xf32>, memref> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, memref> - vector.transfer_write %5, %arg1[%c3, %c3] {masked = [true, true]} : vector<1x1x4x3xf32>, memref> + vector.transfer_write %5, %arg1[%c3, %c3] {in_bounds = [false, false]} : vector<1x1x4x3xf32>, memref> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<5x24xi8>, memref> vector.transfer_write %6, %arg2[%c3, %c3] : vector<5x24xi8>, memref> @@ -69,8 +69,8 @@ %3 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d1)>} : tensor, vector<128xf32> // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : tensor>, vector<1x1x4x3xf32> %4 = vector.transfer_read %arg1[%c3, %c3], %vf0 {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : tensor>, vector<1x1x4x3xf32> - // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {masked = [true, false]} : tensor>, vector<1x1x4x3xf32> - %5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {masked = [true, false]} : tensor>, vector<1x1x4x3xf32> + // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {in_bounds = [false, true]} : tensor>, vector<1x1x4x3xf32> + %5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {in_bounds = [false, true]} : tensor>, vector<1x1x4x3xf32> // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : tensor>, vector<5x24xi8> %6 = vector.transfer_read %arg2[%c3, %c3], %v0 : tensor>, vector<5x24xi8> @@ -82,7 +82,7 @@ // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, tensor> %9 = vector.transfer_write %4, %arg1[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : vector<1x1x4x3xf32>, tensor> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, tensor> - %10 = vector.transfer_write %5, %arg1[%c3, %c3] {masked = [true, true]} : vector<1x1x4x3xf32>, tensor> + %10 = vector.transfer_write %5, %arg1[%c3, %c3] {in_bounds = [false, false]} : vector<1x1x4x3xf32>, tensor> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<5x24xi8>, tensor> %11 = vector.transfer_write %6, %arg2[%c3, %c3] : vector<5x24xi8>, tensor> diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir --- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir @@ -54,7 +54,7 @@ // CHECK-SAME: memref, index, index // CHECK: } // CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst - // CHECK_SAME: {masked = [false, false]} : memref, vector<4x8xf32> + // CHECK_SAME: {in_bounds = [true, true]} : memref, vector<4x8xf32> // LINALG-DAG: %[[c0:.*]] = constant 0 : index // LINALG-DAG: %[[c4:.*]] = constant 4 : index @@ -88,7 +88,7 @@ // LINALG-SAME: memref, index, index // LINALG: } // LINALG: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst - // LINALG_SAME: {masked = [false, false]} : memref, vector<4x8xf32> + // LINALG_SAME: {in_bounds = [true, true]} : memref, vector<4x8xf32> %1 = vector.transfer_read %A[%i, %j], %f0 : memref, vector<4x8xf32> // LINALG: return %[[res]] : vector<4x8xf32> @@ -142,7 +142,7 @@ // CHECK: scf.yield %[[yielded]], %[[c0]], %[[c0]] : // CHECK-SAME: memref, index, index // CHECK: } - // CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} : + // CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {in_bounds = [true, true]} : // CHECK-SAME: memref, vector<4x8xf32> // LINALG-DAG: %[[c0:.*]] = constant 0 : index @@ -178,7 +178,7 @@ // LINALG: scf.yield %[[yielded]], %[[c0]], %[[c0]] : // LINALG-SAME: memref, index, index // LINALG: } - // LINALG: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} : + // LINALG: %[[res:.*]] = vector.transfer_read {{.*}} {in_bounds = [true, true]} : // LINALG-SAME: memref, vector<4x8xf32> %1 = vector.transfer_read %A[%i, %j], %f0 : memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32> diff --git a/mlir/test/Dialect/Vector/vector-transfer-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-lowering.mlir --- a/mlir/test/Dialect/Vector/vector-transfer-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-lowering.mlir @@ -11,8 +11,8 @@ func @transfer_to_load(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xf32>, vector<4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<4xf32>, memref<8x8xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32> return %res : vector<4xf32> } @@ -29,8 +29,8 @@ func @transfer_2D(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, false]} : memref<8x8xf32>, vector<2x4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [false, false]} : vector<2x4xf32>, memref<8x8xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, true]} : memref<8x8xf32>, vector<2x4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true, true]} : vector<2x4xf32>, memref<8x8xf32> return %res : vector<2x4xf32> } @@ -60,42 +60,43 @@ // CHECK-SAME: %[[MEM:.*]]: memref<8x8xvector<2x4xf32>>, // CHECK-SAME: %[[IDX:.*]]: index) -> vector<1x2x4xf32> { // CHECK-NEXT: %[[CF0:.*]] = constant dense<0.000000e+00> : vector<2x4xf32> -// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32> -// CHECK-NEXT: vector.transfer_write %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>> +// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32> +// CHECK-NEXT: vector.transfer_write %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>> // CHECK-NEXT: return %[[RES]] : vector<1x2x4xf32> // CHECK-NEXT: } func @transfer_vector_element_different_types(%mem : memref<8x8xvector<2x4xf32>>, %i : index) -> vector<1x2x4xf32> { %cf0 = constant dense<0.0> : vector<2x4xf32> - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>> return %res : vector<1x2x4xf32> } // ----- -// TODO: transfer_read/write cannot be lowered because there is an unmasked -// dimension. -// CHECK-LABEL: func @transfer_2D_masked( +// TODO: transfer_read/write cannot be lowered because there is a dimension +// that is not guaranteed to be in-bounds. +// CHECK-LABEL: func @transfer_2D_not_inbounds( // CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>, // CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> { // CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32 -// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false, true]} : memref<8x8xf32>, vector<2x4xf32> -// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [true, false]} : vector<2x4xf32>, memref<8x8xf32> +// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true, false]} : memref<8x8xf32>, vector<2x4xf32> +// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [false, true]} : vector<2x4xf32>, memref<8x8xf32> // CHECK-NEXT: return %[[RES]] : vector<2x4xf32> // CHECK-NEXT: } -func @transfer_2D_masked(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> { +func @transfer_2D_not_inbounds(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, true]} : memref<8x8xf32>, vector<2x4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [true, false]} : vector<2x4xf32>, memref<8x8xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, false]} : memref<8x8xf32>, vector<2x4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [false, true]} : vector<2x4xf32>, memref<8x8xf32> return %res : vector<2x4xf32> } // ----- -// TODO: transfer_read/write cannot be lowered because they are masked. -// CHECK-LABEL: func @transfer_masked( +// TODO: transfer_read/write cannot be lowered because they are not guaranteed +// to be in-bounds. +// CHECK-LABEL: func @transfer_not_inbounds( // CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>, // CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> { // CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32 @@ -104,7 +105,7 @@ // CHECK-NEXT: return %[[RES]] : vector<4xf32> // CHECK-NEXT: } -func @transfer_masked(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> { +func @transfer_not_inbounds(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> { %cf0 = constant 0.0 : f32 %res = vector.transfer_read %mem[%i, %i], %cf0 : memref<8x8xf32>, vector<4xf32> vector.transfer_write %res, %mem[%i, %i] : vector<4xf32>, memref<8x8xf32> @@ -119,16 +120,16 @@ // CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32, #{{.*}}>, // CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> { // CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32 -// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false]} : memref<8x8xf32, #{{.*}}>, vector<4xf32> -// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false]} : vector<4xf32>, memref<8x8xf32, #{{.*}}> +// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true]} : memref<8x8xf32, #{{.*}}>, vector<4xf32> +// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32, #{{.*}}> // CHECK-NEXT: return %[[RES]] : vector<4xf32> // CHECK-NEXT: } #layout = affine_map<(d0, d1) -> (d0*16 + d1)> func @transfer_nondefault_layout(%mem : memref<8x8xf32, #layout>, %i : index) -> vector<4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xf32, #layout>, vector<4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<4xf32>, memref<8x8xf32, #layout> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xf32, #layout>, vector<4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32, #layout> return %res : vector<4xf32> } @@ -140,15 +141,15 @@ // CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>, // CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> { // CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32 -// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false], permutation_map = #{{.*}}} : memref<8x8xf32>, vector<4xf32> -// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false], permutation_map = #{{.*}}} : vector<4xf32>, memref<8x8xf32> +// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true], permutation_map = #{{.*}}} : memref<8x8xf32>, vector<4xf32> +// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true], permutation_map = #{{.*}}} : vector<4xf32>, memref<8x8xf32> // CHECK-NEXT: return %[[RES]] : vector<4xf32> // CHECK-NEXT: } func @transfer_perm_map(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false], permutation_map = affine_map<(d0, d1) -> (d0)>} : memref<8x8xf32>, vector<4xf32> - vector.transfer_write %res, %mem[%i, %i] {masked = [false], permutation_map = affine_map<(d0, d1) -> (d0)>} : vector<4xf32>, memref<8x8xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (d0)>} : memref<8x8xf32>, vector<4xf32> + vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (d0)>} : vector<4xf32>, memref<8x8xf32> return %res : vector<4xf32> } @@ -167,7 +168,7 @@ #broadcast = affine_map<(d0, d1) -> (0)> func @transfer_broadcasting(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false], permutation_map = #broadcast} : memref<8x8xf32>, vector<4xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true], permutation_map = #broadcast} : memref<8x8xf32>, vector<4xf32> return %res : vector<4xf32> } @@ -185,7 +186,7 @@ #broadcast = affine_map<(d0, d1) -> (0, 0)> func @transfer_broadcasting_2D(%mem : memref<8x8xf32>, %i : index) -> vector<4x4xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, false], permutation_map = #broadcast} : memref<8x8xf32>, vector<4x4xf32> + %res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, true], permutation_map = #broadcast} : memref<8x8xf32>, vector<4x4xf32> return %res : vector<4x4xf32> } @@ -203,7 +204,7 @@ #broadcast = affine_map<(d0, d1, d2, d3, d4) -> (d1, 0, 0, d4)> func @transfer_broadcasting_complex(%mem : memref<10x20x30x8x8xf32>, %i : index) -> vector<3x2x4x5xf32> { %cf0 = constant 0.0 : f32 - %res = vector.transfer_read %mem[%i, %i, %i, %i, %i], %cf0 {masked = [false, false, false, false], permutation_map = #broadcast} : memref<10x20x30x8x8xf32>, vector<3x2x4x5xf32> + %res = vector.transfer_read %mem[%i, %i, %i, %i, %i], %cf0 {in_bounds = [true, true, true, true], permutation_map = #broadcast} : memref<10x20x30x8x8xf32>, vector<3x2x4x5xf32> return %res : vector<3x2x4x5xf32> } @@ -236,8 +237,8 @@ // CHECK: vector.transfer_read {{.*}} {permutation_map = #[[$MAP0]]} : memref, vector<16x14x7x8xf32> // CHECK: vector.transpose %{{.*}}, [2, 1, 3, 0] : vector<16x14x7x8xf32> to vector<7x14x8x16xf32> - %2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %cst {masked = [false, false, true, false], permutation_map = #map2} : memref, vector<7x14x8x16xf32> -// CHECK: vector.transfer_read {{.*}} {masked = [false, true, false], permutation_map = #[[$MAP1]]} : memref, vector<14x16x7xf32> + %2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, false, true], permutation_map = #map2} : memref, vector<7x14x8x16xf32> +// CHECK: vector.transfer_read {{.*}} {in_bounds = [true, false, true], permutation_map = #[[$MAP1]]} : memref, vector<14x16x7xf32> // CHECK: vector.broadcast %{{.*}} : vector<14x16x7xf32> to vector<8x14x16x7xf32> // CHECK: vector.transpose %{{.*}}, [3, 1, 0, 2] : vector<8x14x16x7xf32> to vector<7x14x8x16xf32> diff --git a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir --- a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir +++ b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir @@ -13,16 +13,16 @@ %c4 = constant 4 : index %c0 = constant 0 : index %cf0 = constant 0.0 : f32 - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> - %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<1x4xf32>) { %1 = addf %acc, %acc : vector<1x4xf32> scf.yield %1 : vector<1x4xf32> } - vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> return } @@ -40,18 +40,18 @@ %c0 = constant 0 : index %c1 = constant 1 : index %cf0 = constant 0.0 : f32 - vector.transfer_write %v1, %arg1[%i, %c0] {masked = [false, false]} : + vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> %x = scf.if %arg0 -> (vector<1x4xf32>) { - %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> scf.yield %0 : vector<1x4xf32> } else { scf.yield %v1 : vector<1x4xf32> } - vector.transfer_write %x, %arg1[%c0, %c0] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c0, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> return } @@ -73,18 +73,18 @@ %c0 = constant 0 : index %c1 = constant 1 : index %cf0 = constant 0.0 : f32 - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> %x = scf.if %arg0 -> (vector<1x4xf32>) { - %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> scf.yield %0 : vector<1x4xf32> } else { - vector.transfer_write %v1, %arg1[%i, %c0] {masked = [false, false]} : + vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> scf.yield %v1 : vector<1x4xf32> } - vector.transfer_write %x, %arg1[%c0, %i] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c0, %i] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> return } @@ -108,24 +108,24 @@ %c0 = constant 0 : index %c1 = constant 1 : index %cf0 = constant 0.0 : f32 - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> %x = scf.if %arg0 -> (vector<1x4xf32>) { scf.yield %v1 : vector<1x4xf32> } else { - %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> scf.yield %0 : vector<1x4xf32> } scf.if %arg0 { - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> } - vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> - vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> - %1 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} : + %1 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> return %1 : vector<1x4xf32> } @@ -144,15 +144,15 @@ %c1 = constant 1 : index %cf0 = constant 0.0 : f32 %x = scf.if %arg0 -> (vector<1x4xf32>) { - vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> - %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> scf.yield %0 : vector<1x4xf32> } else { scf.yield %v1 : vector<1x4xf32> } - vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> return } @@ -172,13 +172,13 @@ %c1 = constant 1 : index %cf0 = constant 0.0 : f32 scf.if %arg0 { - %0 = vector.transfer_read %arg2[%i, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %arg2[%i, %c0], %cf0 {in_bounds = [true, true]} : memref<4x4xf32>, vector<1x4xf32> scf.if %arg1 { - vector.transfer_write %v1, %arg2[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v1, %arg2[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> } - vector.transfer_write %v0, %arg2[%c1, %c0] {masked = [false, false]} : + vector.transfer_write %v0, %arg2[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, memref<4x4xf32> } return @@ -197,16 +197,16 @@ %c4 = constant 4 : index %c0 = constant 0 : index %cf0 = constant 0.0 : f32 - %w0 = vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} : + %w0 = vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<4x4xf32> - %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]} : tensor<4x4xf32>, vector<1x4xf32> %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<1x4xf32>) { %1 = addf %acc, %acc : vector<1x4xf32> scf.yield %1 : vector<1x4xf32> } - %w1 = vector.transfer_write %x, %w0[%c1, %c0] {masked = [false, false]} : + %w1 = vector.transfer_write %x, %w0[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<4x4xf32> return %w1 : tensor<4x4xf32> } @@ -224,16 +224,16 @@ %c4 = constant 4 : index %c0 = constant 0 : index %cf0 = constant 0.0 : f32 - %w0 = vector.transfer_write %v0, %arg1[%c1, %i] {masked = [false, false]} : + %w0 = vector.transfer_write %v0, %arg1[%c1, %i] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<4x4xf32> - %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {masked = [false, false]} : + %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]} : tensor<4x4xf32>, vector<1x4xf32> %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<1x4xf32>) { %1 = addf %acc, %acc : vector<1x4xf32> scf.yield %1 : vector<1x4xf32> } - %w1 = vector.transfer_write %x, %w0[%c1, %c0] {masked = [false, false]} : + %w1 = vector.transfer_write %x, %w0[%c1, %c0] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<4x4xf32> return %w1 : tensor<4x4xf32> } diff --git a/mlir/test/Dialect/Vector/vector-transforms.mlir b/mlir/test/Dialect/Vector/vector-transforms.mlir --- a/mlir/test/Dialect/Vector/vector-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-transforms.mlir @@ -246,10 +246,10 @@ // CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> // CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> -// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32> -// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32> +// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32> // CHECK-NEXT: return func @contraction4x4_ikj_xfer_read(%arg0 : memref<4x2xf32>, @@ -424,10 +424,10 @@ // CHECK-LABEL: func @vector_transfers_vector_element_type // CHECK-DAG: %[[C1:.*]] = constant 1 : index // CHECK-DAG: %[[C0:.*]] = constant 0 : index -// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> -// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> -// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> -// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> +// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {in_bounds = [true, true]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> +// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {in_bounds = [true, true]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32> +// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> +// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {in_bounds = [true, true]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>> func @vector_transfers_vector_element_type() { %c0 = constant 0 : index @@ -577,10 +577,10 @@ // CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> // CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32> -// CHECK-NEXT: %[[VTW0:.*]] = vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32> -// CHECK-NEXT: %[[VTW1:.*]] = vector.transfer_write %[[R1]], %[[VTW0]][%[[C0]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32> -// CHECK-NEXT: %[[VTW2:.*]] = vector.transfer_write %[[R2]], %[[VTW1]][%[[C2]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32> -// CHECK-NEXT: %[[VTW3:.*]] = vector.transfer_write %[[R3]], %[[VTW2]][%[[C2]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32> +// CHECK-NEXT: %[[VTW0:.*]] = vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32> +// CHECK-NEXT: %[[VTW1:.*]] = vector.transfer_write %[[R1]], %[[VTW0]][%[[C0]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32> +// CHECK-NEXT: %[[VTW2:.*]] = vector.transfer_write %[[R2]], %[[VTW1]][%[[C2]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32> +// CHECK-NEXT: %[[VTW3:.*]] = vector.transfer_write %[[R3]], %[[VTW2]][%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32> // CHECK-NEXT: return %[[VTW3]] : tensor<4x4xf32> func @contraction4x4_ikj_xfer_read_tensor(%arg0 : tensor<4x2xf32>, @@ -637,9 +637,9 @@ %c0 = constant 0 : index // CHECK: %[[F0:.+]] = constant 0.000000e+00 : f16 %f0 = constant 0. : f16 - // CHECK: %[[READ:.+]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[F0]] {masked = [false]} : memref<1x4x8x16xf16>, vector<4xf16> + // CHECK: %[[READ:.+]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[F0]] {in_bounds = [true]} : memref<1x4x8x16xf16>, vector<4xf16> // CHECK: %[[CAST:.+]] = vector.shape_cast %[[READ]] : vector<4xf16> to vector<1x4xf16> - %0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {masked = [false, false]} : memref<1x4x8x16xf16>, vector<1x4xf16> + %0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {in_bounds = [true, true]} : memref<1x4x8x16xf16>, vector<1x4xf16> // CHECK: return %[[CAST]] return %0: vector<1x4xf16> } @@ -649,7 +649,7 @@ %c0 = constant 0 : index %f0 = constant 0. : f16 // CHECK: vector.shape_cast %{{.+}} : vector<1xf16> to vector<1x1xf16> - %0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {masked = [false, false]} : memref<1x1x1x1xf16>, vector<1x1xf16> + %0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {in_bounds = [true, true]} : memref<1x1x1x1xf16>, vector<1x1xf16> return %0: vector<1x1xf16> } @@ -658,9 +658,9 @@ // CHECK: %[[C0:.+]] = constant 0 : index %c0 = constant 0 : index // CHECK: %[[CAST:.+]] = vector.shape_cast %{{.*}} : vector<1x4xf16> to vector<4xf16> - // CHECK: vector.transfer_write %[[CAST]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {masked = [false]} : vector<4xf16>, memref<1x4x8x16xf16> + // CHECK: vector.transfer_write %[[CAST]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true]} : vector<4xf16>, memref<1x4x8x16xf16> - vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {masked = [false, false]} : vector<1x4xf16>, memref<1x4x8x16xf16> + vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true]} : vector<1x4xf16>, memref<1x4x8x16xf16> return } @@ -668,7 +668,7 @@ func @cast_away_transfer_write_leading_one_dims_one_element(%arg0: memref<1x1x1x1xf16>, %arg1: vector<1x1xf16>) { %c0 = constant 0 : index // CHECK: vector.shape_cast %{{.+}} : vector<1x1xf16> to vector<1xf16> - vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {masked = [false, false]} : vector<1x1xf16>, memref<1x1x1x1xf16> + vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true]} : vector<1x1xf16>, memref<1x1x1x1xf16> return } diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir @@ -12,10 +12,10 @@ return } -func @transfer_read_unmasked_4(%A : memref, %base: index) { +func @transfer_read_inbounds_4(%A : memref, %base: index) { %fm42 = constant -42.0: f32 %f = vector.transfer_read %A[%base], %fm42 - {permutation_map = affine_map<(d0) -> (d0)>, masked = [false]} : + {permutation_map = affine_map<(d0) -> (d0)>, in_bounds = [true]} : memref, vector<4xf32> vector.print %f: vector<4xf32> return @@ -53,9 +53,9 @@ // Read shifted by 0 and pad with -42: // ( 0, 1, 2, 0, 0, -42, ..., -42) call @transfer_read_1d(%A, %c0) : (memref, index) -> () - // Read unmasked 4 @ 1, guaranteed to not overflow. + // Read in-bounds 4 @ 1, guaranteed to not overflow. // Exercises proper alignment. - call @transfer_read_unmasked_4(%A, %c1) : (memref, index) -> () + call @transfer_read_inbounds_4(%A, %c1) : (memref, index) -> () return } diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir @@ -3,11 +3,11 @@ // RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -func @transfer_write16_unmasked_1d(%A : memref, %base: index) { +func @transfer_write16_inbounds_1d(%A : memref, %base: index) { %f = constant 16.0 : f32 %v = splat %f : vector<16xf32> vector.transfer_write %v, %A[%base] - {permutation_map = affine_map<(d0) -> (d0)>, masked = [false]} + {permutation_map = affine_map<(d0) -> (d0)>, in_bounds = [true]} : vector<16xf32>, memref return } @@ -54,9 +54,9 @@ vector.print %0 : vector<32xf32> // Overwrite with 16 values of 16 at base 3. - // Statically guaranteed to be unmasked. Exercises proper alignment. + // Statically guaranteed to be in-bounds. Exercises proper alignment. %c3 = constant 3: index - call @transfer_write16_unmasked_1d(%A, %c3) : (memref, index) -> () + call @transfer_write16_inbounds_1d(%A, %c3) : (memref, index) -> () %1 = call @transfer_read_1d(%A) : (memref) -> (vector<32xf32>) vector.print %1 : vector<32xf32>