diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h @@ -25,6 +25,74 @@ class AnalysisState; class BufferizableOpInterface; +/// Specifies a fine-grain relationship between buffers to enable more analysis. +enum class BufferRelation { + Unknown, + // TODO: ResultContainsOperand, + // TODO: OperandContainsResult, + Equivalent +}; + +/// A maybe aliasing OpOperand. If `isDefinite` is `true`, the OpOperand is +/// guaranteed to alias at runtime. +struct AliasingOpOperand { + AliasingOpOperand(OpOperand *opOperand, BufferRelation relation, + bool isDefinite = true) + : opOperand(opOperand), relation(relation), isDefinite(isDefinite) {} + + OpOperand *opOperand; + BufferRelation relation; + bool isDefinite; +}; + +/// A maybe aliasing OpResult. If `isDefinite` is `true`, the OpResult is +/// guaranteed to alias at runtime. +struct AliasingOpResult { + AliasingOpResult(OpResult opResult, BufferRelation relation, + bool isDefinite = true) + : opResult(opResult), relation(relation), isDefinite(isDefinite) {} + + OpResult opResult; + BufferRelation relation; + bool isDefinite; +}; + +template class AliasList { +public: + /// Create an empty list of aliases. + AliasList() = default; + + /// Create a list of aliases. + AliasList(std::initializer_list elems) { + for (T alias : elems) + addAlias(alias); + } + + /// Create a list of aliases. + AliasList(SmallVector &&aliases) : aliases(std::move(aliases)) {} + + ArrayRef getAliases() const { return aliases; } + + size_t getNumAliases() const { return aliases.size(); } + + void addAlias(T alias) { aliases.push_back(alias); } + + auto begin() const { return aliases.begin(); } + auto end() const { return aliases.end(); } + +private: + /// The list of aliases. + SmallVector aliases; +}; + +/// A list of possible aliasing OpOperands. This list models the runtime +/// aliasing relationship for an OpResult. +using AliasingOpOperandList = AliasList; + +/// A list of possible aliasing OpResults. This list models the runtime +/// aliasing relationship for an OpOperand. +using AliasingOpResultList = AliasList; + class OpFilter { public: /// An op filter entry. Filters can be used to specify which ops should be @@ -300,14 +368,6 @@ SmallVector stateInitializers; }; -/// Specify fine-grain relationship between buffers to enable more analysis. -enum class BufferRelation { - Unknown, - // TODO: ResultContainsOperand, - // TODO: OperandContainsResult, - Equivalent -}; - /// Return `true` if the given value is a BlockArgument of a func::FuncOp. bool isFunctionArgument(Value value); @@ -315,15 +375,15 @@ /// tensor values. class AnalysisState { public: - /// Determine which OpOperand* will alias with `opResult` if the op is + /// Determine which OpOperand* will alias with `result` if the op is /// bufferized in place. Return all tensor OpOperand* if the op is not /// bufferizable. - SmallVector getAliasingOpOperand(OpResult opResult) const; + AliasingOpOperandList getAliasingOpOperands(OpResult result) const; /// Determine which OpResult will alias with `opOperand` if the op is /// bufferized in place. Return all tensor OpResults if the op is not /// bufferizable. - SmallVector getAliasingOpResult(OpOperand &opOperand) const; + AliasingOpResultList getAliasingOpResults(OpOperand &opOperand) const; /// Return true if `opOperand` bufferizes to a memory read. Return `true` if /// the op is not bufferizable. @@ -566,6 +626,12 @@ const BufferizationOptions &options); namespace detail { +/// This is the default implementation of +/// BufferizableOpInterface::getAliasingOpOperands. Should not be called from +/// other places. +AliasingOpOperandList defaultGetAliasingOpOperands(OpResult opResult, + const AnalysisState &state); + /// This is the default implementation of /// BufferizableOpInterface::getBufferType. Should not be called from other /// places. @@ -585,13 +651,13 @@ bool defaultIsRepetitiveRegion(BufferizableOpInterface bufferizableOp, unsigned index); -/// This is the default implementation of getAliasingOpOperand in case the -/// defining op does not implement the BufferizableOpInterface. -SmallVector unknownGetAliasingOpOperand(OpResult opResult); +/// This is the default implementation of getAliasingOpOperands in case the +/// owner op does not implement the BufferizableOpInterface. +AliasingOpOperandList unknownGetAliasingOpOperands(OpResult opResult); -/// This is the default implementation of getAliasingOpResult in case the +/// This is the default implementation of getAliasingOpResults in case the /// defining op does not implement the BufferizableOpInterface. -SmallVector unknownGetAliasingOpResult(OpOperand &opOperand); +AliasingOpResultList unknownGetAliasingOpResults(OpOperand &opOperand); } // namespace detail } // namespace bufferization diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td @@ -172,22 +172,71 @@ >, InterfaceMethod< /*desc=*/[{ - Return the OpResult that aliases with a given OpOperand when + Return the OpResults that may alias with a given OpOperand when bufferized in-place. This method will never be called on OpOperands that do not have a tensor type. - Note: This method can return multiple OpResults, indicating that a - given OpOperand may at runtime alias with any (or multiple) of the - returned OpResults. + This method can return multiple OpResults, indicating that a given + OpOperand may at runtime alias with any (or multiple) of the returned + OpResults. + + Each alias is specified with a degree of certainty: + + * MAYBE (`isDefinite = false`): At runtime, buffer(opOperand) may + alias with the specified OpResult. + * DEFINITE (`isDefinite = true`, default): At runtime, + buffer(opOperand) is guaranteed to alias the buffer of the specified + OpResult. This is a stronger property than MAYBE and allows for more + precise analyses. DEFINITE properties should be used when possible. + + Furthermore, each alias is specified with a buffer relation: + + * `BufferRelation::Equivalent`: Both aliases are the exact same + buffer. I.e., same size, no offset, same strides. + * `BufferRelation::Unknown`: There is no further information apart + from the fact that both buffers alias. + + False positives are allowed in the list of OpResults, but they can + adversely affect the accuracy of the anlysis. On the contrary, + omitting potential aliases is incorrect. + + One possible (conservative) implementation of this interface method, + that is always safe, is to return all tensor OpResults with + BufferRelation::Unknown and MAYBE. + + Examples: + + ``` + // aliasingOpResults(%t) = DEFINITE {Equivalent %r} + %r = tensor.insert_slice %f into %t : tensor<10xf32> + + // aliasingOpResults(%t) = DEFINITE {Unknown %r} + // Note: "Buffer is subset of buffer" relationship are not yet + // supported, so "Unknown" is the best we can do for now. + %r = tensor.extract_slice %t[0]][5][1] + : tensor<10xf32> to tensor<5xf32> + + // aliasingOpResults(%t1) = MAYBE {Equivalent %r} + // aliasingOpResults(%t2) = MAYBE {Equivalent %r} + %r = arith.select %c, %t1, %t2 : tensor<10xf32> + + // A hypothetical op that bufferizes to rolling a dice and based on + // the result to either return buffer(%t) or a newly allocated copy + // thereof. + // aliasingOpResults(%t) = MAYBE {Equivalent %r} + %r = "dummy.alias_or_copy(%t) : (tensor<10xf32>) -> (tensor<10xf32>)" + ``` }], - /*retType=*/"::mlir::SmallVector<::mlir::OpResult>", - /*methodName=*/"getAliasingOpResult", + /*retType=*/"::mlir::bufferization::AliasingOpResultList", + /*methodName=*/"getAliasingOpResults", /*args=*/(ins "::mlir::OpOperand &":$opOperand, "const ::mlir::bufferization::AnalysisState &":$state), /*methodBody=*/"", /*defaultImplementation=*/[{ // Does not have to be implemented for ops without tensor OpOperands. - llvm_unreachable("getAliasingOpResult not implemented"); + assert(opOperand.get().getType().isa() && + "expected OpOperand with tensor type"); + llvm_unreachable("getAliasingOpResults not implemented"); }] >, InterfaceMethod< @@ -196,57 +245,70 @@ bufferized in-place. This method will never be called on OpResults that do not have a tensor type. - By default, this method is the inverse of `getAliasingOpResult`. Ops + By default, this method is the inverse of `getAliasingOpResults`. Ops with a region that yield values may want to override this method to return the OpOperands that are yielded by the terminator. - Note: This method can return multiple OpOperands, indicating that the - given OpResult may at runtime alias with any (or multiple) of the - returned OpOperands. This can be useful for branches and for ops such - as `arith.select`. + This method can return multiple OpOperands, indicating that a given + OpResult may at runtime alias with any (or multiple) of the returned + OpOperands. + + This property is specified with a degree of certainty: + + * MAYBE (`isDefinite = false`): At runtime, buffer(opResult) may + alias with the specified OpOperand. + * DEFINITE (`isDefinite = true`, default): At runtime, + buffer(opResult) is guaranteed to alias the buffer of the specified + OpOperand. This is a stronger property than MAYBE and allows for + more precise analyses. DEFINITE properties should be used when + possible. + + For each alias, a BufferRelation can be specified: + + * `BufferRelation::Equivalent`: Both aliases are the exact same + buffer. I.e., same size, no offset, same strides. + * `BufferRelation::Unknown`: There is no further information apart + from the fact that both buffers alias. + + False positives are allowed in the list of OpOperands, but they can + adversely affect the accuracy of the anlysis. On the contrary, + omitting potential aliases is incorrect. + + One possible (conservative) implementation of this interface method, + that is always safe, is to return all tensor OpOperands with + BufferRelation::Unknown and MAYBE. + + Note: If this property is MAYBE-ALLOC and the list of OpOperands is + empty, this op definitely bufferizes to a new allocation. In fact, if + the list of OpOperands is empty, this property must be MAYBE-ALLOC. + + Examples: + + ``` + // aliasingOpOperands(%r) = DEFINITE {Equivalent %t} + %r = tensor.insert_slice %f into %t : tensor<10xf32> + + // aliasingOpOperands(%r) = DEFINITE {Unknown %t} + %r = tensor.extract_slice %t[0]][5][1] + : tensor<10xf32> to tensor<5xf32> + + // aliasingOpOperands(%r) = DEFINITE {Equivalent %t1, Equivalent %t2} + %r = arith.select %c, %t1, %t2 : tensor<10xf32> + + // aliasingOpOperands(%r) = MAYBE-ALLOC {} + %r = tensor.empty() : tensor<10xf32> + ``` }], - /*retType=*/"::mlir::SmallVector<::mlir::OpOperand *>", - /*methodName=*/"getAliasingOpOperand", + /*retType=*/"::mlir::bufferization::AliasingOpOperandList", + /*methodName=*/"getAliasingOpOperands", /*args=*/(ins "::mlir::OpResult":$opResult, "const ::mlir::bufferization::AnalysisState &":$state), /*methodBody=*/"", /*defaultImplementation=*/[{ assert(opResult.getType().isa() && "expected OpResult with tensor type"); - SmallVector result; - auto bufferizableOp = - cast($_op.getOperation()); - for (OpOperand &opOperand : $_op.getOperation()->getOpOperands()) { - if (!opOperand.get().getType().isa()) - continue; - SmallVector aliasingOpResults = - bufferizableOp.getAliasingOpResult(opOperand, state); - if (llvm::is_contained(aliasingOpResults, opResult)) - result.push_back(&opOperand); - } - return result; - }] - >, - InterfaceMethod< - /*desc=*/[{ - Return the buffer relation between the given OpResult and its aliasing - OpOperands when bufferized in-place. Most OpOperands have an - "equivalence" relation. This method will never be called on OpResults - that do not have a tensor type. It will also never be called on - OpResults that do not have at least one aliasing OpOperand. - - TODO: Support other relations such as "OpOperand is included in - OpResult". - }], - /*retType=*/"::mlir::bufferization::BufferRelation", - /*methodName=*/"bufferRelation", - /*args=*/(ins "::mlir::OpResult":$opResult, - "const ::mlir::bufferization::AnalysisState &":$state), - /*methodBody=*/"", - /*defaultImplementation=*/[{ - // Does not have to be implemented for ops without tensor OpResults - // that have an aliasing OpOperand. - llvm_unreachable("bufferRelation not implemented"); + return ::mlir::bufferization::detail::defaultGetAliasingOpOperands( + opResult, state); }] >, InterfaceMethod< @@ -285,10 +347,11 @@ conversion. The implementation of this method must be consistent with the - remaining methods, in particular `getAliasingOpOperand`. I.e., a + remaining methods, in particular `getAliasingOpOperands`. I.e., a tensor result `r` may only be replaced with: - a) A buffer that aliases one of buffers in getAliasingOpOperand(r). - b) Or: A newly allocated buffer. + + a) One of the buffers in getAliasingOpOperands(r). + b) Or: A newly allocated buffer (only if MAYBE-ALLOC). This method will never be called on ops that do not have at least one tensor operand/result. @@ -440,7 +503,8 @@ cast(getOperation()); return !bufferizableOp.bufferizesToMemoryRead(opOperand, state) && !bufferizableOp.bufferizesToMemoryWrite(opOperand, state) - && !bufferizableOp.getAliasingOpResult(opOperand, state).empty(); + && bufferizableOp.getAliasingOpResults(opOperand, state) + .getNumAliases() != 0; } }]; } diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td @@ -100,7 +100,7 @@ bool bufferizesToMemoryWrite(OpOperand &opOperand, const AnalysisState &state); - SmallVector getAliasingOpResult( + AliasingOpResultList getAliasingOpResults( OpOperand &opOperand, const AnalysisState &state); FailureOr getBufferType( @@ -248,7 +248,7 @@ return false; } - SmallVector getAliasingOpResult( + AliasingOpResultList getAliasingOpResults( OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -409,7 +409,7 @@ return true; } - SmallVector getAliasingOpResult( + AliasingOpResultList getAliasingOpResults( OpOperand &opOperand, const AnalysisState &state) const { return {}; } diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h --- a/mlir/include/mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h +++ b/mlir/include/mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h @@ -36,21 +36,14 @@ return dstOp.isDpsInit(&opOperand); } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { // Output operands alias with their respective tied OpResults. auto dstOp = cast(op); if (dstOp.isDpsInit(&opOperand)) - return {dstOp.getTiedOpResult(&opOperand)}; + return {{dstOp.getTiedOpResult(&opOperand), BufferRelation::Equivalent}}; return {}; } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - assert(isa(op) && - "expected that op implements DestinationStyleOpInterface"); - return BufferRelation::Equivalent; - } }; } // namespace bufferization diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp @@ -74,14 +74,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {op->getResult(0)}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{op->getResult(0), BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -126,16 +121,10 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {op->getOpResult(0) /*result*/}; - } - - SmallVector - getAliasingOpOperand(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return {&op->getOpOperand(1) /*true_value*/, - &op->getOpOperand(2) /*false_value*/}; + return {{op->getOpResult(0) /*result*/, BufferRelation::Equivalent, + /*isDefinite=*/false}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -199,11 +188,6 @@ memrefType.getElementType()), memrefType.getMemorySpace()); } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Unknown; - } }; } // namespace diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp --- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp +++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp @@ -181,31 +181,35 @@ if (operandType.isa()) return op->emitError("copying of unranked tensors is not implemented"); - SmallVector aliasingOpResults = - state.getAliasingOpResult(opOperand); + AliasingOpResultList aliasingOpResults = + state.getAliasingOpResults(opOperand); // Is the result yielded from a block? Or are deallocations turned off // entirely? In either case, mark the allocation as "escaping", so that it // will not be deallocated. bool escape = !state.getOptions().createDeallocs || - llvm::any_of(aliasingOpResults, [&](Value v) { - return state.isTensorYielded(v); + llvm::any_of(aliasingOpResults, [&](AliasingOpResult a) { + return state.isTensorYielded(a.opResult); }); - if (aliasingOpResults.size() == 1 && + if (aliasingOpResults.getNumAliases() == 1 && !state.bufferizesToMemoryWrite(opOperand) && - state.getAliasingOpOperand(aliasingOpResults.front()).size() == 1 && - !aliasingOpResults.front().getType().isa()) { + state.getAliasingOpOperands(aliasingOpResults.getAliases()[0].opResult) + .getNumAliases() == 1 && + !aliasingOpResults.getAliases()[0] + .opResult.getType() + .isa()) { // The op itself does not write but may create exactly one alias. Instead // of copying the OpOperand, copy the OpResult. The OpResult can sometimes // be smaller than the OpOperand (e.g., in the case of an extract_slice, // where the result is usually a smaller part of the source). Do not apply // this optimization if the OpResult is an unranked tensor (because those // cannot be copied at the moment). - outOfPlaceOpResults.push_back(aliasingOpResults.front()); + OpResult opResult = aliasingOpResults.getAliases()[0].opResult; + outOfPlaceOpResults.push_back(opResult); if (!state.canOmitTensorCopy(opOperand)) - copiedOpResults.insert(aliasingOpResults.front()); + copiedOpResults.insert(opResult); if (escape) - escapingOpResultCopies.insert(aliasingOpResults.front()); + escapingOpResultCopies.insert(opResult); } else { // In all other cases, make a copy of the OpOperand. outOfPlaceOpOperands.push_back(&opOperand); @@ -359,26 +363,26 @@ /// Determine which OpOperand* will alias with `opResult` if the op is /// bufferized in place. Return all tensor OpOperand* if the op is not /// bufferizable. -SmallVector -AnalysisState::getAliasingOpOperand(OpResult opResult) const { +AliasingOpOperandList +AnalysisState::getAliasingOpOperands(OpResult opResult) const { if (Operation *op = opResult.getDefiningOp()) if (auto bufferizableOp = getOptions().dynCastBufferizableOp(op)) - return bufferizableOp.getAliasingOpOperand(opResult, *this); + return bufferizableOp.getAliasingOpOperands(opResult, *this); // The op is not bufferizable. - return detail::unknownGetAliasingOpOperand(opResult); + return detail::unknownGetAliasingOpOperands(opResult); } /// Determine which OpResult will alias with `opOperand` if the op is bufferized /// in place. Return all tensor OpResults if the op is not bufferizable. -SmallVector -AnalysisState::getAliasingOpResult(OpOperand &opOperand) const { +AliasingOpResultList +AnalysisState::getAliasingOpResults(OpOperand &opOperand) const { if (auto bufferizableOp = getOptions().dynCastBufferizableOp(opOperand.getOwner())) - return bufferizableOp.getAliasingOpResult(opOperand, *this); + return bufferizableOp.getAliasingOpResults(opOperand, *this); // The op is not bufferizable. - return detail::unknownGetAliasingOpResult(opOperand); + return detail::unknownGetAliasingOpResults(opOperand); } /// Return true if `opOperand` bufferizes to a memory read. Return `true` if the @@ -440,8 +444,8 @@ OpOperand *uMaybeReading = workingSet.pop_back_val(); // Skip over all ops that neither read nor write (but create an alias). if (bufferizesToAliasOnly(*uMaybeReading)) - for (OpResult opResult : getAliasingOpResult(*uMaybeReading)) - for (OpOperand &use : opResult.getUses()) + for (AliasingOpResult alias : getAliasingOpResults(*uMaybeReading)) + for (OpOperand &use : alias.opResult.getUses()) workingSet.push_back(&use); if (bufferizesToMemoryRead(*uMaybeReading)) return true; @@ -470,24 +474,26 @@ OpResult opResult = value.cast(); BufferizableOpInterface bufferizableOp = options.dynCastBufferizableOp(opResult.getDefiningOp()); - SmallVector opOperands = getAliasingOpOperand(opResult); + AliasingOpOperandList aliases = getAliasingOpOperands(opResult); // Stop iterating in either one of these cases: // * The current op is not bufferizable or excluded in the filter. // * There are no OpOperands to follow. - // * There is an OpOperand, but it is not an equivalent tensor (only if - // `followEquivalentOnly` is set). - if (!bufferizableOp || opOperands.empty() || - (followEquivalentOnly && - bufferizableOp.bufferRelation(opResult, *this) != - BufferRelation::Equivalent)) { + if (!bufferizableOp || aliases.getNumAliases() == 0) { if (alwaysIncludeLeaves) result.insert(value); continue; } - for (OpOperand *o : opOperands) - workingSet.insert(o->get()); + for (AliasingOpOperand a : aliases) { + if (followEquivalentOnly && a.relation != BufferRelation::Equivalent) { + // Stop iterating if `followEquivalentOnly` is set but the alias is not + // equivalent. + result.insert(value); + } else { + workingSet.insert(a.opOperand->get()); + } + } } return result; @@ -522,10 +528,10 @@ return true; // Do not copy if the tensor is never read. - SmallVector aliasingOpResults = getAliasingOpResult(opOperand); + AliasingOpResultList aliases = getAliasingOpResults(opOperand); if (!bufferizesToMemoryRead(opOperand) && - llvm::none_of(aliasingOpResults, - [&](OpResult opResult) { return isValueRead(opResult); })) + llvm::none_of( + aliases, [&](AliasingOpResult a) { return isValueRead(a.opResult); })) return true; // Default: Cannot omit the copy. @@ -593,8 +599,8 @@ // Note: In the absence of detailed analysis information (e.g., there may be // no function call analysis information), this `getAliasingOpResult` is // conservative and may report additional OpResults as potentially aliasing. - for (OpResult opResult : getAliasingOpResult(*operand)) - for (OpOperand &use : opResult.getUses()) + for (AliasingOpResult alias : getAliasingOpResults(*operand)) + for (OpOperand &use : alias.opResult.getUses()) worklist.push_back(&use); } @@ -821,18 +827,18 @@ bool bufferization::detail::defaultResultBufferizesToMemoryWrite( OpResult opResult, const AnalysisState &state) { auto bufferizableOp = cast(opResult.getDefiningOp()); - SmallVector opOperands = - bufferizableOp.getAliasingOpOperand(opResult, state); + AliasingOpOperandList opOperands = + bufferizableOp.getAliasingOpOperands(opResult, state); // Case 1: OpResults that have no aliasing OpOperand usually bufferize to // memory writes. - if (opOperands.empty()) + if (opOperands.getAliases().empty()) return true; // Case 2: If an aliasing OpOperand bufferizes to a memory write, the OpResult // may bufferize to a memory write. - if (llvm::any_of(opOperands, [&](OpOperand *operand) { - return state.bufferizesToMemoryWrite(*operand); + if (llvm::any_of(opOperands, [&](AliasingOpOperand alias) { + return state.bufferizesToMemoryWrite(*alias.opOperand); })) return true; @@ -871,9 +877,9 @@ return false; return state.bufferizesToMemoryWrite(v); }; - for (OpOperand *operand : opOperands) { + for (AliasingOpOperand alias : opOperands) { if (!state - .findValueInReverseUseDefChain(operand->get(), + .findValueInReverseUseDefChain(alias.opOperand->get(), isMemoryWriteInsideOp, /*followEquivalentOnly=*/false) .empty()) @@ -882,6 +888,24 @@ return false; } +// Compute the AliasingOpOperandList for a given OpResult based on +// getAliasingOpResults. +AliasingOpOperandList bufferization::detail::defaultGetAliasingOpOperands( + OpResult opResult, const AnalysisState &state) { + Operation *op = opResult.getDefiningOp(); + SmallVector result; + for (OpOperand &opOperand : op->getOpOperands()) { + if (!opOperand.get().getType().isa()) + continue; + AliasingOpResultList aliasingOpResults = + state.getAliasingOpResults(opOperand); + for (const auto &it : aliasingOpResults) + if (it.opResult == opResult) + result.emplace_back(&opOperand, it.relation, it.isDefinite); + } + return AliasingOpOperandList(std::move(result)); +} + FailureOr bufferization::detail::defaultGetBufferType( Value value, const BufferizationOptions &options, const DenseMap &fixedTypes) { @@ -894,15 +918,13 @@ // Value is an OpResult. Operation *op = getOwnerOfValue(value); auto opResult = value.cast(); - auto bufferizableOp = cast(op); AnalysisState state(options); - auto aliasingOperands = bufferizableOp.getAliasingOpOperand(opResult, state); - if (!aliasingOperands.empty() && - bufferizableOp.bufferRelation(opResult, state) == - BufferRelation::Equivalent) { + AliasingOpOperandList aliases = state.getAliasingOpOperands(opResult); + if (aliases.getNumAliases() > 0 && + aliases.getAliases()[0].relation == BufferRelation::Equivalent) { // If the OpResult has an equivalent OpOperand, both OpResult and // OpOperand bufferize to the exact same buffer type. - Value equivalentOperand = aliasingOperands.front()->get(); + Value equivalentOperand = aliases.getAliases().front().opOperand->get(); return getBufferType(equivalentOperand, options, fixedTypes); } @@ -925,22 +947,22 @@ return regionInterface.isRepetitiveRegion(index); } -SmallVector -bufferization::detail::unknownGetAliasingOpOperand(OpResult opResult) { - // Conservatively assume that everything is aliasing. - SmallVector r; +AliasingOpOperandList +bufferization::detail::unknownGetAliasingOpOperands(OpResult opResult) { + // Conservatively assume that everything may be aliasing. + AliasingOpOperandList r; for (OpOperand &operand : opResult.getDefiningOp()->getOpOperands()) if (operand.get().getType().isa()) - r.push_back(&operand); + r.addAlias({&operand, BufferRelation::Unknown, /*isDefinite=*/false}); return r; } -SmallVector -bufferization::detail::unknownGetAliasingOpResult(OpOperand &opOperand) { - // Conservatively assume that everything is aliasing. - SmallVector r; +AliasingOpResultList +bufferization::detail::unknownGetAliasingOpResults(OpOperand &opOperand) { + // Conservatively assume that everything may be aliasing. + AliasingOpResultList r; for (OpResult result : opOperand.getOwner()->getOpResults()) if (result.getType().isa()) - r.push_back(result); + r.addAlias({result, BufferRelation::Unknown, /*isDefinite=*/false}); return r; } diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp --- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp +++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp @@ -226,9 +226,9 @@ return false; } -SmallVector -AllocTensorOp::getAliasingOpResult(OpOperand &opOperand, - const AnalysisState &state) { +AliasingOpResultList +AllocTensorOp::getAliasingOpResults(OpOperand &opOperand, + const AnalysisState &state) { // This is a new allocation. It does not alias with any other buffer. return {}; } diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp @@ -172,52 +172,37 @@ opOperand.getOperandNumber()); } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { func::CallOp callOp = cast(op); FuncOp funcOp = getCalledFunction(callOp); assert(funcOp && "expected CallOp to a FuncOp"); if (getFuncOpAnalysisState(state, funcOp) != FuncOpAnalysisState::Analyzed) // FuncOp not analyzed yet. Any OpResult may be aliasing. - return detail::unknownGetAliasingOpResult(opOperand); + return detail::unknownGetAliasingOpResults(opOperand); // Get aliasing results from state. const FuncAnalysisState &funcState = getFuncAnalysisState(state); auto aliasingReturnVals = funcState.aliasingReturnVals.lookup(funcOp).lookup( opOperand.getOperandNumber()); - SmallVector result; - for (int64_t resultIdx : aliasingReturnVals) - result.push_back(callOp->getOpResult(resultIdx)); - return result; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - func::CallOp callOp = cast(op); - FuncOp funcOp = getCalledFunction(callOp); - assert(funcOp && "expected CallOp to a FuncOp"); - if (getFuncOpAnalysisState(state, funcOp) != - FuncOpAnalysisState::Analyzed) { - // Function not analyzed yet. The conservative answer is "None". - return BufferRelation::Unknown; - } - const FuncAnalysisState &funcState = getFuncAnalysisState(state); - std::optional maybeEquiv = - getEquivalentFuncArgIdx(funcOp, funcState, opResult.getResultNumber()); - if (maybeEquiv) { -#ifndef NDEBUG - SmallVector aliasingOpOperands = - getAliasingOpOperand(op, opResult, state); - assert(aliasingOpOperands.size() == 1 && - "expected exactly 1 aliasing OpOperand"); - assert(aliasingOpOperands.front()->getOperandNumber() == *maybeEquiv && + // Check if the aliasing OpResult is equivalent to the OpOperand. + std::optional equivalent = {}; + if (aliasingReturnVals.size() == 1) { + equivalent = getEquivalentFuncArgIdx(funcOp, funcState, + aliasingReturnVals.front()); + assert((!equivalent.has_value() || + *equivalent == opOperand.getOperandNumber()) && "inconsistent analysis state"); -#endif - return BufferRelation::Equivalent; } - return BufferRelation::Unknown; + AliasingOpResultList result; + for (int64_t resultIdx : aliasingReturnVals) + result.addAlias({callOp->getOpResult(resultIdx), + equivalent.has_value() ? BufferRelation::Equivalent + : BufferRelation::Unknown, + /*isDefinite=*/equivalent.has_value()}); + return result; } /// All function arguments are writable. It is the responsibility of the @@ -329,7 +314,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp --- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp @@ -147,8 +147,8 @@ if (inplaceBufferized.contains(&operand)) return; markInPlace(operand); - for (OpResult result : state.getAliasingOpResult(operand)) - aliasInfo.unionSets(result, operand.get()); + for (AliasingOpResult alias : state.getAliasingOpResults(operand)) + aliasInfo.unionSets(alias.opResult, operand.get()); ++statNumTensorInPlace; } @@ -636,9 +636,10 @@ // No conflict if the conflicting write and the definition are the same // use. - SmallVector aliasingOpResult = - state.getAliasingOpResult(*uConflictingWrite); - if (aliasingOpResult.size() == 1 && aliasingOpResult[0] == definition) { + AliasingOpResultList aliases = + state.getAliasingOpResults(*uConflictingWrite); + if (aliases.getNumAliases() == 1 && + aliases.getAliases()[0].opResult == definition) { LLVM_DEBUG(llvm::dbgs() << " no conflict: definition and write are same\n"); continue; @@ -697,9 +698,10 @@ // there would then be no flow of data from the extract_slice operand to // its result's uses.) if (!state.bufferizesToMemoryWrite(use)) { - SmallVector opResults = state.getAliasingOpResult(use); - if (llvm::any_of(opResults, - [&](OpResult r) { return state.isValueRead(r); })) + AliasingOpResultList aliases = state.getAliasingOpResults(use); + if (llvm::any_of(aliases, [&](AliasingOpResult a) { + return state.isValueRead(a.opResult); + })) res.insert(&use); } } @@ -743,9 +745,9 @@ DenseSet usesRead, usesWrite; getAliasingReads(usesRead, operand.get(), aliasInfo, state); getAliasingInplaceWrites(usesWrite, operand.get(), aliasInfo, state); - for (OpResult result : state.getAliasingOpResult(operand)) { - getAliasingReads(usesRead, result, aliasInfo, state); - getAliasingInplaceWrites(usesWrite, result, aliasInfo, state); + for (AliasingOpResult alias : state.getAliasingOpResults(operand)) { + getAliasingReads(usesRead, alias.opResult, aliasInfo, state); + getAliasingInplaceWrites(usesWrite, alias.opResult, aliasInfo, state); } if (!checkConsistencyOnly && state.bufferizesToMemoryWrite(operand)) usesWrite.insert(&operand); @@ -796,11 +798,10 @@ continue; // Follow reverse SSA use-def chain. - SmallVector aliasingOpOperands = - state.getAliasingOpOperand(opResult); - for (OpOperand *opOperand : aliasingOpOperands) - if (aliasInfo.isInPlace(*opOperand) || currentOpOperand == opOperand) - worklist.push_back(opOperand->get()); + for (AliasingOpOperand alias : state.getAliasingOpOperands(opResult)) + if (aliasInfo.isInPlace(*alias.opOperand) || + currentOpOperand == alias.opOperand) + worklist.push_back(alias.opOperand->get()); } return false; } @@ -813,8 +814,8 @@ // Collect writes of all aliases of OpOperand and OpResult. DenseSet usesWrite; getAliasingInplaceWrites(usesWrite, operand.get(), aliasInfo, state); - for (OpResult result : state.getAliasingOpResult(operand)) { - getAliasingInplaceWrites(usesWrite, result, aliasInfo, state); + for (AliasingOpResult alias : state.getAliasingOpResults(operand)) { + getAliasingInplaceWrites(usesWrite, alias.opResult, aliasInfo, state); } if (!checkConsistencyOnly && state.bufferizesToMemoryWrite(operand)) usesWrite.insert(&operand); @@ -949,16 +950,21 @@ static void equivalenceAnalysis(SmallVector &ops, BufferizationAliasInfo &aliasInfo, AnalysisState &state) { - for (Operation *op : ops) - if (auto bufferizableOp = state.getOptions().dynCastBufferizableOp(op)) - for (OpResult opResult : op->getOpResults()) - if (opResult.getType().isa()) - for (OpOperand *opOperand : - bufferizableOp.getAliasingOpOperand(opResult, state)) - if (state.isInPlace(*opOperand)) - if (bufferizableOp.bufferRelation(opResult, state) == - BufferRelation::Equivalent) - aliasInfo.unionEquivalenceClasses(opResult, opOperand->get()); + for (Operation *op : ops) { + for (OpOperand &opOperand : op->getOpOperands()) { + if (opOperand.get().getType().isa()) { + if (!state.isInPlace(opOperand)) + // Out-of-place OpOperands bufferize to new allocations and do not + // union equivalence sets. + continue; + AliasingOpResultList aliases = state.getAliasingOpResults(opOperand); + for (AliasingOpResult alias : aliases) { + if (alias.relation == BufferRelation::Equivalent && alias.isDefinite) + aliasInfo.unionEquivalenceClasses(alias.opResult, opOperand.get()); + } + } + } + } } /// Analyze equivalence of tied OpResult/OpOperand pairs of all ops contained @@ -970,7 +976,7 @@ SmallVector ops; op->walk([&](Operation *op) { // No tensors => no buffers. - if (none_of(op->getResultTypes(), isaTensor)) + if (none_of(op->getOperandTypes(), isaTensor)) return; ops.push_back(op); }); diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp @@ -102,8 +102,8 @@ bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { // Operand is written to if it has an aliasing OpResult. - auto bufferizableOp = cast(op); - return !bufferizableOp.getAliasingOpResult(opOperand, state).empty(); + auto dpsOp = cast(op); + return dpsOp.isDpsInit(&opOperand); } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp @@ -57,7 +57,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -104,9 +104,9 @@ struct ExecuteRegionOpInterface : public BufferizableOpInterface::ExternalModel { - SmallVector - getAliasingOpOperand(Operation *op, OpResult opResult, - const AnalysisState &state) const { + AliasingOpOperandList + getAliasingOpOperands(Operation *op, OpResult opResult, + const AnalysisState &state) const { // ExecuteRegionOps do not have tensor OpOperands. The yielded value can be // any SSA value that is in scope. To allow for use-def chain traversal // through ExecuteRegionOps in the analysis, the corresponding yield value @@ -120,7 +120,7 @@ auto yieldOp = dyn_cast( executeRegionOp.getRegion().front().getTerminator()); assert(yieldOp && "expected scf.yield terminator in scf.execute_region"); - return {&yieldOp->getOpOperand(resultNum)}; + return {{&yieldOp->getOpOperand(resultNum), BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -154,19 +154,14 @@ return success(); } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; - } }; /// Bufferization of scf.if. Replace with a new scf.if that yields memrefs. struct IfOpInterface : public BufferizableOpInterface::ExternalModel { - SmallVector - getAliasingOpOperand(Operation *op, OpResult opResult, - const AnalysisState &state) const { + AliasingOpOperandList + getAliasingOpOperands(Operation *op, OpResult opResult, + const AnalysisState &state) const { // IfOps do not have tensor OpOperands. The yielded value can be any SSA // value that is in scope. To allow for use-def chain traversal through // IfOps in the analysis, both corresponding yield values from the then/else @@ -174,8 +169,10 @@ auto ifOp = cast(op); size_t resultNum = std::distance(op->getOpResults().begin(), llvm::find(op->getOpResults(), opResult)); - return {&ifOp.thenYield()->getOpOperand(resultNum), - &ifOp.elseYield()->getOpOperand(resultNum)}; + OpOperand *thenOperand = &ifOp.thenYield()->getOpOperand(resultNum); + OpOperand *elseOperand = &ifOp.elseYield()->getOpOperand(resultNum); + return {{thenOperand, BufferRelation::Equivalent}, + {elseOperand, BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -258,20 +255,6 @@ return getMemRefTypeWithFullyDynamicLayout( opResult.getType().cast(), thenBufferType.getMemorySpace()); } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - // IfOp results are equivalent to their corresponding yield values if both - // yield values are equivalent to each other. - auto bufferizableOp = cast(op); - SmallVector yieldValues = - bufferizableOp.getAliasingOpOperand(opResult, state); - assert(yieldValues.size() == 2 && "expected 2 yield values"); - bool equivalentYields = state.areEquivalentBufferizedValues( - yieldValues[0]->get(), yieldValues[1]->get()); - return equivalentYields ? BufferRelation::Equivalent - : BufferRelation::Unknown; - } }; /// Helper function for loop bufferization. Return the indices of all values @@ -443,10 +426,13 @@ return true; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { auto forOp = cast(op); - return {forOp.getResultForOpOperand(opOperand)}; + OpResult opResult = forOp.getResultForOpOperand(opOperand); + BufferRelation relation = bufferRelation(op, opResult, state); + return {{opResult, relation, + /*isDefinite=*/relation == BufferRelation::Equivalent}}; } BufferRelation bufferRelation(Operation *op, OpResult opResult, @@ -657,7 +643,7 @@ return true; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { auto whileOp = cast(op); unsigned int idx = opOperand.getOperandNumber(); @@ -669,7 +655,10 @@ return {}; // The only aliasing OpResult may be the one at the same index. - return {whileOp->getResult(idx)}; + OpResult opResult = whileOp->getResult(idx); + BufferRelation relation = bufferRelation(op, opResult, state); + return {{opResult, relation, + /*isDefinite=*/relation == BufferRelation::Equivalent}}; } BufferRelation bufferRelation(Operation *op, OpResult opResult, @@ -952,12 +941,21 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - if (isa(op->getParentOp())) - return {op->getParentOp()->getResult(opOperand.getOperandNumber())}; + if (auto ifOp = dyn_cast(op->getParentOp())) { + Region &otherCase = + ifOp->getRegion((op->getParentRegion()->getRegionNumber() + 1) % 2); + auto otherYieldOp = cast(otherCase.back().back()); + bool equivalentYields = state.areEquivalentBufferizedValues( + opOperand.get(), + otherYieldOp->getOperand(opOperand.getOperandNumber())); + return {{op->getParentOp()->getResult(opOperand.getOperandNumber()), + BufferRelation::Equivalent, /*isDefinite=*/equivalentYields}}; + } if (isa(op->getParentOp())) - return {op->getParentOp()->getResult(opOperand.getOperandNumber())}; + return {{op->getParentOp()->getResult(opOperand.getOperandNumber()), + BufferRelation::Equivalent}}; return {}; } @@ -1051,15 +1049,11 @@ return true; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { auto foreachThreadOp = cast(op); - return {foreachThreadOp.getTiedOpResult(&opOperand)}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{{foreachThreadOp.getTiedOpResult(&opOperand), + BufferRelation::Equivalent}}}; } bool isWritable(Operation *op, Value value, diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp @@ -27,9 +27,9 @@ struct AssumingOpInterface : public BufferizableOpInterface::ExternalModel { - SmallVector - getAliasingOpOperand(Operation *op, OpResult opResult, - const AnalysisState &state) const { + AliasingOpOperandList + getAliasingOpOperands(Operation *op, OpResult opResult, + const AnalysisState &state) const { // AssumingOps do not have tensor OpOperands. The yielded value can be any // SSA value that is in scope. To allow for use-def chain traversal through // AssumingOps in the analysis, the corresponding yield value is considered @@ -43,7 +43,7 @@ auto yieldOp = dyn_cast( assumingOp.getDoRegion().front().getTerminator()); assert(yieldOp && "expected shape.assuming_yield terminator"); - return {&yieldOp->getOpOperand(resultNum)}; + return {{&yieldOp->getOpOperand(resultNum), BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -77,11 +77,6 @@ return success(); } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; - } }; /// Bufferization of shape.assuming_yield. Bufferized as part of their enclosing @@ -99,11 +94,13 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { assert(isa(op->getParentOp()) && "expected that parent is an AssumingOp"); - return {op->getParentOp()->getResult(opOperand.getOperandNumber())}; + OpResult opResult = + op->getParentOp()->getResult(opOperand.getOperandNumber()); + return {{opResult, BufferRelation::Equivalent}}; } bool mustBufferizeInPlace(Operation *op, OpOperand &opOperand, diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp @@ -43,7 +43,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -73,7 +73,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -97,14 +97,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {op->getOpResult(0)}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{op->getOpResult(0), BufferRelation::Equivalent}}; } }; @@ -136,18 +131,11 @@ return true; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { // InsertOp returns an alias of its operand. assert(op->getNumResults() == 1); - return op->getResults(); - } - - BufferRelation bufferRelation(Operation *oo, OpResult opResult, - const AnalysisState &state) const { - // InsertOp returns the same object (realloc should not invalidate - // aliases). - return BufferRelation::Equivalent; + return {{op->getOpResult(0), BufferRelation::Equivalent}}; } }; @@ -164,7 +152,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -206,7 +194,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -227,7 +215,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -248,7 +236,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp @@ -41,14 +41,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {op->getResult(0)}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{op->getResult(0), BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -98,16 +93,10 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - if (&opOperand == &op->getOpOperand(0) /*src*/) - return {op->getOpResult(0)}; - return {}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + // TODO: CollapseShapeOp may allocate at runtime. + return {{op->getOpResult(0), BufferRelation::Equivalent}}; } FailureOr @@ -214,7 +203,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -263,16 +252,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - if (&opOperand == &op->getOpOperand(0) /*src*/) - return {op->getOpResult(0)}; - return {}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{op->getOpResult(0), BufferRelation::Equivalent}}; } FailureOr @@ -324,16 +306,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - if (&opOperand == &op->getOpOperand(0) /*source*/) - return {op->getOpResult(0)}; - return {}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Unknown; + return {{op->getOpResult(0), BufferRelation::Unknown}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -397,7 +372,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -837,7 +812,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -935,7 +910,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -968,14 +943,9 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {op->getOpResult(0)}; - } - - BufferRelation bufferRelation(Operation *op, OpResult opResult, - const AnalysisState &state) const { - return BufferRelation::Equivalent; + return {{op->getOpResult(0), BufferRelation::Equivalent}}; } LogicalResult bufferize(Operation *op, RewriterBase &rewriter, @@ -1000,9 +970,9 @@ struct ParallelInsertSliceOpInterface : public BufferizableOpInterface::ExternalModel< ParallelInsertSliceOpInterface, ParallelInsertSliceOp> { - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { - return {}; + return {}; } bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, diff --git a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp --- a/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.cpp @@ -41,7 +41,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; } @@ -110,7 +110,7 @@ return false; } - SmallVector getAliasingOpResult(Operation *op, OpOperand &opOperand, + AliasingOpResultList getAliasingOpResults(Operation *op, OpOperand &opOperand, const AnalysisState &state) const { return {}; }