diff --git a/mlir/include/mlir/IR/Block.h b/mlir/include/mlir/IR/Block.h --- a/mlir/include/mlir/IR/Block.h +++ b/mlir/include/mlir/IR/Block.h @@ -18,7 +18,7 @@ namespace mlir { /// `Block` represents an ordered list of `Operation`s. -class Block : public IRObjectWithUseList, +class Block : public IRObjectWithUseList, public llvm::ilist_node_with_parent { public: explicit Block() {} diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h --- a/mlir/include/mlir/IR/Operation.h +++ b/mlir/include/mlir/IR/Operation.h @@ -237,9 +237,6 @@ // Results //===--------------------------------------------------------------------===// - /// Return true if there are no users of any results of this operation. - bool use_empty(); - unsigned getNumResults() { return numResults; } Value getResult(unsigned idx) { return getOpResult(idx); } @@ -639,32 +636,6 @@ return os; } -/// This class implements use iterator for the Operation. This iterates over all -/// uses of all results of an Operation. -class UseIterator final - : public llvm::iterator_facade_base { -public: - /// Initialize UseIterator for op, specify end to return iterator to last use. - explicit UseIterator(Operation *op, bool end = false); - - UseIterator &operator++(); - Operation *operator->() { return use->getOwner(); } - Operation *operator*() { return use->getOwner(); } - - bool operator==(const UseIterator &other) const; - bool operator!=(const UseIterator &other) const; - -private: - void skipOverResultsWithNoUsers(); - - /// The operation whose uses are being iterated over. - Operation *op; - /// The result of op who's uses are being iterated over. - Operation::result_iterator res; - /// The use of the result. - Value::use_iterator use; -}; } // end namespace mlir namespace llvm { diff --git a/mlir/include/mlir/IR/UseDefLists.h b/mlir/include/mlir/IR/UseDefLists.h --- a/mlir/include/mlir/IR/UseDefLists.h +++ b/mlir/include/mlir/IR/UseDefLists.h @@ -24,7 +24,8 @@ class Operation; class Value; template class ValueUseIterator; -template class ValueUserIterator; +template class FilteredValueUseIterator; +template class ValueUserIterator; //===----------------------------------------------------------------------===// // IRObjectWithUseList @@ -36,49 +37,145 @@ assert(use_empty() && "Cannot destroy a value that still has uses!"); } - /// Returns true if this value has no uses. - bool use_empty() const { return firstUse == nullptr; } + /// Drop all uses of this object from their respective owners. + void dropAllUses() { + while (!use_empty()) + use_begin()->drop(); + } - /// Returns true if this value has exactly one use. - inline bool hasOneUse() const; + /// Replace all uses of 'this' value with the new value, updating anything in + /// the IR that uses 'this' to use the other value instead. When this returns + /// there are zero uses of 'this'. + void replaceAllUsesWith(typename OperandType::ValueType newValue) { + assert(this != OperandType::getUseList(newValue) && + "cannot RAUW a value with itself"); + while (!use_empty()) + use_begin()->set(newValue); + } + + //===--------------------------------------------------------------------===// + // Uses + //===--------------------------------------------------------------------===// - using use_iterator = ValueUseIterator; + using use_iterator = ValueUseIterator; using use_range = iterator_range; - inline use_iterator use_begin() const; - inline use_iterator use_end() const; + use_iterator use_begin() const { return use_iterator(firstUse); } + use_iterator use_end() const { return use_iterator(nullptr); } /// Returns a range of all uses, which is useful for iterating over all uses. - inline use_range getUses() const; + use_range getUses() const { return {use_begin(), use_end()}; } - using user_iterator = ValueUserIterator; - using user_range = iterator_range; + /// Returns true if this value has exactly one use. + bool hasOneUse() const { + return firstUse && firstUse->getNextOperandUsingThisValue() == nullptr; + } - inline user_iterator user_begin() const; - inline user_iterator user_end() const; + /// Returns true if this value has no uses. + bool use_empty() const { return firstUse == nullptr; } - /// Returns a range of all users. - inline user_range getUsers() const; + //===--------------------------------------------------------------------===// + // Users + //===--------------------------------------------------------------------===// - /// Replace all uses of 'this' value with the new value, updating anything in - /// the IR that uses 'this' to use the other value instead. When this returns - /// there are zero uses of 'this'. - void replaceAllUsesWith(IRObjectWithUseList *newValue); + using user_iterator = ValueUserIterator; + using user_range = iterator_range; - /// Drop all uses of this object from their respective owners. - void dropAllUses(); + user_iterator user_begin() const { return user_iterator(use_begin()); } + user_iterator user_end() const { return user_iterator(use_end()); } + + /// Returns a range of all users. + user_range getUsers() const { return {user_begin(), user_end()}; } protected: IRObjectWithUseList() {} - /// Return the first IROperand that is using this value, for use by custom + /// Return the first operand that is using this value, for use by custom /// use/def iterators. - IROperand *getFirstUse() { return firstUse; } - const IROperand *getFirstUse() const { return firstUse; } + OperandType *getFirstUse() const { return firstUse; } private: - friend class IROperand; - IROperand *firstUse = nullptr; + template friend class IROperand; + OperandType *firstUse = nullptr; +}; + +//===----------------------------------------------------------------------===// +// IRMultiObjectWithUseList +//===----------------------------------------------------------------------===// + +/// This class represents multiple IR objects with a single use list. This class +/// provides wrapper functionality for manipulating the uses of a single object. +template +class IRMultiObjectWithUseList : public IRObjectWithUseList { +public: + using BaseType = IRObjectWithUseList; + using ValueType = typename OperandType::ValueType; + + /// Drop all uses of `value` from their respective owners. + void dropAllUses(ValueType value) { + assert(this == OperandType::getUseList(value) && + "value not attached to this use list"); + for (OperandType &use : llvm::make_early_inc_range(getUses(value))) + use.drop(); + } + using BaseType::dropAllUses; + + /// Replace all uses of `oldValue` with the new value, updating anything in + /// the IR that uses 'this' to use the other value instead. When this returns + /// there are zero uses of 'this'. + void replaceAllUsesWith(ValueType oldValue, ValueType newValue) { + assert(this == OperandType::getUseList(oldValue) && + "value not attached to this use list"); + assert(this != OperandType::getUseList(newValue) && + "cannot RAUW a value with itself"); + for (OperandType &use : llvm::make_early_inc_range(getUses(oldValue))) + use.set(newValue); + } + + //===--------------------------------------------------------------------===// + // Uses + //===--------------------------------------------------------------------===// + + using filtered_use_iterator = FilteredValueUseIterator; + using filtered_use_range = iterator_range; + + filtered_use_iterator use_begin(ValueType value) const { + return filtered_use_iterator(this->getFirstUse(), value); + } + filtered_use_iterator use_end(ValueType) const { return use_end(); } + filtered_use_range getUses(ValueType value) const { + return {use_begin(value), use_end(value)}; + } + bool hasOneUse(ValueType value) const { + return mlir::has_single_element(getUses(value)); + } + bool use_empty(ValueType value) const { + return use_begin(value) == use_end(value); + } + using BaseType::getUses; + using BaseType::hasOneUse; + using BaseType::use_begin; + using BaseType::use_empty; + using BaseType::use_end; + + //===--------------------------------------------------------------------===// + // Users + //===--------------------------------------------------------------------===// + + using filtered_user_iterator = + ValueUserIterator; + using filtered_user_range = iterator_range; + + filtered_user_iterator user_begin(ValueType value) const { + return {use_begin(value)}; + } + filtered_user_iterator user_end(ValueType value) const { return {use_end()}; } + filtered_user_range getUsers(ValueType value) const { + return {user_begin(value), user_end(value)}; + } + using BaseType::getUsers; + using BaseType::user_begin; + using BaseType::user_end; }; //===----------------------------------------------------------------------===// @@ -86,19 +183,24 @@ //===----------------------------------------------------------------------===// /// A reference to a value, suitable for use as an operand of an operation. -class IROperand { +/// IRValueTy is the root type to use for values this tracks. Derived operand +/// types are expected to provide the following: +/// * static IRObjectWithUseList *getUseList(IRValueTy value); +/// - Provide the use list that is attached to the given value. +template class IROperand { public: + using ValueType = IRValueTy; + IROperand(Operation *owner) : owner(owner) {} - IROperand(Operation *owner, IRObjectWithUseList *value) - : value(value), owner(owner) { + IROperand(Operation *owner, ValueType value) : value(value), owner(owner) { insertIntoCurrent(); } /// Return the current value being used by this operand. - IRObjectWithUseList *get() const { return value; } + ValueType get() const { return value; } /// Set the current value being used by this operand. - void set(IRObjectWithUseList *newValue) { + void set(ValueType newValue) { // It isn't worth optimizing for the case of switching operands on a single // value. removeFromCurrent(); @@ -123,7 +225,7 @@ /// Return the next operand on the use-list of the value we are referring to. /// This should generally only be used by the internal implementation details /// of the SSA machinery. - IROperand *getNextOperandUsingThisValue() { return nextUse; } + DerivedT *getNextOperandUsingThisValue() { return nextUse; } /// We support a move constructor so IROperand's can be in vectors, but this /// shouldn't be used by general clients. @@ -143,15 +245,15 @@ } private: - /// The value used as this operand. This can be null when in a - /// "dropAllUses" state. - IRObjectWithUseList *value = nullptr; + /// The value used as this operand. This can be null when in a 'dropAllUses' + /// state. + ValueType value = {}; /// The next operand in the use-chain. - IROperand *nextUse = nullptr; + DerivedT *nextUse = nullptr; /// This points to the previous link in the use-chain. - IROperand **back = nullptr; + DerivedT **back = nullptr; /// The operation owner of this operand. Operation *const owner; @@ -169,11 +271,12 @@ } void insertIntoCurrent() { - back = &value->firstUse; - nextUse = value->firstUse; + auto *useList = DerivedT::getUseList(value); + back = &useList->firstUse; + nextUse = useList->firstUse; if (nextUse) nextUse->back = &nextUse; - value->firstUse = this; + useList->firstUse = static_cast(this); } }; @@ -232,62 +335,96 @@ class ValueUseIterator : public std::iterator { public: - ValueUseIterator() = default; - explicit ValueUseIterator(OperandType *current) : current(current) {} - OperandType *operator->() const { return current; } - OperandType &operator*() const { return *current; } + template + ValueUseIteratorImpl(const ValueUseIteratorImpl &other) + : current(other.getOperand()) {} + ValueUseIteratorImpl(OperandType *current = nullptr) : current(current) {} Operation *getUser() const { return current->getOwner(); } + OperandType *getOperand() const { return current; } + + OperandType &operator*() const { return *current; } - ValueUseIterator &operator++() { + using llvm::iterator_facade_base::operator++; + ValueUseIteratorImpl &operator++() { assert(current && "incrementing past end()!"); current = (OperandType *)current->getNextOperandUsingThisValue(); return *this; } - ValueUseIterator operator++(int unused) { - ValueUseIterator copy = *this; - ++*this; - return copy; + bool operator==(const ValueUseIteratorImpl &rhs) const { + return current == rhs.current; } - friend bool operator==(ValueUseIterator lhs, ValueUseIterator rhs) { - return lhs.current == rhs.current; - } +protected: + OperandType *current; +}; - friend bool operator!=(ValueUseIterator lhs, ValueUseIterator rhs) { - return !(lhs == rhs); - } +} // end namespace detail -private: - OperandType *current; +/// An iterator over all of the uses of an IR object. +template +class ValueUseIterator + : public detail::ValueUseIteratorImpl, + OperandType> { +public: + using detail::ValueUseIteratorImpl, + OperandType>::ValueUseIteratorImpl; }; -inline auto IRObjectWithUseList::use_begin() const -> use_iterator { - return use_iterator(firstUse); -} +/// This class represents an iterator of the uses of a IR object that optionally +/// filters on a specific sub-value. This allows for filtering the uses of an +/// IRMultiObjectWithUseList. +template +class FilteredValueUseIterator + : public detail::ValueUseIteratorImpl, + OperandType> { +public: + using BaseT = + detail::ValueUseIteratorImpl, + OperandType>; + + FilteredValueUseIterator() = default; + FilteredValueUseIterator(const ValueUseIterator &it) + : BaseT(it), filterVal(nullptr) {} + FilteredValueUseIterator(OperandType *current, + typename OperandType::ValueType filterVal) + : BaseT(current), filterVal(filterVal) { + findNextValid(); + } -inline auto IRObjectWithUseList::use_end() const -> use_iterator { - return use_iterator(nullptr); -} + using BaseT::operator++; + FilteredValueUseIterator &operator++() { + BaseT::operator++(); + findNextValid(); + return *this; + } + +private: + void findNextValid() { + if (!filterVal) + return; + while (this->current && ((OperandType *)this->current)->get() != filterVal) + BaseT::operator++(); + } -inline auto IRObjectWithUseList::getUses() const -> use_range { - return {use_begin(), use_end()}; -} + /// An optional value to use to filter specific uses. + typename OperandType::ValueType filterVal; +}; -/// Returns true if this value has exactly one use. -inline bool IRObjectWithUseList::hasOneUse() const { - return firstUse && firstUse->getNextOperandUsingThisValue() == nullptr; -} +//===----------------------------------------------------------------------===// +// ValueUserIterator +//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// // ValueUserIterator //===----------------------------------------------------------------------===// /// An iterator over all users of a ValueBase. -template +template class ValueUserIterator final - : public llvm::mapped_iterator, + : public llvm::mapped_iterator { static Operation *unwrap(OperandType &value) { return value.getOwner(); } @@ -296,24 +433,12 @@ using reference = Operation *; /// Initializes the result type iterator to the specified result iterator. - ValueUserIterator(ValueUseIterator it) - : llvm::mapped_iterator, - Operation *(*)(OperandType &)>(it, &unwrap) {} + ValueUserIterator(UseIteratorT it) + : llvm::mapped_iterator( + it, &unwrap) {} Operation *operator->() { return **this; } }; -inline auto IRObjectWithUseList::user_begin() const -> user_iterator { - return user_iterator(use_begin()); -} - -inline auto IRObjectWithUseList::user_end() const -> user_iterator { - return user_iterator(use_end()); -} - -inline auto IRObjectWithUseList::getUsers() const -> user_range { - return {user_begin(), user_end()}; -} - } // namespace mlir #endif diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h --- a/mlir/include/mlir/IR/Value.h +++ b/mlir/include/mlir/IR/Value.h @@ -27,7 +27,7 @@ namespace detail { /// The internal implementation of a Value. -class ValueImpl : public IRObjectWithUseList { +class ValueImpl { protected: /// This enumerates all of the SSA value kinds. enum class Kind { @@ -46,7 +46,8 @@ }; /// The internal implementation of a BlockArgument. -class BlockArgumentImpl : public ValueImpl { +class BlockArgumentImpl : public ValueImpl, + public IRObjectWithUseList { BlockArgumentImpl(Type type, Block *owner) : ValueImpl(Kind::BlockArgument, type), owner(owner) {} @@ -132,13 +133,6 @@ /// place. void setType(Type newType) { impl->typeAndKind.setPointer(newType); } - /// Replace all uses of 'this' value with the new value, updating anything in - /// the IR that uses 'this' to use the other value instead. When this returns - /// there are zero uses of 'this'. - void replaceAllUsesWith(Value newValue) const { - impl->replaceAllUsesWith(newValue.impl); - } - /// If this value is the result of an operation, return the operation that /// defines it. Operation *getDefiningOp() const; @@ -150,32 +144,52 @@ /// Return the Region in which this Value is defined. Region *getParentRegion(); - using use_iterator = ValueUseIterator; + //===--------------------------------------------------------------------===// + // UseLists + //===--------------------------------------------------------------------===// + + /// Provide the use list that is attached to this value. + IRObjectWithUseList *getUseList() const; + + /// Drop all uses of this object from their respective owners. + void dropAllUses() const; + + /// Replace all uses of 'this' value with the new value, updating anything in + /// the IR that uses 'this' to use the other value instead. When this returns + /// there are zero uses of 'this'. + void replaceAllUsesWith(Value newValue) const; + + //===--------------------------------------------------------------------===// + // Uses + + /// This class implements an iterator over the uses of a value. + using use_iterator = FilteredValueUseIterator; using use_range = iterator_range; - inline use_iterator use_begin(); - inline use_iterator use_end(); + use_iterator use_begin() const; + use_iterator use_end() const { return use_iterator(); } /// Returns a range of all uses, which is useful for iterating over all uses. - inline use_range getUses(); + use_range getUses() const { return {use_begin(), use_end()}; } - using user_iterator = ValueUserIterator; - using user_range = iterator_range; + /// Returns true if this value has exactly one use. + bool hasOneUse() const; - user_iterator user_begin() const { return impl->user_begin(); } - user_iterator user_end() const { return impl->user_end(); } + /// Returns true if this value has no uses. + bool use_empty() const; - /// Returns a range of all users. - user_range getUsers() const { return impl->getUsers(); } + //===--------------------------------------------------------------------===// + // Users - /// Returns true if this value has no uses. - bool use_empty() const { return impl->use_empty(); } + using user_iterator = ValueUserIterator; + using user_range = iterator_range; - /// Returns true if this value has exactly one use. - bool hasOneUse() const { return impl->hasOneUse(); } + user_iterator user_begin() const { return use_begin(); } + user_iterator user_end() const { return use_end(); } + user_range getUsers() const { return {user_begin(), user_end()}; } - /// Drop all uses of this object from their respective owners. - void dropAllUses() const { impl->dropAllUses(); } + //===--------------------------------------------------------------------===// + // Utilities void print(raw_ostream &os); void dump(); @@ -201,17 +215,6 @@ return os; } -// Utility functions for iterating through Value uses. -inline auto Value::use_begin() -> use_iterator { - return use_iterator((OpOperand *)impl->getFirstUse()); -} - -inline auto Value::use_end() -> use_iterator { return use_iterator(nullptr); } - -inline auto Value::getUses() -> iterator_range { - return {use_begin(), use_end()}; -} - /// Block arguments are values. class BlockArgument : public Value { public: @@ -248,6 +251,9 @@ /// Allow access to `create` and `destroy`. friend Block; + + /// Allow access to 'getImpl'. + friend Value; }; /// This is a value defined by a result of an operation. diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp --- a/mlir/lib/IR/Operation.cpp +++ b/mlir/lib/IR/Operation.cpp @@ -533,22 +533,13 @@ /// This drops all uses of any values defined by this operation or its nested /// regions, wherever they are located. void Operation::dropAllDefinedValueUses() { - for (auto &val : getOpResults()) - val.dropAllUses(); + dropAllUses(); for (auto ®ion : getRegions()) for (auto &block : region) block.dropAllDefinedValueUses(); } -/// Return true if there are no users of any results of this operation. -bool Operation::use_empty() { - for (auto result : getResults()) - if (!result->use_empty()) - return false; - return true; -} - void Operation::setSuccessor(Block *block, unsigned index) { assert(index < getNumSuccessors()); getBlockOperands()[index].set(block); @@ -1144,50 +1135,3 @@ block.push_back(buildTerminatorOp()); } - -//===----------------------------------------------------------------------===// -// UseIterator -//===----------------------------------------------------------------------===// - -UseIterator::UseIterator(Operation *op, bool end) - : op(op), res(end ? op->result_end() : op->result_begin()) { - // Only initialize current use if there are results/can be uses. - if (op->getNumResults()) - skipOverResultsWithNoUsers(); -} - -UseIterator &UseIterator::operator++() { - // We increment over uses, if we reach the last use then move to next - // result. - if (use != (*res)->use_end()) - ++use; - if (use == (*res)->use_end()) { - ++res; - skipOverResultsWithNoUsers(); - } - return *this; -} - -bool UseIterator::operator==(const UseIterator &other) const { - if (op != other.op) - return false; - if (op->getNumResults() == 0) - return true; - return res == other.res && use == other.use; -} - -bool UseIterator::operator!=(const UseIterator &other) const { - return !(*this == other); -} - -void UseIterator::skipOverResultsWithNoUsers() { - while (res != op->result_end() && (*res)->use_empty()) - ++res; - - // If we are at the last result, then set use to first use of - // first result (sentinel value used for end). - if (res == op->result_end()) - use = {}; - else - use = (*res)->use_begin(); -} diff --git a/mlir/lib/IR/Value.cpp b/mlir/lib/IR/Value.cpp --- a/mlir/lib/IR/Value.cpp +++ b/mlir/lib/IR/Value.cpp @@ -69,19 +69,95 @@ // IRObjectWithUseList implementation. //===----------------------------------------------------------------------===// +/// Provide the use list that is attached to this value. +IRObjectWithUseList *Value::getUseList() const { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl(); + return cast().getOwner(); +} + +/// Drop all uses of this object from their respective owners. +void Value::dropAllUses() const { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl()->dropAllUses(); + return cast().getOwner()->dropAllUses(*this); +} + /// Replace all uses of 'this' value with the new value, updating anything in /// the IR that uses 'this' to use the other value instead. When this returns /// there are zero uses of 'this'. -void IRObjectWithUseList::replaceAllUsesWith(IRObjectWithUseList *newValue) { - assert(this != newValue && "cannot RAUW a value with itself"); - while (!use_empty()) { - use_begin()->set(newValue); - } +void Value::replaceAllUsesWith(Value newValue) const { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl()->replaceAllUsesWith(newValue); + IRMultiObjectWithUseList *useList = cast().getOwner(); + useList->replaceAllUsesWith(*this, newValue); } -/// Drop all uses of this object from their respective owners. -void IRObjectWithUseList::dropAllUses() { - while (!use_empty()) { - use_begin()->drop(); - } +//===--------------------------------------------------------------------===// +// Uses + +auto Value::use_begin() const -> use_iterator { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl()->use_begin(); + return cast().getOwner()->use_begin(*this); +} + +/// Returns true if this value has exactly one use. +bool Value::hasOneUse() const { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl()->hasOneUse(); + return cast().getOwner()->hasOneUse(*this); +} + +/// Returns true if this value has no uses. +bool Value::use_empty() const { + if (BlockArgument arg = dyn_cast()) + return arg.getImpl()->use_empty(); + return cast().getOwner()->use_empty(*this); +} + +//===----------------------------------------------------------------------===// +// BlockOperand +//===----------------------------------------------------------------------===// + +/// Provide the use list that is attached to the given block. +IRObjectWithUseList *BlockOperand::getUseList(Block *value) { + return value; +} + +/// Return which operand this is in the operand list. +unsigned BlockOperand::getOperandNumber() { + return this - &getOwner()->getBlockOperands()[0]; +} + +//===----------------------------------------------------------------------===// +// OpOperand +//===----------------------------------------------------------------------===// + +/// Provide the use list that is attached to the given value. +IRObjectWithUseList *OpOperand::getUseList(Value value) { + return value.getUseList(); +} + +/// Return the current value being used by this operand. +Value OpOperand::get() const { + return IROperand::get(); +} + +/// Return which operand this is in the operand list. +unsigned OpOperand::getOperandNumber() { + return this - &getOwner()->getOpOperands()[0]; +} + +//===----------------------------------------------------------------------===// +// detail::OpaqueValue +//===----------------------------------------------------------------------===// + +/// Implicit conversion from 'Value'. +detail::OpaqueValue::OpaqueValue(Value value) + : impl(value.getAsOpaquePointer()) {} + +/// Implicit conversion back to 'Value'. +detail::OpaqueValue::operator Value() const { + return Value::getFromOpaquePointer(impl); } diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp --- a/mlir/lib/Transforms/ViewOpGraph.cpp +++ b/mlir/lib/Transforms/ViewOpGraph.cpp @@ -29,13 +29,9 @@ using GraphType = Block *; using NodeRef = Operation *; - using ChildIteratorType = UseIterator; - static ChildIteratorType child_begin(NodeRef n) { - return ChildIteratorType(n); - } - static ChildIteratorType child_end(NodeRef n) { - return ChildIteratorType(n, /*end=*/true); - } + using ChildIteratorType = Operation::user_iterator; + static ChildIteratorType child_begin(NodeRef n) { return n->user_begin(); } + static ChildIteratorType child_end(NodeRef n) { return n->user_end(); } // Operation's destructor is private so use Operation* instead and use // mapped iterator.