[mlir] Optimize operand storage such that all operations can have resizable operand lists

[rriddle]

Summary

This revision refactors the structure of the operand storage such that there is no additional memory cost for resizable operand lists until it is required. This is done by using two different internal representations for the operand storage:

• One using trailing operands
• One using a dynamically allocated std::vector<OpOperand>

This allows for removing the resizable operand list bit, and will free up APIs from needing to workaround non-resizable operand lists.

Depends On D78854

[rriddle]

Fix a few typos

[mehdi_amini]

The reported MLIR test failures seem legit?

[mehdi_amini]

mlir/lib/IR/OperationSupport.cpp

134

Using a std::vector adds an extra indirection, you didn't think it was worth having a vector with inlined storage?

[rriddle]

Remove the need for OpOperand copy constructor.

[rriddle]

mlir/lib/IR/OperationSupport.cpp

134

I think it would be worth it, but I don't have an idea of a size that would be good to use in general. I'm deferring that until I can get a few benchmarks that can give me a good indicator here.

[mehdi_amini]

mlir/lib/IR/OperationSupport.cpp

134

Why don't we just allocate inline always the number of operands initially and then grow to the next power of two when we run out of capacity?

[rriddle]

Resolve comments

[rriddle]

Formatting

[mehdi_amini]

Awesome!

mlir/include/mlir/Dialect/Affine/IR/AffineOps.td

def AffineIfOp : Affine_Op<"if",
   ];
 
   let extraClassDeclaration = [{
     static StringRef getConditionAttrName() { return "condition"; }
 
     IntegerSet getIntegerSet();
     void setIntegerSet(IntegerSet newSet);
 
-    /// Sets the integer set with its operands. The size of 'operands' must not
-    /// exceed the current number of operands for this instance, as the operands
-    /// list of AffineIf is not resizable.
+    /// Sets the integer set with its operands.
     void setConditional(IntegerSet set, ValueRange operands);
 
     /// Returns true if an else block exists.
     bool hasElse() { return !elseRegion().empty(); }
 
     Block *getThenBlock() {
       assert(!thenRegion().empty() && "Unexpected empty 'then' region.");
       return &thenRegion().front();

mlir/include/mlir/IR/OpBase.td

class HasParent<string op>
     : ParamNativeOpTrait<"HasParent", op>;
 
 // Op result type is derived from the first attribute. If the attribute is an
 // subclass of `TypeAttrBase`, its value is used, otherwise, the type of the
 // attribute content is used.
 def FirstAttrDerivedResultType :
   GenInternalOpTrait<"FirstAttrDerivedResultType">;
 
-// Op has a resizable operand list. Auto-generated build and parse functions
-// should construct it as such.
-def ResizableOperandList : GenInternalOpTrait<"ResizableOperandList">;
-
 // TODO(antiagainst): Turn the following into normal traits and generate
 // verification for them.
 
 // All variadic operands of the op have the same number of values.
 // A variadic operand contains an array of values whose array size is only
 // known at runtime. This trait requires all variadic operands of an op
 // to have the same array size.
 def SameVariadicOperandSize : GenInternalOpTrait<"SameVariadicOperandSize">;

mlir/include/mlir/IR/Operation.h

: public llvm::ilist_node_with_parent<Operation, Block>,
       private llvm::TrailingObjects<Operation, detail::InLineOpResult,
                                     detail::TrailingOpResult, BlockOperand,
                                     Region, detail::OperandStorage> {
 public:
   /// Create a new Operation with the specific fields.
   static Operation *create(Location location, OperationName name,
                            ArrayRef<Type> resultTypes, ArrayRef<Value> operands,
                            ArrayRef<NamedAttribute> attributes,
-                           ArrayRef<Block *> successors, unsigned numRegions,
-                           bool resizableOperandList);
+                           ArrayRef<Block *> successors, unsigned numRegions);
 
   /// Overload of create that takes an existing NamedAttributeList to avoid
   /// unnecessarily uniquing a list of attributes.
   static Operation *create(Location location, OperationName name,
                            ArrayRef<Type> resultTypes, ArrayRef<Value> operands,
                            NamedAttributeList attributes,
-                           ArrayRef<Block *> successors, unsigned numRegions,
-                           bool resizableOperandList);
+                           ArrayRef<Block *> successors, unsigned numRegions);
 
   /// Create a new Operation from the fields stored in `state`.
   static Operation *create(const OperationState &state);
 
   /// Create a new Operation with the specific fields.
   static Operation *create(Location location, OperationName name,
                            ArrayRef<Type> resultTypes, ArrayRef<Value> operands,
                            NamedAttributeList attributes,
                            ArrayRef<Block *> successors = {},
-                           RegionRange regions = {},
-                           bool resizableOperandList = false);
+                           RegionRange regions = {});
 
   /// The name of an operation is the key identifier for it.
   OperationName getName() { return name; }
 
   /// If this operation has a registered operation description, return it.
   /// Otherwise return null.
   const AbstractOperation *getAbstractOperation() {
     return getName().getAbstractOperation();
public:
   void print(raw_ostream &os, AsmState &state,
              OpPrintingFlags flags = llvm::None);
   void dump();
 
   //===--------------------------------------------------------------------===//
   // Operands
   //===--------------------------------------------------------------------===//
 
-  /// Returns if the operation has a resizable operation list, i.e. operands can
-  /// be added.
-  bool hasResizableOperandsList() { return getOperandStorage().isResizable(); }
-
   /// Replace the current operands of this operation with the ones provided in
-  /// 'operands'. If the operands list is not resizable, the size of 'operands'
-  /// must be less than or equal to the current number of operands.
+  /// 'operands'.
   void setOperands(ValueRange operands);
 
   unsigned getNumOperands() { return getOperandStorage().size(); }
 
   Value getOperand(unsigned idx) { return getOpOperand(idx).get(); }
   void setOperand(unsigned idx, Value value) {
     return getOpOperand(idx).set(value);
   }

mlir/include/mlir/IR/OperationSupport.h

struct OperationState {
   SmallVector<Value, 4> operands;
   /// Types of the results of this operation.
   SmallVector<Type, 4> types;
   SmallVector<NamedAttribute, 4> attributes;
   /// Successors of this operation and their respective operands.
   SmallVector<Block *, 1> successors;
   /// Regions that the op will hold.
   SmallVector<std::unique_ptr<Region>, 1> regions;
-  /// If the operation has a resizable operand list.
-  bool resizableOperandList = false;
 
 public:
   OperationState(Location location, StringRef name);
 
   OperationState(Location location, OperationName name);
 
   OperationState(Location location, StringRef name, ValueRange operands,
                  ArrayRef<Type> types, ArrayRef<NamedAttribute> attributes,
                  ArrayRef<Block *> successors = {},
-                 MutableArrayRef<std::unique_ptr<Region>> regions = {},
-                 bool resizableOperandList = false);
+                 MutableArrayRef<std::unique_ptr<Region>> regions = {});
 
   void addOperands(ValueRange newOperands);
 
   void addTypes(ArrayRef<Type> newTypes) {
     types.append(newTypes.begin(), newTypes.end());
   }
   template <typename RangeT>
   std::enable_if_t<!std::is_convertible<RangeT, ArrayRef<Type>>::value>
public:
   /// created.  When it is, the region bodies will be transferred.
   Region *addRegion();
 
   /// Take a region that should be attached to the Operation.  The body of the
   /// region will be transferred when the Operation is constructed.  If the
   /// region is null, a new empty region will be attached to the Operation.
   void addRegion(std::unique_ptr<Region> &&region);
 
-  /// Sets the operand list of the operation as resizable.
-  void setOperandListToResizable(bool isResizable = true) {
-    resizableOperandList = isResizable;
-  }
-
   /// Get the context held by this operation state.
   MLIRContext *getContext() { return location->getContext(); }
 };
 
 //===----------------------------------------------------------------------===//
 // OperandStorage
 //===----------------------------------------------------------------------===//
 
 namespace detail {
-/// A utility class holding the information necessary to dynamically resize
-/// operands.
-struct ResizableStorage {
-  ResizableStorage() : firstOp(nullptr), capacity(0) {}
-  ResizableStorage(ResizableStorage &&other)
-      : firstOp(other.firstOp), capacity(other.capacity) {
-    other.firstOp = nullptr;
-  }
-  ~ResizableStorage() { cleanupStorage(); }
-
-  /// Cleanup any allocated storage.
-  void cleanupStorage() { free(firstOp); }
-
-  /// Sets the storage pointer to a new dynamically allocated block.
-  void setDynamicStorage(OpOperand *opBegin) {
-    /// Cleanup the old storage if necessary.
-    cleanupStorage();
-    firstOp = opBegin;
-  }
-
-  /// A pointer to the first operand element in a dynamically allocated block.
-  OpOperand *firstOp;
-
-  /// The maximum number of operands that can be currently held by the storage.
-  unsigned capacity;
+/// A utility class representing the information for an inline operand storage.
+struct InlineOperandStorage {
+  /// The number of operands within the storage.
+  unsigned numOperands;
+  /// The total capacity number of operands that the storage can hold.
+  unsigned capacity : 31;
+  /// We reserve a range of bits for use by the operand storage.
+  unsigned reserved : 1;
 };
 
 /// This class handles the management of operation operands. Operands are
-/// stored similarly to the elements of a SmallVector except for two key
-/// differences. The first is the inline storage, which is a trailing objects
-/// array. The second is that being able to dynamically resize the operand list
-/// is optional.
+/// stored either in a trailing array, or a dynamically resizable vector.
 class OperandStorage final
     : private llvm::TrailingObjects<OperandStorage, OpOperand> {
 public:
-  OperandStorage(unsigned numOperands, bool resizable)
-      : numOperands(numOperands), resizable(resizable),
-        storageIsDynamic(false) {}
-
-  ~OperandStorage() {
-    // Manually destruct the operands.
-    for (auto &operand : getOperands())
-      operand.~OpOperand();
-
-    // If the storage is currently in a dynamic allocation, then destruct the
-    // resizable storage.
-    if (storageIsDynamic)
-      getResizableStorage().~ResizableStorage();
-  }
+  OperandStorage(Operation *owner, ValueRange operands);
+  ~OperandStorage();
 
   /// Replace the operands contained in the storage with the ones provided in
   /// 'operands'.
   void setOperands(Operation *owner, ValueRange operands);
 
   /// Erase an operand held by the storage.
   void eraseOperand(unsigned index);
 
   /// Get the operation operands held by the storage.
   MutableArrayRef<OpOperand> getOperands() {
-    return {getRawOperands(), size()};
+    return LLVM_UNLIKELY(isDynamicStorage())
+               ? MutableArrayRef<OpOperand>(getDynamicStorage())
+               : getInlineOperands();
   }
 
   /// Return the number of operands held in the storage.
-  unsigned size() const { return numOperands; }
+  unsigned size() {
+    return LLVM_UNLIKELY(isDynamicStorage()) ? getDynamicStorage().size()
+                                             : getInlineStorage().numOperands;
+  }
 
   /// Returns the additional size necessary for allocating this object.
-  static size_t additionalAllocSize(unsigned numOperands, bool resizable) {
-    // The trailing storage must be able to hold enough space for the number of
-    // operands, or at least the resizable storage if necessary.
-    return std::max(additionalSizeToAlloc<OpOperand>(numOperands),
-                    sizeof(ResizableStorage));
+  static size_t additionalAllocSize(unsigned numOperands) {
+    return additionalSizeToAlloc<OpOperand>(numOperands);
   }
 
-  /// Returns if this storage is resizable.
-  bool isResizable() const { return resizable; }
-
 private:
-  /// Clear the storage and destroy the current operands held by the storage.
-  void clear() { numOperands = 0; }
+  enum : uint64_t {
+    /// The bit used to mark the storage as dynamic.
+    DynamicStorageBit = 1ull << 63ull
+  };
 
-  /// Returns the current pointer for the raw operands array.
-  OpOperand *getRawOperands() {
-    return storageIsDynamic ? getResizableStorage().firstOp
-                            : getTrailingObjects<OpOperand>();
+  /// Returns the inline storage if the storage is current not dynamic.
+  InlineOperandStorage &getInlineStorage() {
+    assert(!isDynamicStorage() && "expected storage to be inline");
+    static_assert(sizeof(InlineOperandStorage) <= sizeof(uint64_t),
+                  "inline storage representation must fit within the opaque "
+                  "representation");
+    return *reinterpret_cast<InlineOperandStorage *>(&representation);
   }
 
-  /// Returns the resizable operand utility class.
-  ResizableStorage &getResizableStorage() {
-    // We represent the resizable storage in the same location as the first
-    // operand.
-    assert(storageIsDynamic);
-    return *reinterpret_cast<ResizableStorage *>(
-        getTrailingObjects<OpOperand>());
+  /// Returns the operands that are trailing this storage.
+  MutableArrayRef<OpOperand> getInlineOperands() {
+    return {getTrailingObjects<OpOperand>(), getInlineStorage().numOperands};
   }
 
-  /// Grow the internal resizable operand storage.
-  void grow(ResizableStorage &resizeUtil, size_t minSize);
-
-  /// The current number of operands, and the current max operand capacity.
-  unsigned numOperands : 30;
+  /// Returns the dynamic storage container.
+  std::vector<OpOperand> &getDynamicStorage() {
+    assert(isDynamicStorage() && "expected dynamic storage");
+    uint64_t maskedRepresentation = representation & ~DynamicStorageBit;
+    return *reinterpret_cast<std::vector<OpOperand> *>(maskedRepresentation);
+  }
 
-  /// Whether this storage is resizable or not.
-  bool resizable : 1;
+  /// Returns true if the storage is currently dynamic.
+  bool isDynamicStorage() const { return representation & DynamicStorageBit; }
 
-  /// If the storage is resizable, this indicates if the operands array is
-  /// currently in the resizable storage or the trailing array.
-  bool storageIsDynamic : 1;
+  /// The current representation of the storage. This is either a
+  /// InlineOperandStorage structured, or a pointer to a vector of OpOperand.
+  uint64_t representation;
 
   /// This stuff is used by the TrailingObjects template.
   friend llvm::TrailingObjects<OperandStorage, OpOperand>;
 };
 } // end namespace detail
 
 //===----------------------------------------------------------------------===//
 // ResultStorage

mlir/include/mlir/IR/UseDefLists.h

 template <typename DerivedT, typename IRValueTy> class IROperand {
 public:
   using ValueType = IRValueTy;
 
   IROperand(Operation *owner) : owner(owner) {}
   IROperand(Operation *owner, ValueType value) : value(value), owner(owner) {
     insertIntoCurrent();
   }
+  IROperand(const IROperand<DerivedT, IRValueTy> &other) : owner(other.owner) {
+    // Only allow using the copy constructor if the other operand doesn't have a
+    // valid value.
+    assert(!other.value && "cannot construct from an operand with a value");
+  }
 
   /// Return the current value being used by this operand.
   ValueType get() const { return value; }
 
   /// Set the current value being used by this operand.
   void set(ValueType newValue) {
     // It isn't worth optimizing for the case of switching operands on a single
     // value.
private:
   DerivedT *nextUse = nullptr;
 
   /// This points to the previous link in the use-chain.
   DerivedT **back = nullptr;
 
   /// The operation owner of this operand.
   Operation *const owner;
 
-  /// Operands are not copyable or assignable.
-  IROperand(const IROperand &use) = delete;
+  /// Operands are not assignable.
   IROperand &operator=(const IROperand &use) = delete;
 
   void removeFromCurrent() {
     if (!back)
       return;
     *back = nextUse;
     if (nextUse)
       nextUse->back = back;

mlir/include/mlir/TableGen/Operator.h

public:
   StringRef getArgName(int index) const;
   var_decorator_range getArgDecorators(int index) const;
 
   // Returns the trait wrapper for the given MLIR C++ `trait`.
   // TODO: We should add a C++ wrapper class for TableGen OpTrait instead of
   // requiring the raw MLIR trait here.
   const OpTrait *getTrait(llvm::StringRef trait) const;
 
-  // Returns "true" if Op has a ResizableOperandList trait.
-  bool hasResizableOperandList() const;
-
   // Regions.
   using const_region_iterator = const NamedRegion *;
   const_region_iterator region_begin() const;
   const_region_iterator region_end() const;
   llvm::iterator_range<const_region_iterator> getRegions() const;
 
   // Returns the number of regions.
   unsigned getNumRegions() const;

mlir/lib/Dialect/Affine/IR/AffineOps.cpp

void AffineForOp::build(Builder *builder, OperationState &result,
 
   // Create a region and a block for the body.  The argument of the region is
   // the loop induction variable.
   Region *bodyRegion = result.addRegion();
   Block *body = new Block();
   body->addArgument(IndexType::get(builder->getContext()));
   bodyRegion->push_back(body);
   ensureTerminator(*bodyRegion, *builder, result.location);
-
-  // Set the operands list as resizable so that we can freely modify the bounds.
-  result.setOperandListToResizable();
 }
 
 void AffineForOp::build(Builder *builder, OperationState &result, int64_t lb,
                         int64_t ub, int64_t step) {
   auto lbMap = AffineMap::getConstantMap(lb, builder->getContext());
   auto ubMap = AffineMap::getConstantMap(ub, builder->getContext());
   return build(builder, result, {}, lbMap, {}, ubMap, step);
 }
static ParseResult parseAffineForOp(OpAsmParser &parser,
   // Parse the body region.
   Region *body = result.addRegion();
   if (parser.parseRegion(*body, inductionVariable, builder.getIndexType()))
     return failure();
 
   AffineForOp::ensureTerminator(*body, builder, result.location);
 
   // Parse the optional attribute list.
-  if (parser.parseOptionalAttrDict(result.attributes))
-    return failure();
-
-  // Set the operands list as resizable so that we can freely modify the bounds.
-  result.setOperandListToResizable();
-  return success();
+  return parser.parseOptionalAttrDict(result.attributes);
 }
 
 static void printBound(AffineMapAttr boundMap,
                        Operation::operand_range boundOperands,
                        const char *prefix, OpAsmPrinter &p) {
   AffineMap map = boundMap.getValue();
 
   // Check if this bound should be printed using custom assembly form.

mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp

static Operation *vectorizeOneOperation(Operation *opInst,
   // Create a clone of the op with the proper operands and return types.
   // TODO(ntv): The following assumes there is always an op with a fixed
   // name that works both in scalar mode and vector mode.
   // TODO(ntv): Is it worth considering an Operation.clone operation which
   // changes the type so we can promote an Operation with less boilerplate?
   OpBuilder b(opInst);
   OperationState newOp(opInst->getLoc(), opInst->getName().getStringRef(),
                        vectorOperands, vectorTypes, opInst->getAttrs(),
-                       /*successors=*/{},
-                       /*regions=*/{}, opInst->hasResizableOperandsList());
+                       /*successors=*/{}, /*regions=*/{});
   return b.createOperation(newOp);
 }
 
 /// Iterates over the forward slice from the loads in the vectorization pattern
 /// and rewrites them using their vectorized counterpart by:
 ///   1. Create the forward slice starting from the loads in the vectorization
 ///   pattern.
 ///   2. Topologically sorts the forward slice.

mlir/lib/IR/Operation.cpp

 // Operation
 //===----------------------------------------------------------------------===//
 
 /// Create a new Operation with the specific fields.
 Operation *Operation::create(Location location, OperationName name,
                              ArrayRef<Type> resultTypes,
                              ArrayRef<Value> operands,
                              ArrayRef<NamedAttribute> attributes,
-                             ArrayRef<Block *> successors, unsigned numRegions,
-                             bool resizableOperandList) {
+                             ArrayRef<Block *> successors,
+                             unsigned numRegions) {
   return create(location, name, resultTypes, operands,
-                NamedAttributeList(attributes), successors, numRegions,
-                resizableOperandList);
+                NamedAttributeList(attributes), successors, numRegions);
 }
 
 /// Create a new Operation from operation state.
 Operation *Operation::create(const OperationState &state) {
   return Operation::create(state.location, state.name, state.types,
                            state.operands, NamedAttributeList(state.attributes),
-                           state.successors, state.regions,
-                           state.resizableOperandList);
+                           state.successors, state.regions);
 }
 
 /// Create a new Operation with the specific fields.
 Operation *Operation::create(Location location, OperationName name,
                              ArrayRef<Type> resultTypes,
                              ArrayRef<Value> operands,
                              NamedAttributeList attributes,
-                             ArrayRef<Block *> successors, RegionRange regions,
-                             bool resizableOperandList) {
+                             ArrayRef<Block *> successors,
+                             RegionRange regions) {
   unsigned numRegions = regions.size();
   Operation *op = create(location, name, resultTypes, operands, attributes,
-                         successors, numRegions, resizableOperandList);
+                         successors, numRegions);
   for (unsigned i = 0; i < numRegions; ++i)
     if (regions[i])
       op->getRegion(i).takeBody(*regions[i]);
   return op;
 }
 
 /// Overload of create that takes an existing NamedAttributeList to avoid
 /// unnecessarily uniquing a list of attributes.
 Operation *Operation::create(Location location, OperationName name,
                              ArrayRef<Type> resultTypes,
                              ArrayRef<Value> operands,
                              NamedAttributeList attributes,
-                             ArrayRef<Block *> successors, unsigned numRegions,
-                             bool resizableOperandList) {
+                             ArrayRef<Block *> successors,
+                             unsigned numRegions) {
   // We only need to allocate additional memory for a subset of results.
   unsigned numTrailingResults = OpResult::getNumTrailing(resultTypes.size());
   unsigned numInlineResults = OpResult::getNumInline(resultTypes.size());
   unsigned numSuccessors = successors.size();
   unsigned numOperands = operands.size();
 
   // Compute the byte size for the operation and the operand storage.
   auto byteSize =
       totalSizeToAlloc<detail::InLineOpResult, detail::TrailingOpResult,
                        BlockOperand, Region, detail::OperandStorage>(
           numInlineResults, numTrailingResults, numSuccessors, numRegions,
           /*detail::OperandStorage*/ 1);
-  byteSize += llvm::alignTo(detail::OperandStorage::additionalAllocSize(
-                                numOperands, resizableOperandList),
-                            alignof(Operation));
+  byteSize +=
+      llvm::alignTo(detail::OperandStorage::additionalAllocSize(numOperands),
   void *rawMem = malloc(byteSize);
 
   // Create the new Operation.
   auto op = ::new (rawMem) Operation(location, name, resultTypes, numSuccessors,
                                      numRegions, attributes);
 
   assert((numSuccessors == 0 || !op->isKnownNonTerminator()) &&
          "unexpected successors in a non-terminator operation");
 
   // Initialize the results.
   for (unsigned i = 0; i < numInlineResults; ++i)
     new (op->getInlineResult(i)) detail::InLineOpResult();
   for (unsigned i = 0; i < numTrailingResults; ++i)
     new (op->getTrailingResult(i)) detail::TrailingOpResult(i);
 
   // Initialize the regions.
   for (unsigned i = 0; i != numRegions; ++i)
     new (&op->getRegion(i)) Region(op);
 
   // Initialize the operands.
-  new (&op->getOperandStorage())
-      detail::OperandStorage(numOperands, resizableOperandList);
-  auto opOperands = op->getOpOperands();
-  for (unsigned i = 0; i != numOperands; ++i)
-    new (&opOperands[i]) OpOperand(op, operands[i]);
+  new (&op->getOperandStorage()) detail::OperandStorage(op, operands);
 
   // Initialize the successors.
   auto blockOperands = op->getBlockOperands();
   for (unsigned i = 0; i != numSuccessors; ++i)
     new (&blockOperands[i]) BlockOperand(op, successors[i]);
 
   return op;
 }
void Operation::replaceUsesOfWith(Value from, Value to) {
   if (from == to)
     return;
   for (auto &operand : getOpOperands())
     if (operand.get() == from)
       operand.set(to);
 }
 
 /// Replace the current operands of this operation with the ones provided in
-/// 'operands'. If the operands list is not resizable, the size of 'operands'
-/// must be less than or equal to the current number of operands.
+/// 'operands'.
 void Operation::setOperands(ValueRange operands) {
   getOperandStorage().setOperands(this, operands);
 }
 
 //===----------------------------------------------------------------------===//
 // Diagnostics
 //===----------------------------------------------------------------------===//
 
Operation *Operation::cloneWithoutRegions(BlockAndValueMapping &mapper) {
 
   // Remap the successors.
   successors.reserve(getNumSuccessors());
   for (Block *successor : getSuccessors())
     successors.push_back(mapper.lookupOrDefault(successor));
 
   // Create the new operation.
   auto *newOp = Operation::create(getLoc(), getName(), getResultTypes(),
-                                  operands, attrs, successors, getNumRegions(),
-                                  hasResizableOperandsList());
+                                  operands, attrs, successors, getNumRegions());
 
   // Remember the mapping of any results.
   for (unsigned i = 0, e = getNumResults(); i != e; ++i)
     mapper.map(getResult(i), newOp->getResult(i));
 
   return newOp;
 }
 

mlir/lib/IR/OperationSupport.cpp

 
 OperationState::OperationState(Location location, OperationName name)
     : location(location), name(name) {}
 
 OperationState::OperationState(Location location, StringRef name,
                                ValueRange operands, ArrayRef<Type> types,
                                ArrayRef<NamedAttribute> attributes,
                                ArrayRef<Block *> successors,
-                               MutableArrayRef<std::unique_ptr<Region>> regions,
-                               bool resizableOperandList)
+                               MutableArrayRef<std::unique_ptr<Region>> regions)
     : location(location), name(name, location->getContext()),
       operands(operands.begin(), operands.end()),
       types(types.begin(), types.end()),
       attributes(attributes.begin(), attributes.end()),
       successors(successors.begin(), successors.end()) {
   for (std::unique_ptr<Region> &r : regions)
     this->regions.push_back(std::move(r));
 }

 void OperationState::addRegion(std::unique_ptr<Region> &&region) {
   regions.push_back(std::move(region));
 }
 
 //===----------------------------------------------------------------------===//
 // OperandStorage
 //===----------------------------------------------------------------------===//
 
+detail::OperandStorage::OperandStorage(Operation *owner, ValueRange operands)
+    : representation(0) {
+  auto &inlineStorage = getInlineStorage();
+  inlineStorage.numOperands = inlineStorage.capacity = operands.size();
+  auto *operandPtrBegin = getTrailingObjects<OpOperand>();
+  for (unsigned i = 0, e = inlineStorage.numOperands; i < e; ++i)
+    new (&operandPtrBegin[i]) OpOperand(owner, operands[i]);
+}
+
+detail::OperandStorage::~OperandStorage() {
+  // If the storage is dynamic, simply delete the dynamic storage container.
+  if (isDynamicStorage()) {
+    delete &getDynamicStorage();
+    return;
+  }
+
+  // Otherwise, manually destruct the trailing operands.
+  InlineOperandStorage &storage = getInlineStorage();
+  ArrayRef<OpOperand> trailingOperands(getTrailingObjects<OpOperand>(),
+                                       storage.numOperands);
+  for (auto &operand : trailingOperands)
+    operand.~OpOperand();
+}
+
 /// Replace the operands contained in the storage with the ones provided in
 /// 'operands'.
 void detail::OperandStorage::setOperands(Operation *owner,
                                          ValueRange operands) {
-  // If the number of operands is less than or equal to the current amount, we
-  // can just update in place.
-  if (operands.size() <= numOperands) {
-    auto opOperands = getOperands();
+  // If the storage is dynamic, simply set update the operand list.
+  if (LLVM_UNLIKELY(isDynamicStorage())) {
+    auto &dynamicStorage = getDynamicStorage();
+    dynamicStorage.resize(operands.size(), OpOperand(owner));
+    for (unsigned i = 0, e = operands.size(); i != e; ++i)
+      dynamicStorage[i].set(operands[i]);
+    return;
+  }
 
+  // Otherwise, the storage is the trailing array. If the number of operands is
+  // less than or equal to the current amount, we can just update in place.
+  InlineOperandStorage &inlineStorage = getInlineStorage();
+  unsigned &numOperands = inlineStorage.numOperands;
+  if (operands.size() <= numOperands) {
     // If the number of new operands is less than the current count, then remove
     // any extra operands.
+    MutableArrayRef<OpOperand> inlineOperands = getInlineOperands();
     for (unsigned i = operands.size(); i != numOperands; ++i)
-      opOperands[i].~OpOperand();
+      inlineOperands[i].~OpOperand();
 
     // Set the operands in place.
     numOperands = operands.size();
     for (unsigned i = 0; i != numOperands; ++i)
-      opOperands[i].set(operands[i]);
+      inlineOperands[i].set(operands[i]);
     return;
   }
 
-  // Otherwise, we need to be resizable.
-  assert(resizable && "Only resizable operations may add operands");
-
-  // If the storage isn't already dynamic, we need to allocate a new buffer for
-  // it.
-  OpOperand *opBegin = nullptr;
-  if (!storageIsDynamic) {
-    // Grow a new storage first to avoid overwriting the existing operands.
-    ResizableStorage newStorage;
-    grow(newStorage, operands.size());
-    opBegin = newStorage.firstOp;
-    storageIsDynamic = true;
-    new (&getResizableStorage()) ResizableStorage(std::move(newStorage));
-  } else {
-    // Otherwise, grow the existing storage if necessary.
-    auto &resizeUtil = getResizableStorage();
-    if (resizeUtil.capacity < operands.size())
-      grow(resizeUtil, operands.size());
-    opBegin = resizeUtil.firstOp;
-  }
-
-  // Set the operands.
+  // If the new operand count is within the original inline capacity, we can
+  // grow inplace.
+  if (operands.size() <= inlineStorage.capacity) {
+    OpOperand *opBegin = getTrailingObjects<OpOperand>();
     for (unsigned i = 0; i != numOperands; ++i)
       opBegin[i].set(operands[i]);
     for (unsigned e = operands.size(); numOperands != e; ++numOperands)
       new (&opBegin[numOperands]) OpOperand(owner, operands[numOperands]);
+    return;
+  }
+
+  // Otherwise, we need to change the storage to dynamic.
+  MutableArrayRef<OpOperand> inlineOperands = getInlineOperands();
+  auto *dynamicStorage = new std::vector<OpOperand>(
+      std::make_move_iterator(inlineOperands.begin()),
+      std::make_move_iterator(inlineOperands.end()));
+
+  // Destroy the original operands.
+  for (auto &operand : inlineOperands)
+    operand.~OpOperand();
+
+  // Update the dynamic storage with the new operands.
+  dynamicStorage->resize(operands.size(), OpOperand(owner));
+  for (unsigned i = 0, e = operands.size(); i != e; ++i)
+    (*dynamicStorage)[i].set(operands[i]);
+
+  // Update the storage representation to use the new dynamic storage.
+  representation = reinterpret_cast<intptr_t>(dynamicStorage);
+  representation |= DynamicStorageBit;
 }
 
 /// Erase an operand held by the storage.
 void detail::OperandStorage::eraseOperand(unsigned index) {
-  assert(index < size());
-  auto operands = getOperands();
-  --numOperands;
+  if (LLVM_UNLIKELY(isDynamicStorage())) {
+    auto &dynamicStorage = getDynamicStorage();
+    dynamicStorage.erase(dynamicStorage.begin() + index);
+    return;
+  }
 
-  // Shift all operands down by 1 if the operand to remove is not at the end.
-  auto indexIt = std::next(operands.begin(), index);
-  if (index != numOperands)
-    std::rotate(indexIt, std::next(indexIt), operands.end());
-  operands[numOperands].~OpOperand();
-}
-
-/// Grow the internal operand storage.
-void detail::OperandStorage::grow(ResizableStorage &resizeUtil,
-                                  size_t minSize) {
-  // Allocate a new storage array.
-  resizeUtil.capacity =
-      std::max(size_t(llvm::NextPowerOf2(resizeUtil.capacity + 2)), minSize);
-  OpOperand *newStorage = static_cast<OpOperand *>(
-      llvm::safe_malloc(resizeUtil.capacity * sizeof(OpOperand)));
-
-  // Move the current operands to the new storage.
-  auto operands = getOperands();
-  std::uninitialized_copy(std::make_move_iterator(operands.begin()),
-                          std::make_move_iterator(operands.end()), newStorage);
+  InlineOperandStorage &inlineStorage = getInlineStorage();
+  MutableArrayRef<OpOperand> inlineOperands = getInlineOperands();
+  assert(index < inlineOperands.size());
+  --inlineStorage.numOperands;
 
-  // Destroy the original operands and update the resizable storage pointer.
-  for (auto &operand : operands)
-    operand.~OpOperand();
-  resizeUtil.setDynamicStorage(newStorage);
+  // Shift all operands down by 1 if the operand to remove is not at the end.
+  if (index != inlineStorage.numOperands) {
+    auto indexIt = std::next(inlineOperands.begin(), index);
+    std::rotate(indexIt, std::next(indexIt), inlineOperands.end());
+  }
+  inlineOperands.back().~OpOperand();
 }
 
 //===----------------------------------------------------------------------===//
 // ResultStorage
 //===----------------------------------------------------------------------===//
 
 /// Returns the parent operation of this trailing result.
 Operation *detail::TrailingOpResult::getOwner() {

mlir/lib/Parser/Parser.cpp

Value OperationParser::createForwardRefPlaceholder(SMLoc loc, Type type) {
   // something with a def/use chain.
   //
   // We create these placeholders as having an empty name, which we know
   // cannot be created through normal user input, allowing us to distinguish
   // them.
   auto name = OperationName("placeholder", getContext());
   auto *op = Operation::create(
       getEncodedSourceLocation(loc), name, type, /*operands=*/{},
-      /*attributes=*/llvm::None, /*successors=*/{}, /*numRegions=*/0,
-      /*resizableOperandList=*/false);
+      /*attributes=*/llvm::None, /*successors=*/{}, /*numRegions=*/0);
   forwardRefPlaceholders[op->getResult(0)] = loc;
   return op->getResult(0);
 }
 
 //===----------------------------------------------------------------------===//
 // Operation Parsing
 //===----------------------------------------------------------------------===//
 
if (name.empty())
     return (emitError("empty operation name is invalid"), nullptr);
   if (name.find('\0') != StringRef::npos)
     return (emitError("null character not allowed in operation name"), nullptr);
 
   consumeToken(Token::string);
 
   OperationState result(srcLocation, name);
 
-  // Generic operations have a resizable operation list.
-  result.setOperandListToResizable();
-
   // Parse the operand list.
   SmallVector<SSAUseInfo, 8> operandInfos;
   if (parseToken(Token::l_paren, "expected '(' to start operand list") ||
       parseOptionalSSAUseList(operandInfos) ||
       parseToken(Token::r_paren, "expected ')' to end operand list")) {
     return nullptr;
   }
 

mlir/lib/TableGen/Operator.cpp

for (const auto &t : traits) {
     } else if (auto opTrait = dyn_cast<tblgen::InterfaceOpTrait>(&t)) {
       if (opTrait->getTrait() == trait)
         return opTrait;
     }
   }
   return nullptr;
 }
 
-bool tblgen::Operator::hasResizableOperandList() const {
-  return getTrait("OpTrait::ResizableOperandList") != nullptr;
-}
-
 auto tblgen::Operator::region_begin() const -> const_region_iterator {
   return regions.begin();
 }
 auto tblgen::Operator::region_end() const -> const_region_iterator {
   return regions.end();
 }
 auto tblgen::Operator::getRegions() const
     -> llvm::iterator_range<const_region_iterator> {

mlir/lib/Transforms/Utils/Utils.cpp

LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   canonicalizeMapAndOperands(&newMap, &newMapOperands);
   // Remove any affine.apply's that became dead as a result of composition.
   for (auto value : affineApplyOps)
     if (value.use_empty())
       value.getDefiningOp()->erase();
 
   // Construct the new operation using this memref.
   OperationState state(op->getLoc(), op->getName());
-  state.setOperandListToResizable(op->hasResizableOperandsList());
   state.operands.reserve(op->getNumOperands() + extraIndices.size());
   // Insert the non-memref operands.
   state.operands.append(op->operand_begin(),
                         op->operand_begin() + memRefOperandPos);
   // Insert the new memref value.
   state.operands.push_back(newMemRef);
 
   // Insert the new memref map operands.

mlir/test/mlir-tblgen/op-operand.td

 
 // CHECK-LABEL: Value OpD::input2
 // CHECK-NEXT: return *getODSOperands(1).begin();
 
 // CHECK-LABEL: OpD::build
 // CHECK-NEXT: odsState.addOperands(input1);
 // CHECK-NEXT: odsState.addOperands(input2);
 // CHECK-NEXT: odsState.addOperands(input3);
-// CHECK-NOT: odsState.setOperandListToResizable
-
-// Check that resizable operand list flag is set up correctly in all generated
-// builders and in the parser.
-def OpE : NS_Op<"resizable_operand_list", [ResizableOperandList]> {
-  let arguments = (ins Variadic<AnyType>:$input);
-  let assemblyFormat = "$input attr-dict `:` type($input)";
-}
-
-// CHECK-LABEL: OpE::build(Builder *odsBuilder, OperationState &odsState, ValueRange
-// CHECK: odsState.setOperandListToResizable()
-
-// CHECK-LABEL: OpE::build(Builder *odsBuilder, OperationState &odsState, ArrayRef<Type>
-// CHECK: odsState.setOperandListToResizable()
-
-// CHECK-LABEL: OpE::build(Builder *, OperationState
-// CHECK: odsState.setOperandListToResizable()
-
-// CHECK-LABEL: OpE::parse
-// CHECK: result.setOperandListToResizable()

mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp

std::string params =
       ", ValueRange operands, ArrayRef<NamedAttribute> attributes";
   if (op.getNumVariadicRegions())
     params += ", unsigned numRegions";
   auto &m = opClass.newMethod("void", "build", params, OpMethod::MP_Static);
   auto &body = m.body();
 
   // Operands
   body << "  " << builderOpState << ".addOperands(operands);\n";
-  if (op.hasResizableOperandList())
-    body << formatv("  {0}.setOperandListToResizable();\n\n", builderOpState);
 
   // Attributes
   body << "  " << builderOpState << ".addAttributes(attributes);\n";
 
   // Create the correct number of regions
   if (int numRegions = op.getNumRegions()) {
     body << llvm::formatv(
         "  for (unsigned i = 0; i != {0}; ++i)\n",
void OpEmitter::genUseAttrAsResultTypeBuilder() {
   if (namedAttr.attr.isTypeAttr()) {
     resultType = "attr.second.cast<TypeAttr>().getValue()";
   } else {
     resultType = "attr.second.getType()";
   }
 
   // Operands
   body << "  " << builderOpState << ".addOperands(operands);\n";
-  if (op.hasResizableOperandList())
-    body << formatv("  {0}.setOperandListToResizable();\n\n", builderOpState);
 
   // Attributes
   body << "  " << builderOpState << ".addAttributes(attributes);\n";
 
   // Result types
   SmallVector<std::string, 2> resultTypes(op.getNumResults(), resultType);
   body << "    " << builderOpState << ".addTypes({"
        << llvm::join(resultTypes, ", ") << "});\n";
void OpEmitter::genCollectiveParamBuilder() {
 
   // Operands
   if (numVariadicOperands == 0 || numNonVariadicOperands != 0)
     body << "  assert(operands.size()"
          << (numVariadicOperands != 0 ? " >= " : " == ")
          << numNonVariadicOperands
          << "u && \"mismatched number of parameters\");\n";
   body << "  " << builderOpState << ".addOperands(operands);\n";
-  if (op.hasResizableOperandList())
-    body << formatv("  {0}.setOperandListToResizable();\n\n", builderOpState);
 
   // Attributes
   body << "  " << builderOpState << ".addAttributes(attributes);\n";
 
   // Create the correct number of regions
   if (int numRegions = op.getNumRegions()) {
     body << llvm::formatv(
         "  for (unsigned i = 0; i != {0}; ++i)\n",
void OpEmitter::genCodeForAddingArgAndRegionForBuilder(OpMethodBody &body,
                                                        bool isRawValueAttr) {
   // Push all operands to the result.
   for (int i = 0, e = op.getNumOperands(); i < e; ++i) {
     std::string argName = getArgumentName(op, i);
     if (op.getOperand(i).isOptional())
       body << "  if (" << argName << ")\n  ";
     body << "  " << builderOpState << ".addOperands(" << argName << ");\n";
   }
-  if (op.hasResizableOperandList())
-    body << formatv("  {0}.setOperandListToResizable();\n", builderOpState);
 
   // If the operation has the operand segment size attribute, add it here.
   if (op.getTrait("OpTrait::AttrSizedOperandSegments")) {
     body << "  " << builderOpState
          << ".addAttribute(\"operand_segment_sizes\", "
             "odsBuilder->getI32VectorAttr({";
     interleaveComma(llvm::seq<int>(0, op.getNumOperands()), body, [&](int i) {
       if (op.getOperand(i).isOptional())

mlir/tools/mlir-tblgen/OpFormatGen.cpp

for (auto &element : elements)
     genElementParser(element.get(), body, attrTypeCtx);
 
   // Generate the code to resolve the operand/result types and successors now
   // that they have been parsed.
   genParserTypeResolution(op, body);
   genParserSuccessorResolution(op, body);
   genParserVariadicSegmentResolution(op, body);
 
-  // Mark the operation as having resizable operand list if required.
-  if (op.hasResizableOperandList())
-    body << "  result.setOperandListToResizable();\n";
-
   body << "  return success();\n";
 }
 
 void OperationFormat::genParserTypeResolution(Operator &op,
                                               OpMethodBody &body) {
   // If any of type resolutions use transformed variables, make sure that the
   // types of those variables are resolved.
   SmallPtrSet<const NamedTypeConstraint *, 8> verifiedVariables;

mlir/unittests/IR/OperationSupportTest.cpp

 //===- OperationSupportTest.cpp - Operation support unit tests ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/StandardTypes.h"
 #include "gtest/gtest.h"
 
 using namespace mlir;
 using namespace mlir::detail;
 
-static Operation *createOp(MLIRContext *context, bool resizableOperands,
+static Operation *createOp(MLIRContext *context,
                            ArrayRef<Value> operands = llvm::None,
                            ArrayRef<Type> resultTypes = llvm::None) {
   context->allowUnregisteredDialects();
-  return Operation::create(
-      UnknownLoc::get(context), OperationName("foo.bar", context), resultTypes,
-      operands, llvm::None, llvm::None, 0, resizableOperands);
+  return Operation::create(UnknownLoc::get(context),
+                           OperationName("foo.bar", context), resultTypes,
+                           operands, llvm::None, llvm::None, 0);
 }
 
 namespace {
 TEST(OperandStorageTest, NonResizable) {
   MLIRContext context;
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*resizableOperands=*/false, /*operands=*/llvm::None,
-               builder.getIntegerType(16));
+      createOp(&context, /*operands=*/llvm::None, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a non-resizable operation with one operand.
-  Operation *user = createOp(&context, /*resizableOperands=*/false, operand,
-                             builder.getIntegerType(16));
-
-  // Sanity check the storage.
-  EXPECT_EQ(user->hasResizableOperandsList(), false);
+  Operation *user = createOp(&context, operand, builder.getIntegerType(16));
 
   // The same number of operands is okay.
   user->setOperands(operand);
   EXPECT_EQ(user->getNumOperands(), 1u);
 
   // Removing is okay.
   user->setOperands(llvm::None);
   EXPECT_EQ(user->getNumOperands(), 0u);
 
   // Destroy the operations.
   user->destroy();
   useOp->destroy();
 }
 
-TEST(OperandStorageDeathTest, AddToNonResizable) {
-  MLIRContext context;
-  Builder builder(&context);
-
-  Operation *useOp =
-      createOp(&context, /*resizableOperands=*/false, /*operands=*/llvm::None,
-               builder.getIntegerType(16));
-  Value operand = useOp->getResult(0);
-
-  // Create a non-resizable operation with one operand.
-  Operation *user = createOp(&context, /*resizableOperands=*/false, operand,
-                             builder.getIntegerType(16));
-
-  // Sanity check the storage.
-  EXPECT_EQ(user->hasResizableOperandsList(), false);
-
-  // Adding operands to a non resizable operation should result in a failure.
-  ASSERT_DEATH(user->setOperands({operand, operand}), "");
-}
-
 TEST(OperandStorageTest, Resizable) {
   MLIRContext context;
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*resizableOperands=*/false, /*operands=*/llvm::None,
-               builder.getIntegerType(16));
+      createOp(&context, /*operands=*/llvm::None, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a resizable operation with one operand.
-  Operation *user = createOp(&context, /*resizableOperands=*/true, operand,
-                             builder.getIntegerType(16));
-
-  // Sanity check the storage.
-  EXPECT_EQ(user->hasResizableOperandsList(), true);
+  Operation *user = createOp(&context, operand, builder.getIntegerType(16));
 
   // The same number of operands is okay.
   user->setOperands(operand);
   EXPECT_EQ(user->getNumOperands(), 1u);
 
   // Removing is okay.
   user->setOperands(llvm::None);
   EXPECT_EQ(user->getNumOperands(), 0u);