[flang][hlfir] Add hlfir.destroy operation.

[jeanPerier]

Summary

Add the operation to mark the end of life of hlfir.expr.
As described in its description this is the easiest solution
to deploy given lowering "knows" where expression value are last
used.
However, inserting these points in lowering will probably make
it harder to do some IR transformation that would move the code
using or creating hlfir.expr (no use should be moved after an
hlfir.destroy).
Once the dust settle with the HLFIR change, it will be worth assessing
the situation and see if an analysis could do a better and safer job at
finding those destruction points.

Depends on D141832

[clementval]

LG other than small nits.

flang/include/flang/Optimizer/HLFIR/HLFIROps.td

507

There might be a word missing here or one that should not be here but the sentence is a bit weird to read.

508

flang/test/HLFIR/destroy-codegen.fir

2

flang/include/flang/Optimizer/HLFIR/HLFIROps.td

let extraClassDeclaration = [{
     mlir::Value getShape() const {return mlir::Value{};}
     mlir::OperandRange getExplicitTypeParams() const {
       // Return an empty range.
       return {(*this)->getOperands().begin(), (*this)->getOperands().begin()};
     }
   }];
 }
 
+def hlfir_DestroyOp : hlfir_Op<"destroy", []> {
+  let summary = "Mark the last use of an hlfir.expr";
+  let description = [{
+    Mark the last use of an hlfir.expr. This will be the point at which the
+    buffer of an hlfir.expr, if any, will be deallocated if it was heap
+    allocated.
+    It is not required to create an hlfir.destroy operation for and hlfir.expr
+    created inside an hlfir.elemental an returned in the hlfir.yield_element.
+    The last use of such expression is implicit and an hlfir.destroy could
+    not be emitted after the hlfir.yield_element since it is a terminator.
+
+    Note that hlfir.destroy are currently generated by Fortran lowering that
+    has a good view of the expression use contexts, but this will need to be
+    revisited if any motion of hlfir.expr is done (like CSE) since
+    transformations should not introduce any hlfir.expr usages after an
+    hlfir.destroy.
+    The future will probably be to identify the last use points automatically
+    in bufferization instead.
+  }];
+
+  let arguments = (ins hlfir_ExprType:$expr);
+
+  let assemblyFormat = "$expr attr-dict `:` qualified(type($expr))";
+}
+
 #endif // FORTRAN_DIALECT_HLFIR_OPS

flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp

matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor,
     builder.create<hlfir::AssignOp>(loc, string, declareOp.getBase());
     mlir::Value bufferizedExpr =
         packageBufferizedExpr(loc, builder, alloca, false);
     rewriter.replaceOp(setLength, bufferizedExpr);
     return mlir::success();
   }
 };
 
+static bool allOtherUsesAreDestroys(mlir::Value value,
+                                    mlir::Operation *currentUse) {
+  for (mlir::Operation *useOp : value.getUsers())
+    if (!mlir::isa<hlfir::DestroyOp>(useOp) && useOp != currentUse)
+      return false;
+  return true;
+}
+
+static void eraseAllUsesInDestroys(mlir::Value value,
+                                   mlir::ConversionPatternRewriter &rewriter) {
+  for (mlir::Operation *useOp : value.getUsers())
+    if (mlir::isa<hlfir::DestroyOp>(useOp))
+      rewriter.eraseOp(useOp);
+}
+
 struct AssociateOpConversion
     : public mlir::OpConversionPattern<hlfir::AssociateOp> {
   using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern;
   explicit AssociateOpConversion(mlir::MLIRContext *ctx)
       : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {}
   mlir::LogicalResult
   matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Location loc = associate->getLoc();
     // If this is the last use of the expression value and this is an hlfir.expr
     // that was bufferized, re-use the storage.
     // Otherwise, create a temp and assign the storage to it.
     mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource());
     const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType());
 
     auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar,
                            mlir::Value flag) {
       associate.getResult(0).replaceAllUsesWith(hlfirVar);
       associate.getResult(1).replaceAllUsesWith(firVar);
       associate.getResult(2).replaceAllUsesWith(flag);
       rewriter.replaceOp(associate, {hlfirVar, firVar, flag});
     };
 
-    if (!isTrivialValue && associate.getSource().hasOneUse()) {
+    if (!isTrivialValue && allOtherUsesAreDestroys(associate.getSource(),
+                                                   associate.getOperation())) {
+      // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr
+      // outside of the hlfir.destroy. Take on the cleaning-up responsibility
+      // for the related hlfir.end_associate, and erase the hlfir.destroy (if
+      // any).
       mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource());
       mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase();
       replaceWith(bufferizedExpr, firBase, mustFree);
+      eraseAllUsesInDestroys(associate.getSource(), rewriter);
       return mlir::success();
     }
     if (isTrivialValue) {
       auto module = associate->getParentOfType<mlir::ModuleOp>();
       fir::FirOpBuilder builder(rewriter, fir::getKindMapping(module));
       auto temp = builder.createTemporary(loc, bufferizedExpr.getType(),
                                           associate.getUniqName());
       builder.create<fir::StoreOp>(loc, bufferizedExpr, temp);
       mlir::Value mustFree = builder.createBool(loc, false);
       replaceWith(temp, temp, mustFree);
       return mlir::success();
     }
     TODO(loc, "hlfir.associate of hlfir.expr with more than one use");
   }
 };
 
+static void genFreeIfMustFree(mlir::Location loc,
+                              mlir::ConversionPatternRewriter &rewriter,
+                              mlir::Value var, mlir::Value mustFree) {
+  auto genFree = [&]() {
+    if (var.getType().isa<fir::BaseBoxType>())
+      TODO(loc, "unbox");
+    if (!var.getType().isa<fir::HeapType>())
+      var = rewriter.create<fir::ConvertOp>(
+          loc, fir::HeapType::get(fir::unwrapRefType(var.getType())), var);
+    rewriter.create<fir::FreeMemOp>(loc, var);
+  };
+  if (auto cstMustFree = fir::getIntIfConstant(mustFree)) {
+    if (*cstMustFree != 0)
+      genFree();
+    // else, nothing to do.
+    return;
+  }
+  TODO(loc, "conditional free");
+}
+
 struct EndAssociateOpConversion
     : public mlir::OpConversionPattern<hlfir::EndAssociateOp> {
   using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern;
   explicit EndAssociateOpConversion(mlir::MLIRContext *ctx)
       : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {}
   mlir::LogicalResult
   matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Value mustFree = adaptor.getMustFree();
     mlir::Location loc = endAssociate->getLoc();
+    genFreeIfMustFree(loc, rewriter, adaptor.getVar(), adaptor.getMustFree());
     rewriter.eraseOp(endAssociate);
-    auto genFree = [&]() {
-      mlir::Value var = adaptor.getVar();
-      if (var.getType().isa<fir::BaseBoxType>())
-        TODO(loc, "unbox");
-      rewriter.create<fir::FreeMemOp>(loc, var);
-    };
-    if (auto cstMustFree = fir::getIntIfConstant(mustFree)) {
-      if (*cstMustFree != 0)
-        genFree();
-      // else, nothing to do.
-      return mlir::success();
-    }
-    TODO(endAssociate.getLoc(), "conditional free");
+};
+
+struct DestroyOpConversion
+    : public mlir::OpConversionPattern<hlfir::DestroyOp> {
+  using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern;
+  explicit DestroyOpConversion(mlir::MLIRContext *ctx)
+      : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {}
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    // If expr was bufferized on the heap, now is time to deallocate the buffer.
+    mlir::Location loc = destroy->getLoc();
+    mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
+    if (!fir::isa_trivial(bufferizedExpr.getType())) {
+      mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr());
+      mlir::Value firBase = hlfir::Entity(bufferizedExpr).getFirBase();
+      genFreeIfMustFree(loc, rewriter, firBase, mustFree);
+    }
+    rewriter.eraseOp(destroy);
+    return mlir::success();
   }
 };
 
 struct NoReassocOpConversion
     : public mlir::OpConversionPattern<hlfir::NoReassocOp> {
   using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern;
   explicit NoReassocOpConversion(mlir::MLIRContext *ctx)
       : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {}
   mlir::LogicalResult
   matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
     rewriter.replaceOpWithNewOp<hlfir::NoReassocOp>(
         noreassoc, getBufferizedExprStorage(adaptor.getVal()));
     return mlir::success();
   }
 };
 
+/// Was \p value created in the mlir block where \p builder is currently set ?
+static bool wasCreatedInCurrentBlock(mlir::Value value,
+                                     fir::FirOpBuilder &builder) {
+  if (mlir::Operation *op = value.getDefiningOp())
+    return op->getBlock() == builder.getBlock();
+  return false;
+}
+
 /// This Listener allows setting both the builder and the rewriter as
 /// listeners. This is required when a pattern uses a firBuilder helper that
 /// may create illegal operations that will need to be translated and requires
 /// notifying the rewriter.
 struct HLFIRListener : public mlir::OpBuilder::Listener {
   HLFIRListener(fir::FirOpBuilder &builder,
                 mlir::ConversionPatternRewriter &rewriter)
       : builder{builder}, rewriter{rewriter} {}
matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor,
     auto yield =
         hlfir::inlineElementalOp(loc, builder, elemental, oneBasedLoopIndices);
     hlfir::Entity elementValue(yield.getElementValue());
     // Skip final AsExpr if any. It would create an element temporary,
     // which is no needed since the element will be assigned right away in
     // the array temporary. An hlfir.as_expr may have been added if the
     // elemental is a "view" over a variable (e.g parentheses or transpose).
     if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) {
+      if (asExpr->hasOneUse() && !asExpr.isMove()) {
         elementValue = hlfir::Entity{asExpr.getVar()};
-      if (asExpr->hasOneUse())
         rewriter.eraseOp(asExpr);
       }
+    }
     rewriter.eraseOp(yield);
     // Assign the element value to the temp element for this iteration.
     auto tempElement =
         hlfir::getElementAt(loc, builder, temp, oneBasedLoopIndices);
     builder.create<hlfir::AssignOp>(loc, elementValue, tempElement);
+    // hlfir.yield_element implicitly marks the end-of-life its operand if
+    // it is an expression created in the hlfir.elemental (since it is its
+    // last use and an hlfir.destroy could not be created afterwards)
+    // Now that this node has been removed and the expression has been used in
+    // the assign, insert an hlfir.destroy to mark the expression end-of-life.
+    // If the expression creation allocated a buffer on the heap inside the
+    // loop, this will ensure the buffer properly deallocated.
+    if (elementValue.getType().isa<hlfir::ExprType>() &&
+        wasCreatedInCurrentBlock(elementValue, builder))
+      builder.create<hlfir::DestroyOp>(loc, elementValue);
     builder.restoreInsertionPoint(insPt);
 
     mlir::Value bufferizedExpr =
         packageBufferizedExpr(loc, builder, temp, cleanup);
     rewriter.replaceOp(elemental, bufferizedExpr);
     return mlir::success();
   }
 };
 
 class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
 public:
   void runOnOperation() override {
     // TODO: make this a pass operating on FuncOp. The issue is that
     // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm
     // intrinsics calls creation. This may create race conflict if the pass is
     // scheduled on FuncOp. A solution could be to provide an optional mutex
     // when building a FirOpBuilder and locking around FuncOp and GlobalOp
     // creation, but this needs a bit more thinking, so at this point the pass
     // is scheduled on the moduleOp.
     auto module = this->getOperation();
     auto *context = &getContext();
     mlir::RewritePatternSet patterns(context);
-    patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
-                    AssociateOpConversion, ConcatOpConversion,
-                    ElementalOpConversion, EndAssociateOpConversion,
-                    NoReassocOpConversion, SetLengthOpConversion>(context);
+    patterns
+        .insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
+                AssociateOpConversion, ConcatOpConversion, DestroyOpConversion,
     mlir::ConversionTarget target(*context);
     target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp,
                         hlfir::EndAssociateOp, hlfir::SetLengthOp,
                         hlfir::YieldElementOp>();
     target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) {
       return llvm::all_of(
                  op->getResultTypes(),
                  [](mlir::Type ty) { return !ty.isa<hlfir::ExprType>(); }) &&

flang/test/HLFIR/destroy-codegen.fir

+// Test hlfir.destroy code generation  and hlfir.yield_element "implicit
+// hlfir.destroy" aspect.
+
+// RUN: fir-opt %s -bufferize-hlfir | FileCheck %s
+
+func.func @test_move_with_cleanup(%arg0 : !fir.ref<!fir.array<100xi32>>) {
+  %must_free = arith.constant true
+  %expr = hlfir.as_expr %arg0 move %must_free: (!fir.ref<!fir.array<100xi32>>, i1) -> !hlfir.expr<100xi32>
+  hlfir.destroy %expr : !hlfir.expr<100xi32>
+  return
+}
+// CHECK-LABEL:   func.func @test_move_with_cleanup(
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.array<100xi32>>) {
+// CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.array<100xi32>>) -> !fir.heap<!fir.array<100xi32>>
+// CHECK:  fir.freemem %[[VAL_5]] : !fir.heap<!fir.array<100xi32>>
+
+func.func @test_move_no_cleanup(%arg0 : !fir.ref<!fir.array<100xi32>>) {
+  %must_free = arith.constant false
+  %expr = hlfir.as_expr %arg0 move %must_free: (!fir.ref<!fir.array<100xi32>>, i1) -> !hlfir.expr<100xi32>
+  hlfir.destroy %expr : !hlfir.expr<100xi32>
+  return
+}
+// CHECK-LABEL:   func.func @test_move_no_cleanup(
+// CHECK-NOT:  fir.freemem
+// CHECK:  return
+
+func.func @test_elemental() {
+  %c100 = arith.constant 100 : index
+  %c20 = arith.constant 20 : index
+  %0 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %3 = hlfir.elemental %0 typeparams %c20 : (!fir.shape<1>, index) -> !hlfir.expr<100x!fir.char<1,20>> {
+  ^bb0(%i: index):
+    %buffer = fir.allocmem !fir.char<1,20>
+    %must_free = arith.constant true
+    %expr = hlfir.as_expr %buffer move %must_free: (!fir.heap<!fir.char<1,20>>, i1) -> !hlfir.expr<!fir.char<1,20>>
+    hlfir.yield_element %expr : !hlfir.expr<!fir.char<1,20>>
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_elemental(
+// CHECK:  fir.do_loop
+// CHECK:    %[[VAL_9:.*]] = fir.allocmem !fir.char<1,20>
+// CHECK:    hlfir.assign %[[VAL_9]] to %{{.*}} : !fir.heap<!fir.char<1,20>>, !fir.ref<!fir.char<1,20>>
+// CHECK:    fir.freemem %[[VAL_9]] : !fir.heap<!fir.char<1,20>>
+// CHECK:  }
+// CHECK:  return
+
+func.func @test_elemental_expr_created_outside_of_loops() {
+  %buffer = fir.allocmem !fir.char<1,20>
+  %must_free = arith.constant true
+  %expr = hlfir.as_expr %buffer move %must_free: (!fir.heap<!fir.char<1,20>>, i1) -> !hlfir.expr<!fir.char<1,20>>
+  %c100 = arith.constant 100 : index
+  %c20 = arith.constant 20 : index
+  %0 = fir.shape %c100 : (index) -> !fir.shape<1>
+  %3 = hlfir.elemental %0 typeparams %c20 : (!fir.shape<1>, index) -> !hlfir.expr<100x!fir.char<1,20>> {
+  ^bb0(%i: index):
+    // No freemem should be inserted inside the loops.
+    hlfir.yield_element %expr : !hlfir.expr<!fir.char<1,20>>
+  }
+  hlfir.destroy %expr : !hlfir.expr<!fir.char<1,20>>
+  return
+}
+// CHECK-LABEL:   func.func @test_elemental_expr_created_outside_of_loops() {
+// CHECK:  %[[VAL_9:.*]] = fir.allocmem !fir.char<1,20>
+// CHECK:  fir.do_loop
+// CHECK:    hlfir.assign %[[VAL_9]] to %{{.*}} : !fir.heap<!fir.char<1,20>>, !fir.ref<!fir.char<1,20>>
+// CHECK-NOT:  fir.freemem
+// CHECK:  }
+// CHECK:  fir.freemem %[[VAL_9]] : !fir.heap<!fir.char<1,20>>
+// CHECK:  return

flang/test/HLFIR/destroy.fir

+// Test hlfir.destroy operation parse, verify (no errors), and unparse.
+
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @test(%expr : !hlfir.expr<?x?xf64>) {
+  hlfir.destroy %expr : !hlfir.expr<?x?xf64>
+  return
+}
+// CHECK-LABEL: func.func @test(
+// CHECK-SAME:    %[[VAL_0:.*]]: !hlfir.expr<?x?xf64>) {
+// CHECK:   hlfir.destroy %[[VAL_0]] : !hlfir.expr<?x?xf64>