diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp --- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp +++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp @@ -15,6 +15,7 @@ #include "flang/Optimizer/Builder/Character.h" #include "flang/Optimizer/Builder/FIRBuilder.h" #include "flang/Optimizer/Builder/HLFIRTools.h" +#include "flang/Optimizer/Builder/IntrinsicCall.h" #include "flang/Optimizer/Builder/MutableBox.h" #include "flang/Optimizer/Builder/Runtime/Assign.h" #include "flang/Optimizer/Builder/Todo.h" @@ -498,6 +499,155 @@ } }; +/// Base class for passes converting transformational intrinsic operations into +/// runtime calls +template +class HlfirIntrinsicConversion : public mlir::OpConversionPattern { + using mlir::OpConversionPattern::OpConversionPattern; + +protected: + struct IntrinsicArgument { + mlir::Value val; // allowed to be null if the argument is absent + mlir::Type desiredType; + }; + + /// Lower the arguments to the intrinsic: adding nesecarry boxing and + /// conversion to match the signature of the intrinsic in the runtime library. + llvm::SmallVector + lowerArguments(mlir::Operation *op, + const llvm::ArrayRef &args, + mlir::ConversionPatternRewriter &rewriter, + const fir::IntrinsicArgumentLoweringRules *argLowering) const { + assert(args.size() == 3 && "Transformational intrinsics have 3 args"); + mlir::Location loc = op->getLoc(); + fir::KindMapping kindMapping{rewriter.getContext()}; + fir::FirOpBuilder builder{rewriter, kindMapping}; + + llvm::SmallVector ret; + + for (size_t i = 0; i < args.size(); ++i) { + mlir::Value arg = args[i].val; + mlir::Type desiredType = args[i].desiredType; + if (!arg) { + ret.emplace_back(fir::getAbsentIntrinsicArgument()); + continue; + } + hlfir::Entity entity{arg}; + + fir::ArgLoweringRule argRules = + fir::lowerIntrinsicArgumentAs(*argLowering, i); + switch (argRules.lowerAs) { + case fir::LowerIntrinsicArgAs::Value: { + if (args[i].desiredType != arg.getType()) { + arg = builder.createConvert(loc, desiredType, arg); + entity = hlfir::Entity{arg}; + } + auto [exv, cleanup] = hlfir::convertToValue(loc, builder, entity); + if (cleanup) + TODO(loc, "extended value cleanup"); + ret.emplace_back(exv); + } break; + case fir::LowerIntrinsicArgAs::Addr: { + auto [exv, cleanup] = + hlfir::convertToAddress(loc, builder, entity, desiredType); + if (cleanup) + TODO(loc, "extended value cleanup"); + ret.emplace_back(exv); + } break; + case fir::LowerIntrinsicArgAs::Box: { + auto [box, cleanup] = + hlfir::convertToBox(loc, builder, entity, desiredType); + if (cleanup) + TODO(loc, "extended value cleanup"); + ret.emplace_back(box); + } break; + case fir::LowerIntrinsicArgAs::Inquired: { + if (args[i].desiredType != arg.getType()) { + arg = builder.createConvert(loc, desiredType, arg); + entity = hlfir::Entity{arg}; + } + // Place hlfir.expr in memory, and unbox fir.boxchar. Other entities + // are translated to fir::ExtendedValue without transofrmation (notably, + // pointers/allocatable are not dereferenced). + // TODO: once lowering to FIR retires, UBOUND and LBOUND can be + // simplified since the fir.box lowered here are now guarenteed to + // contain the local lower bounds thanks to the hlfir.declare (the extra + // rebox can be removed). + auto [exv, cleanup] = + hlfir::translateToExtendedValue(loc, builder, entity); + if (cleanup) + TODO(loc, "extended value cleanup"); + ret.emplace_back(exv); + } break; + } + } + + return ret; + } + + void processReturnValue(mlir::Operation *op, + const fir::ExtendedValue &resultExv, bool mustBeFreed, + fir::FirOpBuilder &builder, + mlir::PatternRewriter &rewriter) const { + mlir::Location loc = op->getLoc(); + + mlir::Value firBase = fir::getBase(resultExv); + mlir::Type firBaseTy = firBase.getType(); + + std::optional resultEntity; + if (fir::isa_trivial(firBaseTy)) { + resultEntity = hlfir::EntityWithAttributes{firBase}; + } else { + resultEntity = + hlfir::genDeclare(loc, builder, resultExv, ".tmp.intrinsic_result", + fir::FortranVariableFlagsAttr{}); + } + + if (resultEntity->isVariable()) { + hlfir::AsExprOp asExpr = builder.create( + loc, *resultEntity, builder.createBool(loc, mustBeFreed)); + resultEntity = hlfir::EntityWithAttributes{asExpr.getResult()}; + } + + rewriter.replaceOp(op, resultEntity->getBase()); + } +}; + +struct SumOpConversion : public HlfirIntrinsicConversion { + using HlfirIntrinsicConversion::HlfirIntrinsicConversion; + + mlir::LogicalResult + matchAndRewrite(hlfir::SumOp sum, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + fir::KindMapping kindMapping{rewriter.getContext()}; + fir::FirOpBuilder builder{rewriter, kindMapping}; + const mlir::Location &loc = sum->getLoc(); + HLFIRListener listener{builder, rewriter}; + builder.setListener(&listener); + + mlir::Type i32 = builder.getI32Type(); + mlir::Type logicalType = fir::LogicalType::get( + builder.getContext(), builder.getKindMap().defaultLogicalKind()); + + llvm::SmallVector inArgs; + inArgs.push_back({sum.getArray(), sum.getArray().getType()}); + inArgs.push_back({sum.getDim(), i32}); + inArgs.push_back({sum.getMask(), logicalType}); + + auto *argLowering = fir::getIntrinsicArgumentLowering("sum"); + llvm::SmallVector args = + lowerArguments(sum, inArgs, rewriter, argLowering); + + mlir::Type scalarResultType = hlfir::getFortranElementType(sum.getType()); + + auto [resultExv, mustBeFreed] = + fir::genIntrinsicCall(builder, loc, "sum", scalarResultType, args); + + processReturnValue(sum, resultExv, mustBeFreed, builder, rewriter); + return mlir::success(); + } +}; + class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase { public: void runOnOperation() override { @@ -515,7 +665,8 @@ .insert(context); + NoReassocOpConversion, SetLengthOpConversion, SumOpConversion>( + context); mlir::ConversionTarget target(*context); target.addIllegalOp> {fir.bindc_name = "a"}, %arg1: !fir.ref {fir.bindc_name = "s"}) { + %0:2 = hlfir.declare %arg0 {uniq_name = "_QFsum1Ea"} : (!fir.box>) -> (!fir.box>, !fir.box>) + %1:2 = hlfir.declare %arg1 {uniq_name = "_QFsum1Es"} : (!fir.ref) -> (!fir.ref, !fir.ref) + %2 = hlfir.sum %0#0 {fastmath = #arith.fastmath} : (!fir.box>) -> !hlfir.expr + hlfir.assign %2 to %1#0 : !hlfir.expr, !fir.ref + hlfir.destroy %2 : !hlfir.expr + return +} +// CHECK-LABEL: func.func @_QPsum1( +// CHECK: %[[ARG0:.*]]: !fir.box> +// CHECK: %[[ARG1:.*]]: !fir.ref +// CHECK-DAG: %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]] +// CHECK-DAG: %[[RES:.*]]:2 = hlfir.declare %[[ARG1]] +// CHECK-DAG: %[[MASK:.*]] = fir.absent !fir.box +// CHECK-DAG: %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 : (!fir.box>) -> !fir.box +// CHECK-DAG: %[[MASK_ARG:.*]] = fir.convert %[[MASK]] : (!fir.box) -> !fir.box +// CHECK: %[[RET:.*]] = fir.call @_FortranASumInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box, !fir.ref, i32, i32, !fir.box) -> i32 +// CHECK-NEXT: hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref +// CHECK-NEXT: return +// CHECK-NEXT: } + +// sum with a by-ref dimension of index type +func.func @_QPsum2(%arg0: !fir.box> {fir.bindc_name = "a"}, %arg1: !fir.box> {fir.bindc_name = "s"}, %arg2: !fir.ref {fir.bindc_name = "d"}) { + %0:2 = hlfir.declare %arg0 {uniq_name = "_QFsum2Ea"} : (!fir.box>) -> (!fir.box>, !fir.box>) + %1:2 = hlfir.declare %arg2 {uniq_name = "_QFsum2Ed"} : (!fir.ref) -> (!fir.ref, !fir.ref) + %2:2 = hlfir.declare %arg1 {uniq_name = "_QFsum2Es"} : (!fir.box>) -> (!fir.box>, !fir.box>) + %3 = fir.load %1#0 : !fir.ref + %4 = hlfir.sum %0#0 dim %3#0 {fastmath = #arith.fastmath} : (!fir.box>, index) -> !hlfir.expr + hlfir.assign %4 to %2#0 : !hlfir.expr, !fir.box> + hlfir.destroy %4 : !hlfir.expr + return +} +// CHECK-LABEL: func.func @_QPsum2( +// CHECK: %[[ARG0:.*]]: !fir.box> +// CHECK: %[[ARG1:.*]]: !fir.box> +// CHECK: %[[ARG2:.*]]: !fir.ref +// CHECK-DAG: %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]] +// CHECK-DAG: %[[RES:.*]]:2 = hlfir.declare %[[ARG1]] +// CHECK-DAG: %[[DIM_VAR:.*]]:2 = hlfir.declare %[[ARG2]] + +// CHECK-DAG: %[[RET_BOX:.*]] = fir.alloca !fir.box>> +// CHECK-DAG: %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap> +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1> +// CHECK-DAG: %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]]) +// CHECK-DAG: fir.store %[[RET_EMBOX]] to %[[RET_BOX]] + +// CHECK-DAG: %[[MASK:.*]] = fir.absent !fir.box +// CHECK-DAG: %[[DIM_IDX:.*]] = fir.load %[[DIM_VAR]]#0 : !fir.ref +// CHECK-DAG: %[[DIM:.*]] = fir.convert %[[DIM_IDX]] : (index) -> i32 + +// CHECK-DAG: %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]] +// CHECK-DAG: %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 : (!fir.box>) -> !fir.box +// CHECK-DAG: %[[MASK_ARG:.*]] = fir.convert %[[MASK]] : (!fir.box) -> !fir.box +// CHECK: %[[NONE:.*]] = fir.call @_FortranASumDim(%[[RET_ARG]], %[[ARRAY_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[MASK_ARG]]) : (!fir.ref>, !fir.box, i32, !fir.ref, i32, !fir.box) -> none + +// CHECK: %[[RET:.*]] = fir.load %[[RET_BOX]] +// CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[RET]] +// CHECK-NEXT: %[[ADDR:.*]] = fir.box_addr %[[RET]] +// CHECK-NEXT: %[[SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1 +// TODO: fix alias analysis in hlfir.assign bufferization +// CHECK-NEXT: %[[TMP:.*]]:2 = hlfir.declare %[[ADDR]](%[[SHIFT]]) {uniq_name = ".tmp.intrinsic_result"} +// CHECK: %[[TUPLE0:.*]] = fir.undefined tuple>, i1> +// CHECK: %[[TUPLE1:.*]] = fir.insert_value %[[TUPLE0]], %[[TRUE:.*]], [1 : index] +// CHECK: %[[TUPLE2:.*]] fir.insert_value %[[TUPLE1]], %[[TMP]]#0, [0 : index] +// CHECK: hlfir.assign %[[TMP]]#0 to %[[RES]]#0 +// CHECK: fir.freemem %[[TMP]]#1 +// CHECK-NEXT: return +// CHECK-NEXT: } + +// sum with scalar mask +func.func @_QPsum3(%arg0: !fir.box> {fir.bindc_name = "a"}, %arg1: !fir.ref {fir.bindc_name = "s"}, %arg2: !fir.ref> {fir.bindc_name = "m"}) { + %0:2 = hlfir.declare %arg0 {uniq_name = "_QFsum3Ea"} : (!fir.box>) -> (!fir.box>, !fir.box>) + %1:2 = hlfir.declare %arg2 {uniq_name = "_QFsum3Em"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) + %2:2 = hlfir.declare %arg1 {uniq_name = "_QFsum3Es"} : (!fir.ref) -> (!fir.ref, !fir.ref) + %3 = hlfir.sum %0#0 mask %1#0 {fastmath = #arith.fastmath} : (!fir.box>, !fir.ref>) -> !hlfir.expr + hlfir.assign %3 to %2#0 : !hlfir.expr, !fir.ref + hlfir.destroy %3 : !hlfir.expr + return +} +// CHECK-LABEL: func.func @_QPsum3( +// CHECK: %[[ARG0:.*]]: !fir.box> +// CHECK: %[[ARG1:.*]]: !fir.ref +// CHECK: %[[ARG2:.*]]: !fir.ref> +// CHECK-DAG: %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]] +// CHECK-DAG: %[[RES:.*]]:2 = hlfir.declare %[[ARG1]] +// CHECK-DAG: %[[MASK:.*]]:2 = hlfir.declare %[[ARG2]] +// CHECK-DAG: %[[MASK_BOX:.*]] = fir.embox %[[MASK]]#1 : (!fir.ref>) -> !fir.box> +// CHECK-DAG: %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 : (!fir.box>) -> !fir.box +// CHECK-DAG: %[[MASK_ARG:.*]] = fir.convert %[[MASK_BOX]] : (!fir.box>) -> !fir.box +// CHECK: %[[RET:.*]] = fir.call @_FortranASumInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box, !fir.ref, i32, i32, !fir.box) -> i32 +// CHECK-NEXT: hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref +// CHECK-NEXT: return +// CHECK-NEXT: } + +// sum with array mask +func.func @_QPsum4(%arg0: !fir.box> {fir.bindc_name = "a"}, %arg1: !fir.ref {fir.bindc_name = "s"}, %arg2: !fir.box>> {fir.bindc_name = "m"}) { + %0:2 = hlfir.declare %arg0 {uniq_name = "_QFsum4Ea"} : (!fir.box>) -> (!fir.box>, !fir.box>) + %1:2 = hlfir.declare %arg2 {uniq_name = "_QFsum4Em"} : (!fir.box>>) -> (!fir.box>>, !fir.box>>) + %2:2 = hlfir.declare %arg1 {uniq_name = "_QFsum4Es"} : (!fir.ref) -> (!fir.ref, !fir.ref) + %3 = hlfir.sum %0#0 mask %1#0 {fastmath = #arith.fastmath} : (!fir.box>, !fir.box>>) -> !hlfir.expr + hlfir.assign %3 to %2#0 : !hlfir.expr, !fir.ref + hlfir.destroy %3 : !hlfir.expr + return +} +// CHECK-LABEL: func.func @_QPsum4( +// CHECK: %[[ARG0:.*]]: !fir.box> +// CHECK: %[[ARG1:.*]]: !fir.ref +// CHECK: %[[ARG2:.*]]: !fir.box>> +// CHECK-DAG: %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]] +// CHECK-DAG: %[[RES:.*]]:2 = hlfir.declare %[[ARG1]] +// CHECK-DAG: %[[MASK:.*]]:2 = hlfir.declare %[[ARG2]] +// CHECK-DAG: %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 : (!fir.box>) -> !fir.box +// CHECK-DAG: %[[MASK_ARG:.*]] = fir.convert %[[MASK]]#1 : (!fir.box>>) -> !fir.box +// CHECK: %[[RET:.*]] = fir.call @_FortranASumInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box, !fir.ref, i32, i32, !fir.box) -> i32 +// CHECK-NEXT: hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref +// CHECK-NEXT: return +// CHECK-NEXT: } + +fir.global internal @_QFsum5Ea : !fir.array<2x2xi32> { + %0 = fir.undefined !fir.array<2x2xi32> + %c1_i32 = arith.constant 1 : i32 + %1 = fir.insert_value %0, %c1_i32, [0 : index, 0 : index] : (!fir.array<2x2xi32>, i32) -> !fir.array<2x2xi32> + %c2_i32 = arith.constant 2 : i32 + %2 = fir.insert_value %1, %c2_i32, [1 : index, 0 : index] : (!fir.array<2x2xi32>, i32) -> !fir.array<2x2xi32> + %c3_i32 = arith.constant 3 : i32 + %3 = fir.insert_value %2, %c3_i32, [0 : index, 1 : index] : (!fir.array<2x2xi32>, i32) -> !fir.array<2x2xi32> + %c4_i32 = arith.constant 4 : i32 + %4 = fir.insert_value %3, %c4_i32, [1 : index, 1 : index] : (!fir.array<2x2xi32>, i32) -> !fir.array<2x2xi32> + %c2 = arith.constant 2 : index + %c2_0 = arith.constant 2 : index + fir.has_value %4 : !fir.array<2x2xi32> +} + +// 3 argument sum, using local variables +func.func @_QPsum5(%arg0: !fir.ref> {fir.bindc_name = "s"}) { + %0 = fir.address_of(@_QFsum5Ea) : !fir.ref> + %c2 = arith.constant 2 : index + %c2_0 = arith.constant 2 : index + %1 = fir.shape %c2, %c2_0 : (index, index) -> !fir.shape<2> + %2:2 = hlfir.declare %0(%1) {uniq_name = "_QFsum5Ea"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) + %c2_1 = arith.constant 2 : index + %3 = fir.shape %c2_1 : (index) -> !fir.shape<1> + %4:2 = hlfir.declare %arg0(%3) {uniq_name = "_QFsum5Es"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) + %c1_i32 = arith.constant 1 : i32 + %true = arith.constant true + %5 = hlfir.sum %2#0 dim %c1_i32 mask %true {fastmath = #arith.fastmath} : (!fir.ref>, i32, i1) -> !hlfir.expr<2xi32> + hlfir.assign %5 to %4#0 : !hlfir.expr<2xi32>, !fir.ref> + hlfir.destroy %5 : !hlfir.expr<2xi32> + return +} +// CHECK-LABEL: func.func @_QPsum5( +// CHECK: %[[ARG0:.*]]: !fir.ref> +// CHECK-DAG: %[[RET_BOX:.*]] = fir.alloca !fir.box>> +// CHECK-DAG: %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap> +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1> +// CHECK-DAG: %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]]) +// CHECK-DAG: fir.store %[[RET_EMBOX]] to %[[RET_BOX]] + +// CHECK-DAG: %[[RES_VAR:.*]] = hlfir.declare %[[ARG0]](%[[RES_SHAPE:.*]]) + +// CHECK-DAG: %[[MASK_ALLOC:.*]] = fir.alloca !fir.logical<4> +// CHECK-DAG: %[[TRUE:.*]] = arith.constant true +// CHECK-DAG: %[[MASK_VAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4> +// CHECK-DAG: fir.store %[[MASK_VAL]] to %[[MASK_ALLOC]] : !fir.ref> +// CHECK-DAG: %[[MASK_BOX:.*]] = fir.embox %[[MASK_ALLOC]] + +// CHECK-DAG: %[[ARRAY_ADDR:.*]] = fir.address_of +// CHECK-DAG: %[[ARRAY_VAR:.*]]:2 = hlfir.declare %[[ARRAY_ADDR]](%[[ARRAY_SHAPE:.*]]) +// CHECK-DAG: %[[ARRAY_BOX:.*]] = fir.embox %[[ARRAY_VAR]]#1(%[[ARRAY_SHAPE:.*]]) + +// CHECK-DAG: %[[DIM:.*]] = arith.constant 1 : i32 + +// CHECK-DAG: %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]] +// CHECK-DAG: %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY_BOX]] : (!fir.box>) -> !fir.box +// CHECK-DAG: %[[MASK_ARG:.*]] = fir.convert %[[MASK_BOX]] : (!fir.box>) -> !fir.box +// CHECK: %[[NONE:.*]] = fir.call @_FortranASumDim(%[[RET_ARG]], %[[ARRAY_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[MASK_ARG]]) : (!fir.ref>, !fir.box, i32, !fir.ref, i32, !fir.box) -> none