[flang][openacc] Add data operands conversion from FIR

[clementval]

Summary

This patch revive an old PR attempt [1] to perform the
data operands conversion needed for translation to LLVMIR.

This is currently not supporting box/class type since they will
normally not reach this pass when the proposed change in this RFC [2]
are implemented.

[1] https://github.com/flang-compiler/f18-llvm-project/pull/915
[2] https://discourse.llvm.org/t/rfc-openacc-dialect-data-operation-improvements/69825/2

Depends on D147824

[razvanlupusoru]

Looks great!

I also realized looking at this work that with the proposal from https://discourse.llvm.org/t/rfc-openacc-dialect-data-operation-improvements/69825 , it will be easier to distinguish data from non-data operands (since all data operands will be in the same operand list).

flang/lib/Optimizer/Transforms/OpenACC/OpenACCDataOperandConversion.cpp

49

nit: should be plural numDataOperands

[PeteSteinfeld]

I'm getting errors when I try to build with these changes. Here's an excerpt from the log file:

[6120/7355] Building CXX object tools/flang/lib/Optimizer/Transforms/CMakeFiles/
obj.FIRTransforms.dir/OpenACC/OpenACCDataOperandConversion.cpp.o
FAILED: tools/flang/lib/Optimizer/Transforms/CMakeFiles/obj.FIRTransforms.dir/Op
enACC/OpenACCDataOperandConversion.cpp.o 
/home/sw/thirdparty/gcc/gcc-9.3.0/linux86-64/bin/g++ -DFLANG_INCLUDE_TESTS=1 -DF
LANG_LITTLE_ENDIAN=1 -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_GNU_SO
URCE -D_LIBCPP_ENABLE_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS
 -D__STDC_LIMIT_MACROS -I/local/home/psteinfeld/main/825/build/tools/flang/lib/O
ptimizer/Transforms -I/local/home/psteinfeld/main/825/flang/lib/Optimizer/Transf
orms -I/local/home/psteinfeld/main/825/flang/include -I/local/home/psteinfeld/ma
in/825/build/tools/flang/include -I/local/home/psteinfeld/main/825/build/include
 -I/local/home/psteinfeld/main/825/llvm/include -isystem /local/home/psteinfeld/
main/825/llvm/../mlir/include -isystem /local/home/psteinfeld/main/825/build/too
ls/mlir/include -isystem /local/home/psteinfeld/main/825/build/tools/clang/inclu
de -isystem /local/home/psteinfeld/main/825/llvm/../clang/include -fPIC -fno-sem
antic-interposition -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra 
-Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializer
s -pedantic -Wno-long-long -Wimplicit-fallthrough -Wno-maybe-uninitialized -Wno-
class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wd
elete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wno-misleading-indentati
on -fdiagnostics-color -ffunction-sections -fdata-sections -Werror -Wno-deprecat
ed-copy -Wno-ctad-maybe-unsupported -fno-strict-aliasing -fno-semantic-interposi
tion -O3 -DNDEBUG  -fno-exceptions -funwind-tables -fno-rtti -UNDEBUG -std=c++17
 -MD -MT tools/flang/lib/Optimizer/Transforms/CMakeFiles/obj.FIRTransforms.dir/O
penACC/OpenACCDataOperandConversion.cpp.o -MF tools/flang/lib/Optimizer/Transfor
ms/CMakeFiles/obj.FIRTransforms.dir/OpenACC/OpenACCDataOperandConversion.cpp.o.d
 -o tools/flang/lib/Optimizer/Transforms/CMakeFiles/obj.FIRTransforms.dir/OpenAC
C/OpenACCDataOperandConversion.cpp.o -c /local/home/psteinfeld/main/825/flang/li
b/Optimizer/Transforms/OpenACC/OpenACCDataOperandConversion.cpp
/local/home/psteinfeld/main/825/flang/lib/Optimizer/Transforms/OpenACC/OpenACCDa
taOperandConversion.cpp: In instantiation of ‘mlir::LogicalResult {anonymous}::L
egalizeDataOpForLLVMTranslation<Op>::matchAndRewrite(Op, typename Op::Adaptor, m
lir::ConversionPatternRewriter&) const [with Op = mlir::acc::UpdateOp; typename 
Op::Adaptor = mlir::acc::UpdateOpAdaptor]’:
/local/home/psteinfeld/main/825/flang/lib/Optimizer/Transforms/OpenACC/OpenACCDa
taOperandConversion.cpp:42:3:   required from here
/local/home/psteinfeld/main/825/flang/lib/Optimizer/Transforms/OpenACC/OpenACCDa
taOperandConversion.cpp:62:11: error: ignoring returned value of type ‘mlir::Log
icalResult’, declared with attribute nodiscard [-Werror=unused-result]
   62 |           builder.notifyMatchFailure(op, "BaseBoxType not supported");
      |           ^~~~~~~
In file included from /local/home/psteinfeld/main/825/mlir/include/mlir/Rewrite/
FrozenRewritePatternSet.h:12,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/Transfor
ms/DialectConversion.h:16,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/Conversi
on/LLVMCommon/TypeConverter.h:18,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/Conversi
on/LLVMCommon/Pattern.h:13,
                 from /local/home/psteinfeld/main/825/flang/lib/Optimizer/Transf
orms/OpenACC/OpenACCDataOperandConversion.cpp:11:
/local/home/psteinfeld/main/825/mlir/include/mlir/IR/PatternMatch.h:621:17: note
: in call to ‘mlir::LogicalResult mlir::RewriterBase::notifyMatchFailure(ArgT&&,
 const char*) [with ArgT = mlir::acc::UpdateOp&]’, declared here
  621 |   LogicalResult notifyMatchFailure(ArgT &&arg, const char *msg) {
      |                 ^~~~~~~~~~~~~~~~~~
In file included from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Visit
ors.h:17,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/AttrT
ypeSubElements.h:18,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Stora
geUniquerSupport.h:16,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/TypeS
upport.h:17,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Types
.h:12,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Value
.h:16,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Block
Support.h:16,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Opera
tionSupport.h:17,
                 from /local/home/psteinfeld/main/825/mlir/include/mlir/IR/Diale
ct.h:17,
                 from /local/home/psteinfeld/main/825/flang/include/flang/Optimi
zer/Dialect/FIRDialect.h:16,
                 from /local/home/psteinfeld/main/825/flang/lib/Optimizer/Transf
orms/OpenACC/OpenACCDataOperandConversion.cpp:9:

[clementval]

Rebase + fix an issue

[clementval]

Address review comments

[clementval]

@PeteSteinfeld I rebased and fix an issue. You should be able to build it now.

[PeteSteinfeld]

All builds and tests correctly and looks good.

flang/include/flang/Optimizer/Transforms/Passes.h

 #define GEN_PASS_DECL_CFGCONVERSION
 #define GEN_PASS_DECL_EXTERNALNAMECONVERSION
 #define GEN_PASS_DECL_MEMREFDATAFLOWOPT
 #define GEN_PASS_DECL_SIMPLIFYINTRINSICS
 #define GEN_PASS_DECL_MEMORYALLOCATIONOPT
 #define GEN_PASS_DECL_SIMPLIFYREGIONLITE
 #define GEN_PASS_DECL_ALGEBRAICSIMPLIFICATION
 #define GEN_PASS_DECL_POLYMORPHICOPCONVERSION
+#define GEN_PASS_DECL_OPENACCDATAOPERANDCONVERSION
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
 std::unique_ptr<mlir::Pass> createAbstractResultOnFuncOptPass();
 std::unique_ptr<mlir::Pass> createAbstractResultOnGlobalOptPass();
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
 std::unique_ptr<mlir::Pass> createFirToCfgPass();

 std::unique_ptr<mlir::Pass>
 createMemoryAllocationPass(bool dynOnHeap, std::size_t maxStackSize);
 std::unique_ptr<mlir::Pass> createAnnotateConstantOperandsPass();
 std::unique_ptr<mlir::Pass> createSimplifyRegionLitePass();
 std::unique_ptr<mlir::Pass> createAlgebraicSimplificationPass();
 std::unique_ptr<mlir::Pass>
 createAlgebraicSimplificationPass(const mlir::GreedyRewriteConfig &config);
 std::unique_ptr<mlir::Pass> createPolymorphicOpConversionPass();
+std::unique_ptr<mlir::Pass> createOpenACCDataOperandConversionPass();
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_TRANSFORMS_PASSES_H

flang/include/flang/Optimizer/Transforms/Passes.td

let description = [{
     introducing an intermediate FIR level that simplifies code geneation. 
   }];
   let constructor = "::fir::createPolymorphicOpConversionPass()";
   let dependentDialects = [
     "fir::FIROpsDialect", "mlir::func::FuncDialect"
   ];
 }
 
-  
+def OpenACCDataOperandConversion : Pass<"fir-openacc-data-operand-conversion", "::mlir::func::FuncOp"> {
+  let summary = "Convert the FIR operands in OpenACC ops to LLVM dialect";
+  let dependentDialects = ["mlir::LLVM::LLVMDialect"];
+  let options = [
+    Option<"useOpaquePointers", "use-opaque-pointers", "bool",
+           /*default=*/"true", "Generate LLVM IR using opaque pointers "
+           "instead of typed pointers">,
+  ];
+}
 #endif // FLANG_OPTIMIZER_TRANSFORMS_PASSES

flang/lib/Optimizer/Transforms/CMakeLists.txt

add_flang_library(FIRTransforms
   MemoryAllocation.cpp
   StackArrays.cpp
   MemRefDataFlowOpt.cpp
   SimplifyRegionLite.cpp
   AlgebraicSimplification.cpp
   SimplifyIntrinsics.cpp
   AddDebugFoundation.cpp
   PolymorphicOpConversion.cpp
+  OpenACC/OpenACCDataOperandConversion.cpp
 
   DEPENDS
   FIRDialect
   FIROptTransformsPassIncGen
 
   LINK_LIBS
   FIRBuilder
+  FIRCodeGen
   FIRDialect
   FIRDialectSupport
   FIRSupport
   HLFIRDialect
   MLIRAffineUtils
   MLIRFuncDialect
   MLIRLLVMDialect
   MLIRMathTransforms
   MLIROpenACCDialect
   MLIROpenMPDialect
 )

flang/lib/Optimizer/Transforms/OpenACC/OpenACCDataOperandConversion.cpp

+//===- OpenACCDataOperandConversion.cpp - OpenACC data operand conversion -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
+#include "mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/Pass/Pass.h"
+
+namespace fir {
+#define GEN_PASS_DEF_OPENACCDATAOPERANDCONVERSION
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+#define DEBUG_TYPE "flang-openacc-conversion"
+#include "../CodeGen/TypeConverter.h"
+
+using namespace fir;
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// Conversion patterns
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+template <typename Op>
+class LegalizeDataOpForLLVMTranslation : public ConvertOpToLLVMPattern<Op> {
+  using ConvertOpToLLVMPattern<Op>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(Op op, typename Op::Adaptor adaptor,
+                  ConversionPatternRewriter &builder) const override {
+    Location loc = op.getLoc();
+    fir::LLVMTypeConverter &converter =
+        *static_cast<fir::LLVMTypeConverter *>(this->getTypeConverter());
+
+    unsigned numDataOperands = op.getNumDataOperands();
+
+    // Keep the non data operands without modification.
+    auto nonDataOperands = adaptor.getOperands().take_front(
+        adaptor.getOperands().size() - numDataOperands);
+    SmallVector<Value> convertedOperands;
+    convertedOperands.append(nonDataOperands.begin(), nonDataOperands.end());
+
+    // Go over the data operand and legalize them for translation.
+    for (unsigned idx = 0; idx < numDataOperands; ++idx) {
+      Value originalDataOperand = op.getDataOperand(idx);
+      if (auto refTy =
+              originalDataOperand.getType().dyn_cast<fir::ReferenceType>()) {
+        if (refTy.getEleTy().isa<fir::BaseBoxType>())
+          return builder.notifyMatchFailure(op, "BaseBoxType not supported");
+        mlir::Type convertedType =
+            converter.convertType(refTy).cast<mlir::LLVM::LLVMPointerType>();
+        mlir::Value castedOperand =
+            builder
+                .create<mlir::UnrealizedConversionCastOp>(loc, convertedType,
+                                                          originalDataOperand)
+                .getResult(0);
+        convertedOperands.push_back(castedOperand);
+      } else {
+        // Type not supported.
+        return builder.notifyMatchFailure(op, "expecting a reference type");
+      }
+    }
+
+    builder.replaceOpWithNewOp<Op>(op, TypeRange(), convertedOperands,
+                                   op.getOperation()->getAttrs());
+
+    return success();
+  }
+};
+} // namespace
+
+namespace {
+struct OpenACCDataOperandConversion
+    : public fir::impl::OpenACCDataOperandConversionBase<
+          OpenACCDataOperandConversion> {
+  using Base::Base;
+
+  void runOnOperation() override;
+};
+} // namespace
+
+void OpenACCDataOperandConversion::runOnOperation() {
+  auto op = getOperation();
+  auto *context = op.getContext();
+
+  // Convert to OpenACC operations with LLVM IR dialect
+  RewritePatternSet patterns(context);
+  LowerToLLVMOptions options(context);
+  options.useOpaquePointers = useOpaquePointers;
+  fir::LLVMTypeConverter converter(
+      op.getOperation()->getParentOfType<mlir::ModuleOp>(), true);
+  patterns.add<LegalizeDataOpForLLVMTranslation<acc::DataOp>>(converter);
+  patterns.add<LegalizeDataOpForLLVMTranslation<acc::EnterDataOp>>(converter);
+  patterns.add<LegalizeDataOpForLLVMTranslation<acc::ExitDataOp>>(converter);
+  patterns.add<LegalizeDataOpForLLVMTranslation<acc::ParallelOp>>(converter);
+  patterns.add<LegalizeDataOpForLLVMTranslation<acc::UpdateOp>>(converter);
+
+  ConversionTarget target(*context);
+  target.addLegalDialect<fir::FIROpsDialect>();
+  target.addLegalDialect<LLVM::LLVMDialect>();
+  target.addLegalOp<UnrealizedConversionCastOp>();
+
+  auto allDataOperandsAreConverted = [](ValueRange operands) {
+    for (Value operand : operands) {
+      if (!operand.getType().isa<LLVM::LLVMPointerType>())
+        return false;
+    }
+    return true;
+  };
+
+  target.addDynamicallyLegalOp<acc::DataOp>(
+      [allDataOperandsAreConverted](acc::DataOp op) {
+        return allDataOperandsAreConverted(op.getCopyOperands()) &&
+               allDataOperandsAreConverted(op.getCopyinOperands()) &&
+               allDataOperandsAreConverted(op.getCopyinReadonlyOperands()) &&
+               allDataOperandsAreConverted(op.getCopyoutOperands()) &&
+               allDataOperandsAreConverted(op.getCopyoutZeroOperands()) &&
+               allDataOperandsAreConverted(op.getCreateOperands()) &&
+               allDataOperandsAreConverted(op.getCreateZeroOperands()) &&
+               allDataOperandsAreConverted(op.getNoCreateOperands()) &&
+               allDataOperandsAreConverted(op.getPresentOperands()) &&
+               allDataOperandsAreConverted(op.getDeviceptrOperands()) &&
+               allDataOperandsAreConverted(op.getAttachOperands());
+      });
+
+  target.addDynamicallyLegalOp<acc::EnterDataOp>(
+      [allDataOperandsAreConverted](acc::EnterDataOp op) {
+        return allDataOperandsAreConverted(op.getCopyinOperands()) &&
+               allDataOperandsAreConverted(op.getCreateOperands()) &&
+               allDataOperandsAreConverted(op.getCreateZeroOperands()) &&
+               allDataOperandsAreConverted(op.getAttachOperands());
+      });
+
+  target.addDynamicallyLegalOp<acc::ExitDataOp>(
+      [allDataOperandsAreConverted](acc::ExitDataOp op) {
+        return allDataOperandsAreConverted(op.getCopyoutOperands()) &&
+               allDataOperandsAreConverted(op.getDeleteOperands()) &&
+               allDataOperandsAreConverted(op.getDetachOperands());
+      });
+
+  target.addDynamicallyLegalOp<acc::ParallelOp>(
+      [allDataOperandsAreConverted](acc::ParallelOp op) {
+        return allDataOperandsAreConverted(op.getReductionOperands()) &&
+               allDataOperandsAreConverted(op.getCopyOperands()) &&
+               allDataOperandsAreConverted(op.getCopyinOperands()) &&
+               allDataOperandsAreConverted(op.getCopyinReadonlyOperands()) &&
+               allDataOperandsAreConverted(op.getCopyoutOperands()) &&
+               allDataOperandsAreConverted(op.getCopyoutZeroOperands()) &&
+               allDataOperandsAreConverted(op.getCreateOperands()) &&
+               allDataOperandsAreConverted(op.getCreateZeroOperands()) &&
+               allDataOperandsAreConverted(op.getNoCreateOperands()) &&
+               allDataOperandsAreConverted(op.getPresentOperands()) &&
+               allDataOperandsAreConverted(op.getDevicePtrOperands()) &&
+               allDataOperandsAreConverted(op.getAttachOperands()) &&
+               allDataOperandsAreConverted(op.getGangPrivateOperands()) &&
+               allDataOperandsAreConverted(op.getGangFirstPrivateOperands());
+      });
+
+  target.addDynamicallyLegalOp<acc::UpdateOp>(
+      [allDataOperandsAreConverted](acc::UpdateOp op) {
+        return allDataOperandsAreConverted(op.getHostOperands()) &&
+               allDataOperandsAreConverted(op.getDeviceOperands());
+      });
+
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    signalPassFailure();
+}

flang/test/Transforms/OpenACC/convert-data-operands-to-llvmir.fir

+// RUN: fir-opt -fir-openacc-data-operand-conversion='use-opaque-pointers=1' -split-input-file %s | FileCheck %s
+
+func.func @_QQsub1() attributes {fir.bindc_name = "arr"} {
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+  acc.data copy(%0 : !fir.ref<!fir.array<10xf32>>) {
+    acc.terminator
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @_QQsub1() attributes {fir.bindc_name = "arr"} {
+// CHECK: %[[ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[ADDR]] : !fir.ref<!fir.array<10xf32>> to !llvm.ptr<array<10 x f32>>
+// CHECK: acc.data copy(%[[CAST]] : !llvm.ptr<array<10 x f32>>)
+
+// -----
+
+func.func @_QQsub_enter_exit() attributes {fir.bindc_name = "a"} {
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+  acc.enter_data copyin(%0 : !fir.ref<!fir.array<10xf32>>)
+  acc.exit_data copyout(%0 : !fir.ref<!fir.array<10xf32>>)
+  return
+}
+
+// CHECK-LABEL: func.func @_QQsub_enter_exit() attributes {fir.bindc_name = "a"} {
+// CHECK: %[[ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+// CHECK: %[[CAST0:.*]] = builtin.unrealized_conversion_cast %[[ADDR]] : !fir.ref<!fir.array<10xf32>> to !llvm.ptr<array<10 x f32>>
+// CHECK: acc.enter_data copyin(%[[CAST0]] : !llvm.ptr<array<10 x f32>>)
+// CHECK: %[[CAST1:.*]] = builtin.unrealized_conversion_cast %[[ADDR]] : !fir.ref<!fir.array<10xf32>> to !llvm.ptr<array<10 x f32>>
+// CHECK: acc.exit_data copyout(%[[CAST1]] : !llvm.ptr<array<10 x f32>>)
+
+// -----
+
+func.func @_QQsub_update() attributes {fir.bindc_name = "a"} {
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+  acc.update device(%0 : !fir.ref<!fir.array<10xf32>>)
+  return
+}
+
+// CHECK-LABEL: func.func @_QQsub_update() attributes {fir.bindc_name = "a"} {
+// CHECK: %[[ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[ADDR]] : !fir.ref<!fir.array<10xf32>> to !llvm.ptr<array<10 x f32>>
+// CHECK: acc.update device(%[[CAST]] : !llvm.ptr<array<10 x f32>>)
+
+// -----
+
+func.func @_QQsub_parallel() attributes {fir.bindc_name = "test"} {
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+  %1 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+  acc.parallel copyin(%0: !fir.ref<!fir.array<10xf32>>) {
+    acc.loop {
+      %c1_i32 = arith.constant 1 : i32
+      %2 = fir.convert %c1_i32 : (i32) -> index
+      %c10_i32 = arith.constant 10 : i32
+      %3 = fir.convert %c10_i32 : (i32) -> index
+      %c1 = arith.constant 1 : index
+      %4 = fir.convert %2 : (index) -> i32
+      %5:2 = fir.do_loop %arg0 = %2 to %3 step %c1 iter_args(%arg1 = %4) -> (index, i32) {
+        fir.store %arg1 to %1 : !fir.ref<i32>
+        %6 = fir.load %1 : !fir.ref<i32>
+        %7 = fir.convert %6 : (i32) -> f32
+        %c10_i64 = arith.constant 10 : i64
+        %c1_i64 = arith.constant 1 : i64
+        %8 = arith.subi %c10_i64, %c1_i64 : i64
+        %9 = fir.coordinate_of %0, %8 : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+        fir.store %7 to %9 : !fir.ref<f32>
+        %10 = arith.addi %arg0, %c1 : index
+        %11 = fir.convert %c1 : (index) -> i32
+        %12 = fir.load %1 : !fir.ref<i32>
+        %13 = arith.addi %12, %11 : i32
+        fir.result %10, %13 : index, i32
+      }
+      fir.store %5#1 to %1 : !fir.ref<i32>
+      acc.yield
+    }
+    acc.yield
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @_QQsub_parallel() attributes {fir.bindc_name = "test"} {
+// CHECK: %[[ADDR:.*]] = fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xf32>>
+// CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[ADDR]] : !fir.ref<!fir.array<10xf32>> to !llvm.ptr<array<10 x f32>>
+// CHECK: acc.parallel copyin(%[[CAST]]: !llvm.ptr<array<10 x f32>>) {