diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h --- a/flang/include/flang/Lower/OpenACC.h +++ b/flang/include/flang/Lower/OpenACC.h @@ -22,6 +22,7 @@ class Type; class OpBuilder; namespace acc { +class FirstprivateRecipeOp; class PrivateRecipeOp; } } // namespace mlir @@ -57,6 +58,13 @@ llvm::StringRef, mlir::Location, mlir::Type); +/// Get a acc.firstprivate.recipe op for the given type or create it if it does +/// not exist yet. +mlir::acc::FirstprivateRecipeOp createOrGetFirstprivateRecipe(mlir::OpBuilder &, + llvm::StringRef, + mlir::Location, + mlir::Type); + } // namespace lower } // namespace Fortran diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -500,6 +500,36 @@ return recipe; } +mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( + mlir::OpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc, + mlir::Type ty) { + mlir::ModuleOp mod = + builder.getBlock()->getParent()->getParentOfType(); + if (auto recipe = + mod.lookupSymbol(recipeName)) + return recipe; + + auto crtPos = builder.saveInsertionPoint(); + mlir::OpBuilder modBuilder(mod.getBodyRegion()); + auto recipe = + modBuilder.create(loc, recipeName, ty); + builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), + {ty}, {loc}); + builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); + builder.create( + loc, recipe.getInitRegion().front().getArgument(0)); + + // Add emtpy copy region for firstprivate. TODO add copy sequence. + builder.createBlock(&recipe.getCopyRegion(), recipe.getCopyRegion().end(), + {ty, ty}, {loc, loc}); + builder.setInsertionPointToEnd(&recipe.getCopyRegion().back()); + builder.create(loc); + + builder.restoreInsertionPoint(crtPos); + return recipe; +} + +template static void genPrivatizations(const Fortran::parser::AccObjectList &objectList, Fortran::lower::AbstractConverter &converter, @@ -515,11 +545,18 @@ mlir::Value baseAddr = gatherDataOperandAddrAndBounds( converter, builder, semanticsContext, stmtCtx, accObject, operandLocation, asFortran, bounds); - std::string recipeName = fir::getTypeAsString( - baseAddr.getType(), converter.getKindMap(), "privatization"); - mlir::acc::PrivateRecipeOp recipe = - Fortran::lower::createOrGetPrivateRecipe( - builder, recipeName, operandLocation, baseAddr.getType()); + Op recipe; + if constexpr (std::is_same_v) { + std::string recipeName = fir::getTypeAsString( + baseAddr.getType(), converter.getKindMap(), "privatization"); + recipe = Fortran::lower::createOrGetPrivateRecipe( + builder, recipeName, operandLocation, baseAddr.getType()); + } else { + std::string recipeName = fir::getTypeAsString( + baseAddr.getType(), converter.getKindMap(), "firstprivatization"); + recipe = Fortran::lower::createOrGetFirstprivateRecipe( + builder, recipeName, operandLocation, baseAddr.getType()); + } privatizations.push_back(mlir::SymbolRefAttr::get( builder.getContext(), recipe.getSymName().str())); dataOperands.push_back(baseAddr); @@ -756,8 +793,9 @@ } else if (const auto *privateClause = std::get_if( &clause.u)) { - genPrivatizations(privateClause->v, converter, semanticsContext, stmtCtx, - privateOperands, privatizations); + genPrivatizations( + privateClause->v, converter, semanticsContext, stmtCtx, + privateOperands, privatizations); } else if (std::get_if(&clause.u)) { // Reduction clause is left out for the moment as the clause will probably // end up having its own operation. @@ -879,7 +917,7 @@ llvm::SmallVector reductionOperands, privateOperands, firstprivateOperands; - llvm::SmallVector privatizations; + llvm::SmallVector privatizations, firstPrivatizations; // Async, wait and self clause have optional values but can be present with // no value as well. When there is no value, the op has an attribute to @@ -1026,13 +1064,15 @@ } else if (const auto *privateClause = std::get_if( &clause.u)) { - genPrivatizations(privateClause->v, converter, semanticsContext, stmtCtx, - privateOperands, privatizations); + genPrivatizations( + privateClause->v, converter, semanticsContext, stmtCtx, + privateOperands, privatizations); } else if (const auto *firstprivateClause = std::get_if( &clause.u)) { - genObjectList(firstprivateClause->v, converter, semanticsContext, stmtCtx, - firstprivateOperands); + genPrivatizations( + firstprivateClause->v, converter, semanticsContext, stmtCtx, + firstprivateOperands, firstPrivatizations); } else if (std::get_if(&clause.u)) { TODO(clauseLocation, "compute construct reduction clause lowering"); } @@ -1076,6 +1116,9 @@ if (!privatizations.empty()) computeOp.setPrivatizationsAttr( mlir::ArrayAttr::get(builder.getContext(), privatizations)); + if (!firstPrivatizations.empty()) + computeOp.setFirstprivatizationsAttr( + mlir::ArrayAttr::get(builder.getContext(), firstPrivatizations)); } auto insPt = builder.saveInsertionPoint(); diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 --- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -454,7 +462,7 @@ a(i) = b(i) END DO -! CHECK: acc.parallel firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: acc.loop private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: fir.do_loop ! CHECK: acc.yield diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90 --- a/flang/test/Lower/OpenACC/acc-parallel.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10x10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -298,7 +306,7 @@ !$acc parallel private(a) firstprivate(b) private(c) !$acc end parallel -! CHECK: acc.parallel firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_10x10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90 --- a/flang/test/Lower/OpenACC/acc-serial-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -370,7 +378,7 @@ a(i) = b(i) END DO -! CHECK: acc.serial firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { +! CHECK: acc.serial firstprivate(@firstprivatization_ref_10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: acc.loop private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: fir.do_loop ! CHECK: acc.yield diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90 --- a/flang/test/Lower/OpenACC/acc-serial.f90 +++ b/flang/test/Lower/OpenACC/acc-serial.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10x10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -241,7 +249,7 @@ !$acc serial private(a) firstprivate(b) private(c) !$acc end serial -! CHECK: acc.serial firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { +! CHECK: acc.serial firstprivate(@firstprivatization_ref_10x10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -639,7 +639,8 @@ Variadic:$reductionOperands, Variadic:$gangPrivateOperands, OptionalAttr:$privatizations, - Variadic:$gangFirstPrivateOperands, + Variadic:$gangFirstPrivateOperands, + OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, OptionalAttr:$defaultAttr); @@ -657,8 +658,9 @@ oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` $async `:` type($async) `)` - | `firstprivate` `(` $gangFirstPrivateOperands `:` - type($gangFirstPrivateOperands) `)` + | `firstprivate` `(` custom($gangFirstPrivateOperands, + type($gangFirstPrivateOperands), $firstprivatizations) + `)` | `num_gangs` `(` $numGangs `:` type($numGangs) `)` | `num_workers` `(` $numWorkers `:` type($numWorkers) `)` | `private` `(` custom( @@ -707,7 +709,8 @@ Variadic:$reductionOperands, Variadic:$gangPrivateOperands, OptionalAttr:$privatizations, - Variadic:$gangFirstPrivateOperands, + Variadic:$gangFirstPrivateOperands, + OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, OptionalAttr:$defaultAttr); @@ -725,8 +728,9 @@ oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` $async `:` type($async) `)` - | `firstprivate` `(` $gangFirstPrivateOperands `:` - type($gangFirstPrivateOperands) `)` + | `firstprivate` `(` custom($gangFirstPrivateOperands, + type($gangFirstPrivateOperands), $firstprivatizations) + `)` | `private` `(` custom( $gangPrivateOperands, type($gangPrivateOperands), $privatizations) `)` diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -388,6 +388,19 @@ acc.terminator } +acc.firstprivate.recipe @privatization_memref_10xf32 : memref<10xf32> init { +^bb0(%arg0: memref<10xf32>): + %0 = memref.alloc() : memref<10xf32> + acc.yield %0 : memref<10xf32> +} copy { +^bb0(%arg0: memref<10xf32>, %arg1: memref<10xf32>): + acc.terminator +} destroy { +^bb0(%arg0: memref<10xf32>): + memref.dealloc %arg0 : memref<10xf32> + acc.terminator +} + func.func @testparallelop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10xf32>) -> () { %i64value = arith.constant 1 : i64 %i32value = arith.constant 1 : i32 @@ -424,7 +437,7 @@ } acc.parallel vector_length(%idxValue: index) { } - acc.parallel private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(%b: memref<10xf32>) { + acc.parallel private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@privatization_memref_10xf32 -> %b: memref<10xf32>) { } acc.parallel { } attributes {defaultAttr = #acc} @@ -475,7 +488,7 @@ // CHECK-NEXT: } // CHECK: acc.parallel vector_length([[IDXVALUE]] : index) { // CHECK-NEXT: } -// CHECK: acc.parallel firstprivate([[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { +// CHECK: acc.parallel firstprivate(@privatization_memref_10xf32 -> [[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { // CHECK-NEXT: } // CHECK: acc.parallel { // CHECK-NEXT: } attributes {defaultAttr = #acc} @@ -522,6 +535,19 @@ // CHECK-LABEL: acc.firstprivate.recipe @privatization_memref_20xf32 : memref<20xf32> init +acc.firstprivate.recipe @privatization_memref_10xf32 : memref<10xf32> init { +^bb0(%arg0: memref<10xf32>): + %0 = memref.alloc() : memref<10xf32> + acc.yield %0 : memref<10xf32> +} copy { +^bb0(%arg0: memref<10xf32>, %arg1: memref<10xf32>): + acc.terminator +} destroy { +^bb0(%arg0: memref<10xf32>): + memref.dealloc %arg0 : memref<10xf32> + acc.terminator +} + func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10xf32>) -> () { %i64value = arith.constant 1 : i64 %i32value = arith.constant 1 : i32 @@ -540,7 +566,7 @@ } acc.serial wait(%i64value, %i32value, %idxValue : i64, i32, index) { } - acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(%b: memref<10xf32>) { + acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@privatization_memref_10xf32 -> %b: memref<10xf32>) { } acc.serial { } attributes {defaultAttr = #acc} @@ -576,7 +602,7 @@ // CHECK-NEXT: } // CHECK: acc.serial wait([[I64VALUE]], [[I32VALUE]], [[IDXVALUE]] : i64, i32, index) { // CHECK-NEXT: } -// CHECK: acc.serial firstprivate([[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { +// CHECK: acc.serial firstprivate(@privatization_memref_10xf32 -> [[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { // CHECK-NEXT: } // CHECK: acc.serial { // CHECK-NEXT: } attributes {defaultAttr = #acc}