diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h --- a/flang/include/flang/Lower/OpenACC.h +++ b/flang/include/flang/Lower/OpenACC.h @@ -62,6 +62,13 @@ createOrGetReductionRecipe(mlir::OpBuilder &, llvm::StringRef, mlir::Location, mlir::Type, mlir::acc::ReductionOperator); +/// Get a acc.firstprivate.recipe op for the given type or create it if it does +/// not exist yet. +mlir::acc::FirstprivateRecipeOp createOrGetFirstprivateRecipe(mlir::OpBuilder &, + llvm::StringRef, + mlir::Location, + mlir::Type); + } // namespace lower } // namespace Fortran diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -499,6 +499,36 @@ return recipe; } +mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( + mlir::OpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc, + mlir::Type ty) { + mlir::ModuleOp mod = + builder.getBlock()->getParent()->getParentOfType(); + if (auto recipe = + mod.lookupSymbol(recipeName)) + return recipe; + + auto crtPos = builder.saveInsertionPoint(); + mlir::OpBuilder modBuilder(mod.getBodyRegion()); + auto recipe = + modBuilder.create(loc, recipeName, ty); + builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), + {ty}, {loc}); + builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); + builder.create( + loc, recipe.getInitRegion().front().getArgument(0)); + + // Add empty copy region for firstprivate. TODO add copy sequence. + builder.createBlock(&recipe.getCopyRegion(), recipe.getCopyRegion().end(), + {ty, ty}, {loc, loc}); + builder.setInsertionPointToEnd(&recipe.getCopyRegion().back()); + builder.create(loc); + + builder.restoreInsertionPoint(crtPos); + return recipe; +} + +template static void genPrivatizations(const Fortran::parser::AccObjectList &objectList, Fortran::lower::AbstractConverter &converter, @@ -514,11 +544,18 @@ mlir::Value baseAddr = gatherDataOperandAddrAndBounds( converter, builder, semanticsContext, stmtCtx, accObject, operandLocation, asFortran, bounds); - std::string recipeName = fir::getTypeAsString( - baseAddr.getType(), converter.getKindMap(), "privatization"); - mlir::acc::PrivateRecipeOp recipe = - Fortran::lower::createOrGetPrivateRecipe( - builder, recipeName, operandLocation, baseAddr.getType()); + Op recipe; + if constexpr (std::is_same_v) { + std::string recipeName = fir::getTypeAsString( + baseAddr.getType(), converter.getKindMap(), "privatization"); + recipe = Fortran::lower::createOrGetPrivateRecipe( + builder, recipeName, operandLocation, baseAddr.getType()); + } else { + std::string recipeName = fir::getTypeAsString( + baseAddr.getType(), converter.getKindMap(), "firstprivatization"); + recipe = Fortran::lower::createOrGetFirstprivateRecipe( + builder, recipeName, operandLocation, baseAddr.getType()); + } privatizations.push_back(mlir::SymbolRefAttr::get( builder.getContext(), recipe.getSymName().str())); dataOperands.push_back(baseAddr); @@ -869,8 +906,9 @@ } else if (const auto *privateClause = std::get_if( &clause.u)) { - genPrivatizations(privateClause->v, converter, semanticsContext, stmtCtx, - privateOperands, privatizations); + genPrivatizations( + privateClause->v, converter, semanticsContext, stmtCtx, + privateOperands, privatizations); } else if (const auto *reductionClause = std::get_if( &clause.u)) { @@ -1004,7 +1042,8 @@ llvm::SmallVector reductionOperands, privateOperands, firstprivateOperands; - llvm::SmallVector privatizations, reductionRecipes; + llvm::SmallVector privatizations, firstPrivatizations, + reductionRecipes; // Async, wait and self clause have optional values but can be present with // no value as well. When there is no value, the op has an attribute to @@ -1151,13 +1190,15 @@ } else if (const auto *privateClause = std::get_if( &clause.u)) { - genPrivatizations(privateClause->v, converter, semanticsContext, stmtCtx, - privateOperands, privatizations); + genPrivatizations( + privateClause->v, converter, semanticsContext, stmtCtx, + privateOperands, privatizations); } else if (const auto *firstprivateClause = std::get_if( &clause.u)) { - genObjectList(firstprivateClause->v, converter, semanticsContext, stmtCtx, - firstprivateOperands); + genPrivatizations( + firstprivateClause->v, converter, semanticsContext, stmtCtx, + firstprivateOperands, firstPrivatizations); } else if (const auto *reductionClause = std::get_if( &clause.u)) { @@ -1207,6 +1248,9 @@ if (!reductionRecipes.empty()) computeOp.setReductionRecipesAttr( mlir::ArrayAttr::get(builder.getContext(), reductionRecipes)); + if (!firstPrivatizations.empty()) + computeOp.setFirstprivatizationsAttr( + mlir::ArrayAttr::get(builder.getContext(), firstPrivatizations)); } auto insPt = builder.saveInsertionPoint(); diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 --- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -456,7 +464,7 @@ a(i) = b(i) END DO -! CHECK: acc.parallel firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: acc.loop private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: fir.do_loop ! CHECK: acc.yield diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90 --- a/flang/test/Lower/OpenACC/acc-parallel.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10x10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -300,7 +308,7 @@ !$acc parallel private(a) firstprivate(b) private(c) !$acc end parallel -! CHECK: acc.parallel firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_10x10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90 --- a/flang/test/Lower/OpenACC/acc-serial-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -372,7 +380,7 @@ a(i) = b(i) END DO -! CHECK: acc.serial firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { +! CHECK: acc.serial firstprivate(@firstprivatization_ref_10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: acc.loop private(@privatization_ref_10xf32 -> %[[A]] : !fir.ref>) { ! CHECK: fir.do_loop ! CHECK: acc.yield diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90 --- a/flang/test/Lower/OpenACC/acc-serial.f90 +++ b/flang/test/Lower/OpenACC/acc-serial.f90 @@ -2,6 +2,14 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { +! CHECK: ^bb0(%arg0: !fir.ref>): +! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_10x10xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: acc.yield %{{.*}} : !fir.ref> @@ -243,7 +251,7 @@ !$acc serial private(a) firstprivate(b) private(c) !$acc end serial -! CHECK: acc.serial firstprivate(%[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { +! CHECK: acc.serial firstprivate(@firstprivatization_ref_10x10xf32 -> %[[B]] : !fir.ref>) private(@privatization_ref_10x10xf32 -> %[[A]] : !fir.ref>, @privatization_ref_10x10xf32 -> %[[C]] : !fir.ref>) { ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -639,7 +639,8 @@ OptionalAttr:$reductionRecipes, Variadic:$gangPrivateOperands, OptionalAttr:$privatizations, - Variadic:$gangFirstPrivateOperands, + Variadic:$gangFirstPrivateOperands, + OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, OptionalAttr:$defaultAttr); @@ -657,8 +658,9 @@ oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` $async `:` type($async) `)` - | `firstprivate` `(` $gangFirstPrivateOperands `:` - type($gangFirstPrivateOperands) `)` + | `firstprivate` `(` custom($gangFirstPrivateOperands, + type($gangFirstPrivateOperands), $firstprivatizations) + `)` | `num_gangs` `(` $numGangs `:` type($numGangs) `)` | `num_workers` `(` $numWorkers `:` type($numWorkers) `)` | `private` `(` custom( @@ -709,7 +711,8 @@ OptionalAttr:$reductionRecipes, Variadic:$gangPrivateOperands, OptionalAttr:$privatizations, - Variadic:$gangFirstPrivateOperands, + Variadic:$gangFirstPrivateOperands, + OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, OptionalAttr:$defaultAttr); @@ -727,8 +730,9 @@ oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` $async `:` type($async) `)` - | `firstprivate` `(` $gangFirstPrivateOperands `:` - type($gangFirstPrivateOperands) `)` + | `firstprivate` `(` custom($gangFirstPrivateOperands, + type($gangFirstPrivateOperands), $firstprivatizations) + `)` | `private` `(` custom( $gangPrivateOperands, type($gangPrivateOperands), $privatizations) `)` diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -396,6 +396,19 @@ acc.terminator } +acc.firstprivate.recipe @privatization_memref_10xf32 : memref<10xf32> init { +^bb0(%arg0: memref<10xf32>): + %0 = memref.alloc() : memref<10xf32> + acc.yield %0 : memref<10xf32> +} copy { +^bb0(%arg0: memref<10xf32>, %arg1: memref<10xf32>): + acc.terminator +} destroy { +^bb0(%arg0: memref<10xf32>): + memref.dealloc %arg0 : memref<10xf32> + acc.terminator +} + func.func @testparallelop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10xf32>) -> () { %i64value = arith.constant 1 : i64 %i32value = arith.constant 1 : i32 @@ -432,7 +445,7 @@ } acc.parallel vector_length(%idxValue: index) { } - acc.parallel private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(%b: memref<10xf32>) { + acc.parallel private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@privatization_memref_10xf32 -> %b: memref<10xf32>) { } acc.parallel { } attributes {defaultAttr = #acc} @@ -483,7 +496,7 @@ // CHECK-NEXT: } // CHECK: acc.parallel vector_length([[IDXVALUE]] : index) { // CHECK-NEXT: } -// CHECK: acc.parallel firstprivate([[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { +// CHECK: acc.parallel firstprivate(@privatization_memref_10xf32 -> [[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { // CHECK-NEXT: } // CHECK: acc.parallel { // CHECK-NEXT: } attributes {defaultAttr = #acc} @@ -518,7 +531,19 @@ acc.terminator } -acc.firstprivate.recipe @privatization_memref_10xf32 : memref<10xf32> init { +// Test optional destroy region +acc.firstprivate.recipe @firstprivatization_memref_20xf32 : memref<20xf32> init { +^bb0(%arg0: memref<20xf32>): + %0 = memref.alloc() : memref<20xf32> + acc.yield %0 : memref<20xf32> +} copy { +^bb0(%arg0: memref<20xf32>, %arg1: memref<20xf32>): + acc.terminator +} + +// CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_memref_20xf32 : memref<20xf32> init + +acc.firstprivate.recipe @firstprivatization_memref_10xf32 : memref<10xf32> init { ^bb0(%arg0: memref<10xf32>): %0 = memref.alloc() : memref<10xf32> acc.yield %0 : memref<10xf32> @@ -531,18 +556,6 @@ acc.terminator } -// Test optional destroy region -acc.firstprivate.recipe @privatization_memref_20xf32 : memref<20xf32> init { -^bb0(%arg0: memref<20xf32>): - %0 = memref.alloc() : memref<20xf32> - acc.yield %0 : memref<20xf32> -} copy { -^bb0(%arg0: memref<20xf32>, %arg1: memref<20xf32>): - acc.terminator -} - -// CHECK-LABEL: acc.firstprivate.recipe @privatization_memref_20xf32 : memref<20xf32> init - func.func @testserialop(%a: memref<10xf32>, %b: memref<10xf32>, %c: memref<10x10xf32>) -> () { %i64value = arith.constant 1 : i64 %i32value = arith.constant 1 : i32 @@ -561,7 +574,7 @@ } acc.serial wait(%i64value, %i32value, %idxValue : i64, i32, index) { } - acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(%b: memref<10xf32>) { + acc.serial private(@privatization_memref_10_f32 -> %a : memref<10xf32>, @privatization_memref_10_10_f32 -> %c : memref<10x10xf32>) firstprivate(@firstprivatization_memref_10xf32 -> %b: memref<10xf32>) { } acc.serial { } attributes {defaultAttr = #acc} @@ -597,7 +610,7 @@ // CHECK-NEXT: } // CHECK: acc.serial wait([[I64VALUE]], [[I32VALUE]], [[IDXVALUE]] : i64, i32, index) { // CHECK-NEXT: } -// CHECK: acc.serial firstprivate([[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { +// CHECK: acc.serial firstprivate(@firstprivatization_memref_10xf32 -> [[ARGB]] : memref<10xf32>) private(@privatization_memref_10_f32 -> [[ARGA]] : memref<10xf32>, @privatization_memref_10_10_f32 -> [[ARGC]] : memref<10x10xf32>) { // CHECK-NEXT: } // CHECK: acc.serial { // CHECK-NEXT: } attributes {defaultAttr = #acc}