diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -179,7 +179,8 @@ mlir::Location &loc, Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OmpClauseList *clauses = nullptr, const SmallVector &args = {}, - bool outerCombined = false) { + bool outerCombined = false, + const Fortran::parser::Expr *expr = nullptr) { fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); // If an argument for the region is provided then create the block with that // argument. Also update the symbol's address with the mlir argument value. @@ -190,11 +191,21 @@ std::size_t loopVarTypeSize = 0; for (const Fortran::semantics::Symbol *arg : args) loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size()); - mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize); + mlir::Type varType; + if constexpr (std::is_same_v) { + // In case of AtomicUpdate assignment statement, let LHS variable type = + // RHS expression type + Fortran::lower::StatementContext stmtCtx; + mlir::Value result = fir::getBase( + converter.genExprValue(*Fortran::semantics::GetExpr(*expr), stmtCtx)); + varType = result.getType(); + } else { + varType = getLoopVarType(converter, loopVarTypeSize); + } SmallVector tiv; SmallVector locs; for (int i = 0; i < (int)args.size(); i++) { - tiv.push_back(loopVarType); + tiv.push_back(varType); locs.push_back(loc); } firOpBuilder.createBlock(&op.getRegion(), {}, tiv, locs); @@ -205,7 +216,7 @@ mlir::Value val = fir::getBase(op.getRegion().front().getArgument(argIndex)); mlir::Value temp = firOpBuilder.createTemporary( - loc, loopVarType, + loc, varType, llvm::ArrayRef{ Fortran::lower::getAdaptToByRefAttr(firOpBuilder)}); storeOp = firOpBuilder.create(loc, val, temp); @@ -230,6 +241,11 @@ if constexpr (std::is_same_v) { mlir::ValueRange results; firOpBuilder.create(loc, results); + } else if constexpr (std::is_same_v) { + Fortran::lower::StatementContext stmtCtx; + auto result = fir::getBase( + converter.genExprValue(*Fortran::semantics::GetExpr(*expr), stmtCtx)); + firOpBuilder.create(loc, result); } else { firOpBuilder.create(loc); } @@ -931,6 +947,87 @@ to_address, hint, memory_order); } +static void +genOmpAtomicUpdate(Fortran::lower::AbstractConverter &converter, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpAtomicUpdate &atomicUpdate) { + auto &firOpBuilder = converter.getFirOpBuilder(); + auto currentLocation = converter.getCurrentLocation(); + mlir::Value address; + SmallVector symbolVector; + const Fortran::parser::OmpAtomicClauseList &rightHandClauseList = + std::get<2>(atomicUpdate.t); + const Fortran::parser::OmpAtomicClauseList &leftHandClauseList = + std::get<0>(atomicUpdate.t); + const auto &assignmentStmtExpr = + std::get(std::get<3>(atomicUpdate.t).statement.t); + const auto &assignmentStmtVariable = std::get( + std::get<3>(atomicUpdate.t).statement.t); + Fortran::lower::StatementContext stmtCtx; + if (auto varDesignator = std::get_if< + Fortran::common::Indirection>( + &assignmentStmtVariable.u)) { + if (const auto *name = getDesignatorNameIfDataRef(varDesignator->value())) { + address = fir::getBase(converter.genExprAddr( + *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx)); + symbolVector.push_back(name->symbol); + } + } + // If no hint clause is specified, the effect is as if + // hint(omp_sync_hint_none) had been specified. + mlir::IntegerAttr hint = nullptr; + mlir::omp::ClauseMemoryOrderKindAttr memory_order = nullptr; + genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint, + memory_order); + genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint, + memory_order); + auto atomicUpdateOp = firOpBuilder.create( + currentLocation, address, hint, memory_order); + createBodyOfOp(atomicUpdateOp, converter, + currentLocation, eval, nullptr, + symbolVector, false, &assignmentStmtExpr); +} + +static void genOmpAtomic(Fortran::lower::AbstractConverter &converter, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpAtomic &atomicConstruct) { + auto &firOpBuilder = converter.getFirOpBuilder(); + auto currentLocation = converter.getCurrentLocation(); + mlir::Value address; + SmallVector symbolVector; + const Fortran::parser::OmpAtomicClauseList &atomicClauseList = + std::get(atomicConstruct.t); + const auto &assignmentStmtExpr = std::get( + std::get>( + atomicConstruct.t) + .statement.t); + const auto &assignmentStmtVariable = std::get( + std::get>( + atomicConstruct.t) + .statement.t); + Fortran::lower::StatementContext stmtCtx; + if (auto varDesignator = std::get_if< + Fortran::common::Indirection>( + &assignmentStmtVariable.u)) { + if (const auto *name = getDesignatorNameIfDataRef(varDesignator->value())) { + address = fir::getBase(converter.genExprAddr( + *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx)); + symbolVector.push_back(name->symbol); + } + } + // If no hint clause is specified, the effect is as if + // hint(omp_sync_hint_none) had been specified. + mlir::IntegerAttr hint = nullptr; + mlir::omp::ClauseMemoryOrderKindAttr memory_order = nullptr; + genOmpAtomicHintAndMemoryOrderClauses(converter, atomicClauseList, hint, + memory_order); + auto atomicUpdateOp = firOpBuilder.create( + currentLocation, address, hint, memory_order); + createBodyOfOp(atomicUpdateOp, converter, + currentLocation, eval, nullptr, + symbolVector, false, &assignmentStmtExpr); +} + static void genOMP(Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, @@ -942,9 +1039,14 @@ [&](const Fortran::parser::OmpAtomicWrite &atomicWrite) { genOmpAtomicWrite(converter, eval, atomicWrite); }, + [&](const Fortran::parser::OmpAtomicUpdate &atomicUpdate) { + genOmpAtomicUpdate(converter, eval, atomicUpdate); + }, + [&](const Fortran::parser::OmpAtomic &atomicConstruct) { + genOmpAtomic(converter, eval, atomicConstruct); + }, [&](const auto &) { - TODO(converter.getCurrentLocation(), - "Atomic update & capture"); + TODO(converter.getCurrentLocation(), "Atomic capture"); }, }, atomicConstruct.u); diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90 new file mode 100644 --- /dev/null +++ b/flang/test/Lower/OpenMP/atomic-update.f90 @@ -0,0 +1,155 @@ +! This test checks lowering of atomic update construct +! RUN: bbc -fopenmp -emit-fir %s -o - | \ +! RUN: FileCheck %s + +program OmpAtomicUpdate + use omp_lib + integer :: x, y, z + integer, pointer :: a, b + integer, target :: c, d + a=>c + b=>d + +!CHECK: %[[TEMP_1:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_2:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_3:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_4:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_5:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_6:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_7:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_8:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_9:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: %[[TEMP_10:.*]] = fir.alloca i32 {adapt.valuebyref} +!CHECK: {{.*}} = fir.alloca !fir.box> {bindc_name = "a", uniq_name = "_QFEa"} +!CHECK: {{.*}} = fir.alloca !fir.ptr {uniq_name = "_QFEa.addr"} +!CHECK: {{.*}} = fir.zero_bits !fir.ptr +!CHECK: fir.store {{.*}} to {{.*}} : !fir.ref> +!CHECK: {{.*}} = fir.alloca !fir.box> {bindc_name = "b", uniq_name = "_QFEb"} +!CHECK: %[[b_ADDR:.*]] = fir.alloca !fir.ptr {uniq_name = "_QFEb.addr"} +!CHECK: {{.*}} = fir.zero_bits !fir.ptr +!CHECK: fir.store {{.*}} to {{.*}} : !fir.ref> +!CHECK: {{.*}} = fir.address_of(@_QFEc) : !fir.ref +!CHECK: {{.*}} = fir.address_of(@_QFEd) : !fir.ref +!CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} +!CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} +!CHECK: %[[VAR_Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFEz"} +!CHECK: {{.*}} = fir.convert {{.*}} : (!fir.ref) -> !fir.ptr +!CHECK: fir.store {{.*}} to {{.*}} : !fir.ref> +!CHECK: {{.*}} = fir.convert {{.*}} : (!fir.ref) -> !fir.ptr +!CHECK: fir.store {{.*}} to {{.*}} : !fir.ref> +!CHECK: %[[LOADED_a_ADDR:.*]] = fir.load {{.*}} : !fir.ref> + + +!CHECK: omp.atomic.update %[[LOADED_a_ADDR]] : !fir.ptr { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_10]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[TEMP_10]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[b_ADDR]] : !fir.ref> +!CHECK: {{.*}} = fir.load {{.*}} : !fir.ptr +!CHECK: %[[RESULT:.*]] = arith.addi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } + !$omp atomic update + a = a + b + + +!CHECK: omp.atomic.update %[[VAR_Y]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_9]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[TEMP_9]] : !fir.ref +!CHECK: {{.*}} = arith.constant 1 : i32 +!CHECK: %[[RESULT:.*]] = arith.addi %{{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } +!CHECK: omp.atomic.update %[[VAR_Z]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_8]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_X]] : !fir.ref +!CHECK: %{{.*}} = fir.load %[[TEMP_8]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.muli {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } + !$omp atomic + y = y + 1 + !$omp atomic update + z = x * z + +!CHECK: omp.atomic.update memory_order(relaxed) hint(uncontended) %[[VAR_X]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_7]] : !fir.ref +!CHECK: %{{.*}} = fir.load %[[TEMP_7]] : !fir.ref +!CHECK: %{{.*}} = arith.constant 1 : i32 +!CHECK: %[[RESULT:.*]] = arith.subi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK:} +!CHECK: omp.atomic.update memory_order(relaxed) %[[VAR_Y]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_6]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_X]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[TEMP_6]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_Z]] : !fir.ref +!CHECK: {{.*}} = arith.cmpi sgt, {{.*}}, {{.*}} : i32 +!CHECK: {{.*}} = arith.select {{.*}}, {{.*}}, {{.*}} : i32 +!CHECK: {{.*}} = arith.cmpi sgt, {{.*}}, {{.*}} : i32 +!CHECK: %[[RESULT:.*]] = arith.select {{.*}}, {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } +!CHECK: omp.atomic.update memory_order(relaxed) hint(contended) %[[VAR_Z]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_5]] : !fir.ref +!CHECK: %{{.*}} = fir.load %[[TEMP_5]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_X]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.addi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } + !$omp atomic relaxed update hint(omp_sync_hint_uncontended) + x = x - 1 + !$omp atomic update relaxed + y = max(x, y, z) + !$omp atomic relaxed hint(omp_sync_hint_contended) + z = z + x + +!CHECK: omp.atomic.update memory_order(release) hint(contended) %[[VAR_Z]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_4]] : !fir.ref +!CHECK: {{.*}} = arith.constant 10 : i32 +!CHECK: {{.*}} = fir.load %[[TEMP_4]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.muli {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } +!CHECK: omp.atomic.update memory_order(release) hint(speculative) %[[VAR_X]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_3]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[TEMP_3]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_Z]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.divsi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } + !$omp atomic release update hint(omp_lock_hint_contended) + z = z * 10 + !$omp atomic hint(omp_lock_hint_speculative) update release + x = x / z + +!CHECK: omp.atomic.update memory_order(seq_cst) hint(nonspeculative) %[[VAR_Y]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_2]] : !fir.ref +!CHECK: {{.*}} = arith.constant 10 : i32 +!CHECK: {{.*}} = fir.load %[[TEMP_2]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.addi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } +!CHECK: omp.atomic.update memory_order(seq_cst) %[[VAR_Z]] : !fir.ref { +!CHECK: ^bb0(%[[ARG:.*]]: i32): +!CHECK: fir.store %[[ARG]] to %[[TEMP_1]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[VAR_Y]] : !fir.ref +!CHECK: {{.*}} = fir.load %[[TEMP_1]] : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.addi {{.*}}, {{.*}} : i32 +!CHECK: omp.yield(%[[RESULT]] : i32) +!CHECK: } +!CHECK: return +!CHECK: } + !$omp atomic hint(omp_sync_hint_nonspeculative) seq_cst + y = 10 + y + !$omp atomic seq_cst update + z = y + z +end program OmpAtomicUpdate