diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -373,6 +373,161 @@ } } +static void genOmpAtomicHintAndMemoryOrderClauses( + Fortran::lower::AbstractConverter &converter, + const Fortran::parser::OmpAtomicClauseList &clauseList, + mlir::IntegerAttr &hint, mlir::StringAttr &memory_order) { + auto &firOpBuilder = converter.getFirOpBuilder(); + for (const auto &clause : clauseList.v) { + if (auto ompClause = std::get_if(&clause.u)) { + if (auto hintClause = + std::get_if(&ompClause->u)) { + const auto *expr = Fortran::semantics::GetExpr(hintClause->v); + uint64_t hintExprValue = *Fortran::evaluate::ToInt64(*expr); + if (hintExprValue == 0) + hint = nullptr; + else + hint = firOpBuilder.getI64IntegerAttr(hintExprValue); + } + } else if (auto ompMemoryOrderClause = + std::get_if( + &clause.u)) { + if (std::get_if( + &ompMemoryOrderClause->v.u)) { + memory_order = + firOpBuilder.getStringAttr(omp::stringifyClauseMemoryOrderKind( + omp::ClauseMemoryOrderKind::Acquire)); + } else if (std::get_if( + &ompMemoryOrderClause->v.u)) { + memory_order = + firOpBuilder.getStringAttr(omp::stringifyClauseMemoryOrderKind( + omp::ClauseMemoryOrderKind::Relaxed)); + } else if (std::get_if( + &ompMemoryOrderClause->v.u)) { + memory_order = + firOpBuilder.getStringAttr(omp::stringifyClauseMemoryOrderKind( + omp::ClauseMemoryOrderKind::Seq_cst)); + } else if (std::get_if( + &ompMemoryOrderClause->v.u)) { + memory_order = + firOpBuilder.getStringAttr(omp::stringifyClauseMemoryOrderKind( + omp::ClauseMemoryOrderKind::Release)); + } + } + } +} + +static void +genOmpAtomicWrite(Fortran::lower::AbstractConverter &converter, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpAtomicWrite &atomicWrite) { + auto &firOpBuilder = converter.getFirOpBuilder(); + auto currentLocation = converter.getCurrentLocation(); + mlir::Value address; + // If no hint clause is specified, the effect is as if + // hint(omp_sync_hint_none) had been specified. + mlir::IntegerAttr hint = nullptr; + mlir::StringAttr memory_order = nullptr; + const Fortran::parser::OmpAtomicClauseList &rightHandClauseList = + std::get<2>(atomicWrite.t); + const Fortran::parser::OmpAtomicClauseList &leftHandClauseList = + std::get<0>(atomicWrite.t); + const auto &assignmentStmtExpr = + std::get(std::get<3>(atomicWrite.t).statement.t); + const auto &assignmentStmtVariable = std::get( + std::get<3>(atomicWrite.t).statement.t); + Fortran::lower::StatementContext stmtCtx; + auto value = fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(assignmentStmtExpr), stmtCtx)); + if (auto varDesignator = std::get_if< + Fortran::common::Indirection>( + &assignmentStmtVariable.u)) { + if (const auto *name = getDesignatorNameIfDataRef(varDesignator->value())) { + address = converter.getSymbolAddress(*name->symbol); + } + } + + genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint, + memory_order); + genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint, + memory_order); + firOpBuilder.create( + currentLocation, address, value, hint ? hint : nullptr, + memory_order + ? mlir::omp::ClauseMemoryOrderKindAttr::get( + firOpBuilder.getContext(), + omp::symbolizeClauseMemoryOrderKind(memory_order).getValue()) + : nullptr); +} + +static void genOmpAtomicRead(Fortran::lower::AbstractConverter &converter, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OmpAtomicRead &atomicRead) { + auto &firOpBuilder = converter.getFirOpBuilder(); + auto currentLocation = converter.getCurrentLocation(); + mlir::Value to_address; + mlir::Value from_address; + // If no hint clause is specified, the effect is as if + // hint(omp_sync_hint_none) had been specified. + mlir::IntegerAttr hint = nullptr; + mlir::StringAttr memory_order = nullptr; + const Fortran::parser::OmpAtomicClauseList &rightHandClauseList = + std::get<2>(atomicRead.t); + const Fortran::parser::OmpAtomicClauseList &leftHandClauseList = + std::get<0>(atomicRead.t); + const auto &assignmentStmtExpr = + std::get(std::get<3>(atomicRead.t).statement.t); + const auto &assignmentStmtVariable = std::get( + std::get<3>(atomicRead.t).statement.t); + if (auto exprDesignator = std::get_if< + Fortran::common::Indirection>( + &assignmentStmtExpr.u)) { + if (const auto *name = + getDesignatorNameIfDataRef(exprDesignator->value())) { + from_address = converter.getSymbolAddress(*name->symbol); + } + } + + if (auto varDesignator = std::get_if< + Fortran::common::Indirection>( + &assignmentStmtVariable.u)) { + if (const auto *name = getDesignatorNameIfDataRef(varDesignator->value())) { + to_address = converter.getSymbolAddress(*name->symbol); + } + } + + genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint, + memory_order); + genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint, + memory_order); + firOpBuilder.create( + currentLocation, from_address, to_address, hint ? hint : nullptr, + memory_order + ? mlir::omp::ClauseMemoryOrderKindAttr::get( + firOpBuilder.getContext(), + omp::symbolizeClauseMemoryOrderKind(memory_order).getValue()) + : nullptr); +} + +static void +genOMP(Fortran::lower::AbstractConverter &converter, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) { + std::visit(Fortran::common::visitors{ + [&](const Fortran::parser::OmpAtomicRead &atomicRead) { + genOmpAtomicRead(converter, eval, atomicRead); + }, + [&](const Fortran::parser::OmpAtomicWrite &atomicWrite) { + genOmpAtomicWrite(converter, eval, atomicWrite); + }, + [&](const auto &) { + TODO(converter.getCurrentLocation(), + "Atomic update & capture"); + }, + }, + atomicConstruct.u); +} + void Fortran::lower::genOpenMPConstruct( Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, @@ -406,7 +561,7 @@ genOMP(converter, eval, blockConstruct); }, [&](const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) { - TODO(converter.getCurrentLocation(), "OpenMPAtomicConstruct"); + genOMP(converter, eval, atomicConstruct); }, [&](const Fortran::parser::OpenMPCriticalConstruct &criticalConstruct) { diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -3369,6 +3369,15 @@ mlir::omp::MasterOp>([&](Operation *op) { return typeConverter.isLegal(&op->getRegion(0)); }); + + target.addDynamicallyLegalOp([&](Operation *op) { + return typeConverter.isLegal(op->getOperandTypes()); + }); + + target.addDynamicallyLegalOp([&](Operation *op) { + return typeConverter.isLegal(op->getOperandTypes()); + }); + target.addLegalDialect(); // required NOPs for applying a full conversion diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp --- a/flang/lib/Optimizer/Dialect/FIRType.cpp +++ b/flang/lib/Optimizer/Dialect/FIRType.cpp @@ -12,6 +12,7 @@ #include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/Dialect/FIRDialect.h" +#include "mlir/Dialect/OpenMP/OpenMPDialect.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinDialect.h" #include "mlir/IR/Diagnostics.h" @@ -898,10 +899,42 @@ // FIROpsDialect //===----------------------------------------------------------------------===// +namespace { +/// Model for FIR pointer like types that already provide a `getElementType` +/// method +template +struct PointerLikeModel + : public mlir::omp::PointerLikeType::ExternalModel, T> { + mlir::Type getElementType(mlir::Type pointer) const { + return pointer.cast().getElementType(); + } +}; + +template +struct AlternativePointerLikeModel + : public mlir::omp::PointerLikeType::ExternalModel< + AlternativePointerLikeModel, T> { + mlir::Type getElementType(mlir::Type pointer) const { + return pointer.cast().getEleTy(); + } +}; + +} // end namespace void FIROpsDialect::registerTypes() { addTypes(); + fir::ReferenceType::attachInterface>( + *getContext()); + + fir::PointerType::attachInterface>( + *getContext()); + + fir::HeapType::attachInterface>( + *getContext()); + + fir::LLVMPointerType::attachInterface< + AlternativePointerLikeModel>(*getContext()); } diff --git a/flang/test/Lower/OpenMP/atomic01.f90 b/flang/test/Lower/OpenMP/atomic01.f90 new file mode 100644 --- /dev/null +++ b/flang/test/Lower/OpenMP/atomic01.f90 @@ -0,0 +1,74 @@ +! RUN: %flang_fc1 -fopenmp -emit-fir %s -o - | \ +! RUN: FileCheck %s --check-prefix=FIRDialect +! RUN: %flang_fc1 -fopenmp -emit-fir %s -o - | fir-opt --fir-to-llvm-ir | \ +! RUN: FileCheck %s --check-prefix=LLVMDialect + +! This test checks the lowering of atomic read + +!FIRDialect: func @_QQmain() { +!FIRDialect: %[[VAR_A:.*]] = fir.address_of(@_QFEa) : !fir.ref> +!FIRDialect: %[[VAR_B:.*]] = fir.address_of(@_QFEb) : !fir.ref> +!FIRDialect: %[[VAR_C:.*]] = fir.alloca !fir.logical<4> {bindc_name = "c", uniq_name = "_QFEc"} +!FIRDialect: %[[VAR_D:.*]] = fir.alloca !fir.logical<4> {bindc_name = "d", uniq_name = "_QFEd"} +!FIRDialect: %[[VAR_E:.*]] = fir.address_of(@_QFEe) : !fir.ref> +!FIRDialect: %[[VAR_F:.*]] = fir.address_of(@_QFEf) : !fir.ref> +!FIRDialect: %[[VAR_G:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"} +!FIRDialect: %[[VAR_H:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"} +!FIRDialect: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} +!FIRDialect: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} +!FIRDialect: omp.atomic.read %[[VAR_X]] = %[[VAR_Y]] memory_order(acquire) hint(uncontended) : !fir.ref +!FIRDialect: omp.atomic.read %[[VAR_A]] = %[[VAR_B]] memory_order(relaxed) : !fir.ref> +!FIRDialect: omp.atomic.read %[[VAR_C]] = %[[VAR_D]] memory_order(seq_cst) hint(contended) : !fir.ref> +!FIRDialect: omp.atomic.read %[[VAR_E]] = %[[VAR_F]] hint(speculative) : !fir.ref> +!FIRDialect: omp.atomic.read %[[VAR_G]] = %[[VAR_H]] hint(nonspeculative) : !fir.ref +!FIRDialect: omp.atomic.read %[[VAR_G]] = %[[VAR_H]] : !fir.ref +!FIRDialect: return +!FIRDialect: } + +!LLVMDialect: llvm.func @_QQmain() { +!LLVMDialect: %[[LLVM_VAR_A:.*]] = llvm.mlir.addressof @_QFEa : !llvm.ptr> +!LLVMDialect: %[[LLVM_VAR_B:.*]] = llvm.mlir.addressof @_QFEb : !llvm.ptr> +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_C:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "c", in_type = !fir.logical<4>, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEc"} : (i64) -> !llvm.ptr +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_D:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "d", in_type = !fir.logical<4>, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEd"} : (i64) -> !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_E:.*]] = llvm.mlir.addressof @_QFEe : !llvm.ptr> +!LLVMDialect: %[[LLVM_VAR_F:.*]] = llvm.mlir.addressof @_QFEf : !llvm.ptr> +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_G:.*]] = llvm.alloca {{.*}} x f32 {bindc_name = "g", in_type = f32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEg"} : (i64) -> !llvm.ptr +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_H:.*]] = llvm.alloca {{.*}} x f32 {bindc_name = "h", in_type = f32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEh"} : (i64) -> !llvm.ptr +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_X:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "x", in_type = i32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEx"} : (i64) -> !llvm.ptr +!LLVMDialect: {{.*}} = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_Y:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "y", in_type = i32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEy"} : (i64) -> !llvm.ptr +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_X]] = %[[LLVM_VAR_Y]] memory_order(acquire) hint(uncontended) : !llvm.ptr +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_A]] = %[[LLVM_VAR_B]] memory_order(relaxed) : !llvm.ptr> +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_C]] = %[[LLVM_VAR_D]] memory_order(seq_cst) hint(contended) : !llvm.ptr +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_E]] = %[[LLVM_VAR_F]] hint(speculative) : !llvm.ptr> +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_G]] = %[[LLVM_VAR_H]] hint(nonspeculative) : !llvm.ptr +!LLVMDialect: omp.atomic.read %[[LLVM_VAR_G]] = %[[LLVM_VAR_H]] : !llvm.ptr +!LLVMDialect: llvm.return +!LLVMDialect: } + +program OmpAtomic + + use omp_lib + integer :: x, y + character :: a, b + logical :: c, d + character(8) :: e, f + real g, h + !$omp atomic acquire read hint(omp_sync_hint_uncontended) + x = y + !$omp atomic relaxed read hint(omp_sync_hint_none) + a = b + !$omp atomic read seq_cst hint(omp_sync_hint_contended) + c = d + !$omp atomic read hint(omp_sync_hint_speculative) + e = f + !$omp atomic read hint(omp_sync_hint_nonspeculative) + g = h + !$omp atomic read + g = h +end program OmpAtomic diff --git a/flang/test/Lower/OpenMP/atomic02.f90 b/flang/test/Lower/OpenMP/atomic02.f90 new file mode 100644 --- /dev/null +++ b/flang/test/Lower/OpenMP/atomic02.f90 @@ -0,0 +1,64 @@ +! RUN: %flang_fc1 -fopenmp -emit-fir %s -o - | \ +! RUN: FileCheck %s --check-prefix=FIRDialect +! RUN: %flang_fc1 -fopenmp -emit-fir %s -o - | fir-opt --fir-to-llvm-ir | \ +! RUN: FileCheck %s --check-prefix=LLVMDialect + +! This test checks the lowering of atomic write + +!FIRDialect: func @_QQmain() { +!FIRDialect: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} +!FIRDialect: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} +!FIRDialect: %[[VAR_Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFEz"} +!FIRDialect: %[[CONST_44:.*]] = arith.constant 44 : i32 +!FIRDialect: omp.atomic.write %[[VAR_X]] = %[[CONST_44]] hint(uncontended) memory_order(seq_cst) : !fir.ref, i32 +!FIRDialect: %[[CONST_7:.*]] = arith.constant 7 : i32 +!FIRDialect: {{.*}} = fir.load %[[VAR_Y]] : !fir.ref +!FIRDialect: %[[VAR_7y:.*]] = arith.muli %c7_i32, {{.*}} : i32 +!FIRDialect: omp.atomic.write %[[VAR_X]] = %[[VAR_7y]] memory_order(relaxed) : !fir.ref, i32 +!FIRDialect: %[[CONST_10:.*]] = arith.constant 10 : i32 +!FIRDialect: {{.*}} = fir.load %[[VAR_X]] : !fir.ref +!FIRDialect: {{.*}} = arith.muli %[[CONST_10]], {{.*}} : i32 +!FIRDialect: {{.*}} = fir.load %[[VAR_Z]] : !fir.ref +!FIRDialect: %[[CONST_2:.*]] = arith.constant 2 : i32 +!FIRDialect: {{.*}} = arith.divsi {{.*}}, %[[CONST_2]] : i32 +!FIRDialect: {{.*}} = arith.addi {{.*}}, {{.*}} : i32 +!FIRDialect: omp.atomic.write %[[VAR_Y]] = {{.*}} hint(speculative) memory_order(release) : !fir.ref, i32 +!FIRDialect: return +!FIRDialect: } + +!LLVMDialect: llvm.func @_QQmain() { +!LLVMDialect: %[[LLVM_VAR_1:.*]] = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_X:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "x", in_type = i32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEx"} : (i64) -> !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_1_SECOND:.*]] = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_Y:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "y", in_type = i32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEy"} : (i64) -> !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_1_THIRD:.*]] = llvm.mlir.constant(1 : i64) : i64 +!LLVMDialect: %[[LLVM_VAR_Z:.*]] = llvm.alloca {{.*}} x i32 {bindc_name = "z", in_type = i32, operand_segment_sizes = dense<0> : vector<2xi32>, uniq_name = "_QFEz"} : (i64) -> !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_44:.*]] = llvm.mlir.constant(44 : i32) : i32 +!LLVMDialect: omp.atomic.write %[[LLVM_VAR_X]] = %[[LLVM_VAR_44]] hint(uncontended) memory_order(seq_cst) : !llvm.ptr, i32 +!LLVMDialect: %[[LLVM_VAR_7:.*]] = llvm.mlir.constant(7 : i32) : i32 +!LLVMDialect: {{.*}} = llvm.load %[[LLVM_VAR_Y]] : !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_MUL_RESULT:.*]] = llvm.mul %[[LLVM_VAR_7]], {{.*}} : i32 +!LLVMDialect: omp.atomic.write %[[LLVM_VAR_X]] = %[[LLVM_VAR_MUL_RESULT]] memory_order(relaxed) : !llvm.ptr, i32 +!LLVMDialect: %[[LLVM_VAR_10:.*]] = llvm.mlir.constant(10 : i32) : i32 +!LLVMDialect: {{.*}} = llvm.load %[[LLVM_VAR_X]] : !llvm.ptr +!LLVMDialect: {{.*}} = llvm.mul %[[LLVM_VAR_10]], {{.*}} : i32 +!LLVMDialect: {{.*}} = llvm.load %[[LLVM_VAR_Z]] : !llvm.ptr +!LLVMDialect: %[[LLVM_VAR_2:.*]] = llvm.mlir.constant(2 : i32) : i32 +!LLVMDialect: {{.*}} = llvm.sdiv {{.*}}, %[[LLVM_VAR_2]] : i32 +!LLVMDialect: {{.*}} = llvm.add {{.*}} : i32 +!LLVMDialect: omp.atomic.write %[[LLVM_VAR_Y]] = {{.*}} hint(speculative) memory_order(release) : !llvm.ptr, i32 +!LLVMDialect: llvm.return +!LLVMDialect: } + +program OmpAtomicWrite + use omp_lib + integer :: x, y, z + !$omp atomic seq_cst write hint(omp_sync_hint_uncontended) + x = 8*4 + 12 + + !$omp atomic write relaxed + x = 7 * y + + !$omp atomic write release hint(omp_sync_hint_speculative) + y = 10*x + z/2 +end program OmpAtomicWrite diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp --- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp +++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp @@ -45,13 +45,43 @@ return success(); } }; + +struct AtomicReadOpConversion + : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(omp::AtomicReadOp curOp, + typename omp::AtomicReadOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + rewriter.replaceOpWithNewOp( + curOp, TypeRange(), adaptor.getOperands(), curOp->getAttrs()); + return success(); + } +}; + +struct AtomicWriteOpConversion + : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(omp::AtomicWriteOp curOp, + typename omp::AtomicWriteOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + rewriter.replaceOpWithNewOp( + curOp, TypeRange(), adaptor.getOperands(), curOp->getAttrs()); + return success(); + } +}; + } // namespace void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns) { patterns.add, RegionOpConversion, - RegionOpConversion>(converter); + RegionOpConversion, AtomicReadOpConversion, + AtomicWriteOpConversion>(converter); } namespace {