diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h --- a/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -1487,6 +1487,11 @@ virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C); + /// Emit code for doacross ordered directive with 'doacross' clause. + /// \param C 'doacross' clause with 'sink|source' dependence type. + virtual void emitDoacrossOrdered(CodeGenFunction &CGF, + const OMPDoacrossClause *C); + /// Translates the native parameter of outlined function if this is required /// for target. /// \param FD Field decl from captured record for the parameter. @@ -2240,6 +2245,11 @@ void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override; + /// Emit code for doacross ordered directive with 'doacross' clause. + /// \param C 'doacross' clause with 'sink|source' dependence type. + void emitDoacrossOrdered(CodeGenFunction &CGF, + const OMPDoacrossClause *C) override; + /// Translates the native parameter of outlined function if this is required /// for target. /// \param FD Field decl from captured record for the parameter. @@ -2262,6 +2272,32 @@ }; } // namespace CodeGen +// Utility for openmp doacross clause kind +namespace { +template class OMPDoacrossKind { +public: + bool isSink(const T *) { return false; } + bool isSource(const T *) { return false; } +}; +template <> class OMPDoacrossKind { +public: + bool isSink(const OMPDependClause *C) { + return C->getDependencyKind() == OMPC_DEPEND_sink; + } + bool isSource(const OMPDependClause *C) { + return C->getDependencyKind() == OMPC_DEPEND_source; + } +}; +template <> class OMPDoacrossKind { +public: + bool isSource(const OMPDoacrossClause *C) { + return (C->getDependenceType() == OMPC_DOACROSS_source); + } + bool isSink(const OMPDoacrossClause *C) { + return (C->getDependenceType() == OMPC_DOACROSS_sink); + } +}; +} // namespace } // namespace clang #endif diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -11381,8 +11381,10 @@ llvm::ArrayRef(FiniArgs)); } -void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, - const OMPDependClause *C) { +template +static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, + const T *C, llvm::Value *ULoc, + llvm::Value *ThreadID) { QualType Int64Ty = CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); llvm::APInt Size(/*numBits=*/32, C->getNumLoops()); @@ -11399,21 +11401,35 @@ /*Volatile=*/false, Int64Ty); } llvm::Value *Args[] = { - emitUpdateLocation(CGF, C->getBeginLoc()), - getThreadID(CGF, C->getBeginLoc()), - CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; + ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; llvm::FunctionCallee RTLFn; - if (C->getDependencyKind() == OMPC_DEPEND_source) { + llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); + OMPDoacrossKind ODK; + if (ODK.isSource(C)) { RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), OMPRTL___kmpc_doacross_post); } else { - assert(C->getDependencyKind() == OMPC_DEPEND_sink); + assert(ODK.isSink(C) && "Expect sink modifier."); RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), OMPRTL___kmpc_doacross_wait); } CGF.EmitRuntimeCall(RTLFn, Args); } +void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, + const OMPDependClause *C) { + return EmitDoacrossOrdered( + CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), + getThreadID(CGF, C->getBeginLoc())); +} + +void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, + const OMPDoacrossClause *C) { + return EmitDoacrossOrdered( + CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), + getThreadID(CGF, C->getBeginLoc())); +} + void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef Args) const { @@ -12400,6 +12416,11 @@ llvm_unreachable("Not supported in SIMD-only mode"); } +void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, + const OMPDoacrossClause *C) { + llvm_unreachable("Not supported in SIMD-only mode"); +} + const VarDecl * CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const { diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -5839,37 +5839,46 @@ return Fn; } +template +static void emitRestoreIP(CodeGenFunction &CGF, const T *C, + llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, + llvm::OpenMPIRBuilder &OMPBuilder) { + + unsigned NumLoops = C->getNumLoops(); + QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth( + /*DestWidth=*/64, /*Signed=*/1); + llvm::SmallVector StoreValues; + for (unsigned I = 0; I < NumLoops; I++) { + const Expr *CounterVal = C->getLoopData(I); + assert(CounterVal); + llvm::Value *StoreValue = CGF.EmitScalarConversion( + CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, + CounterVal->getExprLoc()); + StoreValues.emplace_back(StoreValue); + } + OMPDoacrossKind ODK; + bool IsDependSource = ODK.isSource(C); + CGF.Builder.restoreIP( + OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops, + StoreValues, ".cnt.addr", IsDependSource)); +} + void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { if (CGM.getLangOpts().OpenMPIRBuilder) { llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; - if (S.hasClausesOfKind()) { + if (S.hasClausesOfKind() || + S.hasClausesOfKind()) { // The ordered directive with depend clause. - assert(!S.hasAssociatedStmt() && - "No associated statement must be in ordered depend construct."); + assert(!S.hasAssociatedStmt() && "No associated statement must be in " + "ordered depend|doacross construct."); InsertPointTy AllocaIP(AllocaInsertPt->getParent(), AllocaInsertPt->getIterator()); - for (const auto *DC : S.getClausesOfKind()) { - unsigned NumLoops = DC->getNumLoops(); - QualType Int64Ty = CGM.getContext().getIntTypeForBitwidth( - /*DestWidth=*/64, /*Signed=*/1); - llvm::SmallVector StoreValues; - for (unsigned I = 0; I < NumLoops; I++) { - const Expr *CounterVal = DC->getLoopData(I); - assert(CounterVal); - llvm::Value *StoreValue = EmitScalarConversion( - EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, - CounterVal->getExprLoc()); - StoreValues.emplace_back(StoreValue); - } - bool IsDependSource = false; - if (DC->getDependencyKind() == OMPC_DEPEND_source) - IsDependSource = true; - Builder.restoreIP(OMPBuilder.createOrderedDepend( - Builder, AllocaIP, NumLoops, StoreValues, ".cnt.addr", - IsDependSource)); - } + for (const auto *DC : S.getClausesOfKind()) + emitRestoreIP(*this, DC, AllocaIP, OMPBuilder); + for (const auto *DC : S.getClausesOfKind()) + emitRestoreIP(*this, DC, AllocaIP, OMPBuilder); } else { // The ordered directive with threads or simd clause, or without clause. // Without clause, it behaves as if the threads clause is specified. @@ -5916,6 +5925,13 @@ CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); return; } + if (S.hasClausesOfKind()) { + assert(!S.hasAssociatedStmt() && + "No associated statement must be in ordered doacross construct."); + for (const auto *DC : S.getClausesOfKind()) + CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); + return; + } const auto *C = S.getSingleClause(); auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, PrePostActionTy &Action) { diff --git a/clang/test/OpenMP/ordered_doacross_codegen.c b/clang/test/OpenMP/ordered_doacross_codegen.c --- a/clang/test/OpenMP/ordered_doacross_codegen.c +++ b/clang/test/OpenMP/ordered_doacross_codegen.c @@ -2,13 +2,21 @@ // RUN: %clang_cc1 -fopenmp -triple x86_64-unknown-unknown -emit-pch -o %t %s // RUN: %clang_cc1 -fopenmp -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NORMAL +// RUN: %clang_cc1 -verify -fopenmp -triple x86_64-unknown-unknown -emit-llvm %s -o - -fopenmp-version=52 | FileCheck %s --check-prefixes=CHECK,CHECK-NORMAL // RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IRBUILDER +// RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -fopenmp-version=52 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IRBUILDER // RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -emit-pch -o %t %s // RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IRBUILDER +// RUN: %clang_cc1 -fopenmp -fopenmp-version=52 -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -emit-pch -o %t %s +// RUN: %clang_cc1 -fopenmp -fopenmp-version=52 -fopenmp-enable-irbuilder -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IRBUILDER + // RUN: %clang_cc1 -verify -fopenmp-simd -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s +// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=52 -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s // RUN: %clang_cc1 -fopenmp-simd -triple x86_64-unknown-unknown -emit-pch -o %t %s // RUN: %clang_cc1 -fopenmp-simd -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s +// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=52 -triple x86_64-unknown-unknown -emit-pch -o %t %s +// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=52 -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s // SIMD-ONLY0-NOT: {{__kmpc|__tgt}} // expected-no-diagnostics @@ -51,7 +59,11 @@ // CHECK-NORMAL-NEXT: call void @__kmpc_doacross_post(ptr [[IDENT]], i32 [[GTID]], ptr [[TMP]]) // CHECK-IRBUILDER-NEXT: [[GTID1:%.+]] = call i32 @__kmpc_global_thread_num(ptr [[IDENT:@.+]]) // CHECK-IRBUILDER-NEXT: call void @__kmpc_doacross_post(ptr [[IDENT]], i32 [[GTID1]], ptr [[TMP]]) +#if _OPENMP >= 202111 +#pragma omp ordered doacross(source:) +#else #pragma omp ordered depend(source) +#endif c[i] = c[i] + 1; foo(); // CHECK: call void @foo() @@ -66,7 +78,11 @@ // CHECK-NORMAL-NEXT: call void @__kmpc_doacross_wait(ptr [[IDENT]], i32 [[GTID]], ptr [[TMP]]) // CHECK-IRBUILDER-NEXT: [[GTID2:%.+]] = call i32 @__kmpc_global_thread_num(ptr [[IDENT:@.+]]) // CHECK-IRBUILDER-NEXT: call void @__kmpc_doacross_wait(ptr [[IDENT]], i32 [[GTID2]], ptr [[TMP]]) +#if _OPENMP >= 202111 +#pragma omp ordered doacross(sink : i - 2) +#else #pragma omp ordered depend(sink : i - 2) +#endif d[i] = a[i - 2]; } // CHECK: call void @__kmpc_for_static_fini(