[OPENMP] Refactoring of codegen for OpenMP directives.

[ABataev]

Summary

Refactored API of OpenMPRuntime for compatibility with combined directives.

[rjmccall]

This looks great, thanks.

cfe/trunk/lib/CodeGen/CGOpenMPRuntime.h

 class OMPExecutableDirective;
 class VarDecl;
 
 namespace CodeGen {
 
 class CodeGenFunction;
 class CodeGenModule;
 
+typedef llvm::function_ref<void(CodeGenFunction &)> RegionCodeGenTy;
+
 class CGOpenMPRuntime {
+private:
   enum OpenMPRTLFunction {
     /// \brief Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
     /// kmpc_micro microtask, ...);
     OMPRTL__kmpc_fork_call,
     /// \brief Call to void *__kmpc_threadprivate_cached(ident_t *loc,
     /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
     OMPRTL__kmpc_threadprivate_cached,
     /// \brief Call to void __kmpc_threadprivate_register( ident_t *,
private:
   ///
   llvm::Value *getCriticalRegionLock(StringRef CriticalName);
 
 public:
   explicit CGOpenMPRuntime(CodeGenModule &CGM);
   virtual ~CGOpenMPRuntime() {}
   virtual void clear();
 
-  /// \brief Emits outlined function for the specified OpenMP directive \a D.
-  /// This outlined function has type void(*)(kmp_int32 *ThreadID, kmp_int32
-  /// BoundID, struct context_vars*).
+  /// \brief Emits outlined function for the specified OpenMP parallel directive
+  /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
+  /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
-  ///
-  virtual llvm::Value *emitOutlinedFunction(const OMPExecutableDirective &D,
-                                            const VarDecl *ThreadIDVar);
+  /// \param CodeGen Code generation sequence for the \a D directive.
+  virtual llvm::Value *
+  emitParallelOutlinedFunction(const OMPExecutableDirective &D,
+                               const VarDecl *ThreadIDVar,
+                               const RegionCodeGenTy &CodeGen);
 
   /// \brief Emits outlined function for the OpenMP task directive \a D. This
   /// outlined function has type void(*)(kmp_int32 ThreadID, kmp_int32
   /// PartID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
-  /// \param PartIDVar If not nullptr - variable used for part id in tasks.
+  /// \param CodeGen Code generation sequence for the \a D directive.
   ///
   virtual llvm::Value *emitTaskOutlinedFunction(const OMPExecutableDirective &D,
                                                 const VarDecl *ThreadIDVar,
-                                                const VarDecl *PartIDVar);
+                                                const RegionCodeGenTy &CodeGen);
 
   /// \brief Cleans up references to the objects in finished function.
   ///
   void functionFinished(CodeGenFunction &CGF);
 
   /// \brief Emits code for parallel call of the \a OutlinedFn with variables
   /// captured in a record which address is stored in \a CapturedStruct.
   /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
virtual void emitSerialCall(CodeGenFunction &CGF, SourceLocation Loc,
                               llvm::Value *OutlinedFn,
                               llvm::Value *CapturedStruct);
 
   /// \brief Emits a critical region.
   /// \param CriticalName Name of the critical region.
   /// \param CriticalOpGen Generator for the statement associated with the given
   /// critical region.
   virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
-                                  const std::function<void()> &CriticalOpGen,
+                                  const RegionCodeGenTy &CriticalOpGen,
                                   SourceLocation Loc);
 
   /// \brief Emits a master region.
   /// \param MasterOpGen Generator for the statement associated with the given
   /// master region.
   virtual void emitMasterRegion(CodeGenFunction &CGF,
-                                const std::function<void()> &MasterOpGen,
+                                const RegionCodeGenTy &MasterOpGen,
                                 SourceLocation Loc);
 
   /// \brief Emits code for a taskyield directive.
   virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
 
   /// \brief Emits a single region.
   /// \param SingleOpGen Generator for the statement associated with the given
   /// single region.
   virtual void emitSingleRegion(CodeGenFunction &CGF,
-                                const std::function<void()> &SingleOpGen,
+                                const RegionCodeGenTy &SingleOpGen,
                                 SourceLocation Loc,
                                 ArrayRef<const Expr *> CopyprivateVars,
                                 ArrayRef<const Expr *> SrcExprs,
                                 ArrayRef<const Expr *> DstExprs,
                                 ArrayRef<const Expr *> AssignmentOps);
 
   /// \brief Emit an implicit/explicit barrier for OpenMP threads.
   /// \param Kind Directive for which this implicit barrier call must be
public:
   /// /*part_id*/, captured_struct */*__context*/);
   /// \param SharedsTy A type which contains references the shared variables.
   /// \param Shareds Context with the list of shared variables from the \a
   /// TaskFunction.
   virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, bool Tied,
                             llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
                             llvm::Value *TaskFunction, QualType SharedsTy,
                             llvm::Value *Shareds);
+  /// \brief Emit code for the directive that does not require outlining.
+  ///
+  /// \param CodeGen Code generation sequence for the \a D directive.
+  virtual void emitInlinedDirective(CodeGenFunction &CGF,
+                                    const RegionCodeGenTy &CodeGen);
 };
 
-/// \brief RAII for emitting code of CapturedStmt without function outlining.
-class InlinedOpenMPRegionRAII {
-  CodeGenFunction &CGF;
-
-public:
-  InlinedOpenMPRegionRAII(CodeGenFunction &CGF,
-                          const OMPExecutableDirective &D);
-  ~InlinedOpenMPRegionRAII();
-};
 } // namespace CodeGen
 } // namespace clang
 
 #endif

cfe/trunk/lib/CodeGen/CGOpenMPRuntime.cpp

 
 using namespace clang;
 using namespace CodeGen;
 
 namespace {
 /// \brief Base class for handling code generation inside OpenMP regions.
 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
 public:
-  CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS)
-      : CGCapturedStmtInfo(CS, CR_OpenMP), Directive(D) {}
+  /// \brief Kinds of OpenMP regions used in codegen.
+  enum CGOpenMPRegionKind {
+    /// \brief Region with outlined function for standalone 'parallel'
+    /// directive.
+    ParallelOutlinedRegion,
+    /// \brief Region with outlined function for standalone 'task' directive.
+    TaskOutlinedRegion,
+    /// \brief Region for constructs that do not require function outlining,
+    /// like 'for', 'sections', 'atomic' etc. directives.
+    InlinedRegion,
+  };
 
-  CGOpenMPRegionInfo(const OMPExecutableDirective &D)
-      : CGCapturedStmtInfo(CR_OpenMP), Directive(D) {}
+  CGOpenMPRegionInfo(const CapturedStmt &CS,
+                     const CGOpenMPRegionKind RegionKind,
+                     const RegionCodeGenTy &CodeGen)
+      : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
+        CodeGen(CodeGen) {}
+
+  CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
+                     const RegionCodeGenTy &CodeGen)
+      : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind),
+        CodeGen(CodeGen) {}
 
   /// \brief Get a variable or parameter for storing global thread id
   /// inside OpenMP construct.
   virtual const VarDecl *getThreadIDVariable() const = 0;
 
+  /// \brief Emit the captured statement body.
+  virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
+
   /// \brief Get an LValue for the current ThreadID variable.
   /// \return LValue for thread id variable. This LValue always has type int32*.
   virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
 
-    /// \brief Emit the captured statement body.
-  virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
+  CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
 
   static bool classof(const CGCapturedStmtInfo *Info) {
     return Info->getKind() == CR_OpenMP;
   }
+
 protected:
-  /// \brief OpenMP executable directive associated with the region.
-  const OMPExecutableDirective &Directive;
+  CGOpenMPRegionKind RegionKind;
+  const RegionCodeGenTy &CodeGen;
 };
 
 /// \brief API for captured statement code generation in OpenMP constructs.
 class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo {
 public:
-  CGOpenMPOutlinedRegionInfo(const OMPExecutableDirective &D,
-                             const CapturedStmt &CS, const VarDecl *ThreadIDVar)
-      : CGOpenMPRegionInfo(D, CS), ThreadIDVar(ThreadIDVar) {
+  CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
+                             const RegionCodeGenTy &CodeGen)
+      : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen),
+        ThreadIDVar(ThreadIDVar) {
     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
   }
   /// \brief Get a variable or parameter for storing global thread id
   /// inside OpenMP construct.
   virtual const VarDecl *getThreadIDVariable() const override {
     return ThreadIDVar;
   }
+
   /// \brief Get the name of the capture helper.
   StringRef getHelperName() const override { return ".omp_outlined."; }
 
+  static bool classof(const CGCapturedStmtInfo *Info) {
+    return CGOpenMPRegionInfo::classof(Info) &&
+           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+               ParallelOutlinedRegion;
+  }
+
 private:
   /// \brief A variable or parameter storing global thread id for OpenMP
   /// constructs.
   const VarDecl *ThreadIDVar;
 };
 
 /// \brief API for captured statement code generation in OpenMP constructs.
 class CGOpenMPTaskOutlinedRegionInfo : public CGOpenMPRegionInfo {
 public:
-  CGOpenMPTaskOutlinedRegionInfo(const OMPExecutableDirective &D,
-                                 const CapturedStmt &CS,
+  CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
                                  const VarDecl *ThreadIDVar,
-                                 const VarDecl *PartIDVar)
-      : CGOpenMPRegionInfo(D, CS), ThreadIDVar(ThreadIDVar),
-        PartIDVar(PartIDVar) {
+                                 const RegionCodeGenTy &CodeGen)
+      : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen),
+        ThreadIDVar(ThreadIDVar) {
     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
   }
   /// \brief Get a variable or parameter for storing global thread id
   /// inside OpenMP construct.
   virtual const VarDecl *getThreadIDVariable() const override {
     return ThreadIDVar;
   }
 
   /// \brief Get an LValue for the current ThreadID variable.
   virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
 
-  /// \brief Emit the captured statement body.
-  virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
-
   /// \brief Get the name of the capture helper.
   StringRef getHelperName() const override { return ".omp_outlined."; }
 
+  static bool classof(const CGCapturedStmtInfo *Info) {
+    return CGOpenMPRegionInfo::classof(Info) &&
+           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+               TaskOutlinedRegion;
+  }
+
 private:
   /// \brief A variable or parameter storing global thread id for OpenMP
   /// constructs.
   const VarDecl *ThreadIDVar;
-  /// \brief A variable or parameter storing part id for OpenMP tasking
-  /// constructs.
-  const VarDecl *PartIDVar;
 };
 
 /// \brief API for inlined captured statement code generation in OpenMP
 /// constructs.
 class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
 public:
-  CGOpenMPInlinedRegionInfo(const OMPExecutableDirective &D,
-                            CodeGenFunction::CGCapturedStmtInfo *OldCSI)
-      : CGOpenMPRegionInfo(D), OldCSI(OldCSI),
+  CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
+                            const RegionCodeGenTy &CodeGen)
+      : CGOpenMPRegionInfo(InlinedRegion, CodeGen), OldCSI(OldCSI),
         OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
   // \brief Retrieve the value of the context parameter.
   virtual llvm::Value *getContextValue() const override {
     if (OuterRegionInfo)
       return OuterRegionInfo->getContextValue();
     llvm_unreachable("No context value for inlined OpenMP region");
   }
+  virtual void setContextValue(llvm::Value *V) override {
+    if (OuterRegionInfo) {
+      OuterRegionInfo->setContextValue(V);
+      return;
+    }
+    llvm_unreachable("No context value for inlined OpenMP region");
+  }
   /// \brief Lookup the captured field decl for a variable.
   virtual const FieldDecl *lookup(const VarDecl *VD) const override {
     if (OuterRegionInfo)
       return OuterRegionInfo->lookup(VD);
     llvm_unreachable("Trying to reference VarDecl that is neither local nor "
                      "captured in outer OpenMP region");
   }
   virtual FieldDecl *getThisFieldDecl() const override {
     if (OuterRegionInfo)
       return OuterRegionInfo->getThisFieldDecl();
     return nullptr;
   }
   /// \brief Get a variable or parameter for storing global thread id
   /// inside OpenMP construct.
   virtual const VarDecl *getThreadIDVariable() const override {
     if (OuterRegionInfo)
       return OuterRegionInfo->getThreadIDVariable();
     return nullptr;
   }
 
   /// \brief Get the name of the capture helper.
   virtual StringRef getHelperName() const override {
+    if (auto *OuterRegionInfo = getOldCSI())
+      return OuterRegionInfo->getHelperName();
     llvm_unreachable("No helper name for inlined OpenMP construct");
   }
 
   CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
 
+  static bool classof(const CGCapturedStmtInfo *Info) {
+    return CGOpenMPRegionInfo::classof(Info) &&
+           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
+  }
+
 private:
   /// \brief CodeGen info about outer OpenMP region.
   CodeGenFunction::CGCapturedStmtInfo *OldCSI;
   CGOpenMPRegionInfo *OuterRegionInfo;
 };
+
+/// \brief RAII for emitting code of OpenMP constructs.
+class InlinedOpenMPRegionRAII {
+  CodeGenFunction &CGF;
+
+public:
+  /// \brief Constructs region for combined constructs.
+  /// \param CodeGen Code generation sequence for combined directives. Includes
+  /// a list of functions used for code generation of implicitly inlined
+  /// regions.
+  InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen)
+      : CGF(CGF) {
+    // Start emission for the construct.
+    CGF.CapturedStmtInfo =
+        new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen);
+  }
+  ~InlinedOpenMPRegionRAII() {
+    // Restore original CapturedStmtInfo only if we're done with code emission.
+    auto *OldCSI =
+        cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
+    delete CGF.CapturedStmtInfo;
+    CGF.CapturedStmtInfo = OldCSI;
+  }
+};
+
 } // namespace
 
 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
   return CGF.MakeNaturalAlignAddrLValue(
       CGF.Builder.CreateAlignedLoad(
           CGF.GetAddrOfLocalVar(getThreadIDVariable()),
           CGF.PointerAlignInBytes),
       getThreadIDVariable()
           ->getType()
           ->castAs<PointerType>()
           ->getPointeeType());
 }
 
-void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
-  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
-  CGF.EmitOMPPrivateClause(Directive, PrivateScope);
-  CGF.EmitOMPFirstprivateClause(Directive, PrivateScope);
-  if (PrivateScope.Privatize())
-    // Emit implicit barrier to synchronize threads and avoid data races.
-    CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, Directive.getLocStart(),
-                                               OMPD_unknown);
-  CGCapturedStmtInfo::EmitBody(CGF, S);
+void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
+  // 1.2.2 OpenMP Language Terminology
+  // Structured block - An executable statement with a single entry at the
+  // top and a single exit at the bottom.
+  // The point of exit cannot be a branch out of the structured block.
+  // longjmp() and throw() must not violate the entry/exit criteria.
+  CGF.EHStack.pushTerminate();
+  {
+    CodeGenFunction::RunCleanupsScope Scope(CGF);
+    CodeGen(CGF);
+  }
+  CGF.EHStack.popTerminate();
 }
 
 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
     CodeGenFunction &CGF) {
   return CGF.MakeNaturalAlignAddrLValue(
       CGF.GetAddrOfLocalVar(getThreadIDVariable()),
       getThreadIDVariable()->getType());
 }
 
-void CGOpenMPTaskOutlinedRegionInfo::EmitBody(CodeGenFunction &CGF,
-                                              const Stmt *S) {
-  if (PartIDVar) {
-    // TODO: emit code for untied tasks.
-  }
-  CGCapturedStmtInfo::EmitBody(CGF, S);
-}
-
 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
     : CGM(CGM), DefaultOpenMPPSource(nullptr), KmpRoutineEntryPtrTy(nullptr) {
   IdentTy = llvm::StructType::create(
       "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
       CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
       CGM.Int8PtrTy /* psource */, nullptr);
   // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
   llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
                                llvm::PointerType::getUnqual(CGM.Int32Ty)};
   Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
 }
 
 void CGOpenMPRuntime::clear() {
   InternalVars.clear();
 }
 
 llvm::Value *
-CGOpenMPRuntime::emitOutlinedFunction(const OMPExecutableDirective &D,
-                                      const VarDecl *ThreadIDVar) {
+CGOpenMPRuntime::emitParallelOutlinedFunction(const OMPExecutableDirective &D,
+                                              const VarDecl *ThreadIDVar,
+                                              const RegionCodeGenTy &CodeGen) {
   assert(ThreadIDVar->getType()->isPointerType() &&
          "thread id variable must be of type kmp_int32 *");
   const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
   CodeGenFunction CGF(CGM, true);
-  CGOpenMPOutlinedRegionInfo CGInfo(D, *CS, ThreadIDVar);
+  CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
   CGF.CapturedStmtInfo = &CGInfo;
   return CGF.GenerateCapturedStmtFunction(*CS);
 }
 
 llvm::Value *
 CGOpenMPRuntime::emitTaskOutlinedFunction(const OMPExecutableDirective &D,
                                           const VarDecl *ThreadIDVar,
-                                          const VarDecl *PartIDVar) {
+                                          const RegionCodeGenTy &CodeGen) {
   assert(!ThreadIDVar->getType()->isPointerType() &&
          "thread id variable must be of type kmp_int32 for tasks");
   auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
   CodeGenFunction CGF(CGM, true);
-  CGOpenMPTaskOutlinedRegionInfo CGInfo(D, *CS, ThreadIDVar, PartIDVar);
+  CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
   CGF.CapturedStmtInfo = &CGInfo;
   return CGF.GenerateCapturedStmtFunction(*CS);
 }
 
 llvm::Value *
 CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
   llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
   if (!Entry) {
return Elem.second = new llvm::GlobalVariable(
              Elem.first());
 }
 
 llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
   llvm::Twine Name(".gomp_critical_user_", CriticalName);
   return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
 }
 
-void CGOpenMPRuntime::emitCriticalRegion(
-    CodeGenFunction &CGF, StringRef CriticalName,
-    const std::function<void()> &CriticalOpGen, SourceLocation Loc) {
+namespace {
+class CallEndCleanup : public EHScopeStack::Cleanup {
+private:
+  const RegionCodeGenTy CodeGen;
+
+public:
+  CallEndCleanup(const RegionCodeGenTy &CodeGen) : CodeGen(CodeGen) {}
+  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { CodeGen(CGF); }
+};
+} // namespace
+
+void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
+                                         StringRef CriticalName,
+                                         const RegionCodeGenTy &CriticalOpGen,
+                                         SourceLocation Loc) {
   auto RegionLock = getCriticalRegionLock(CriticalName);
   // __kmpc_critical(ident_t *, gtid, Lock);
   // CriticalOpGen();
   // __kmpc_end_critical(ident_t *, gtid, Lock);
   // Prepare arguments and build a call to __kmpc_critical
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
                          RegionLock};
+  {
+    CodeGenFunction::RunCleanupsScope Scope(CGF);
     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args);
-  CriticalOpGen();
+    emitInlinedDirective(CGF, CriticalOpGen);
     // Build a call to __kmpc_end_critical
-  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
+    CGF.EHStack.pushCleanup<CallEndCleanup>(
+        NormalAndEHCleanup, [this, Args](CodeGenFunction &CGF) {
+          CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_critical),
+                              Args);
+        });
+  }
 }
 
 static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
-                       const std::function<void()> &BodyOpGen) {
+                       const RegionCodeGenTy &BodyOpGen) {
   llvm::Value *CallBool = CGF.EmitScalarConversion(
       IfCond,
       CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
       CGF.getContext().BoolTy);
 
   auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
   auto *ContBlock = CGF.createBasicBlock("omp_if.end");
   // Generate the branch (If-stmt)
   CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
   CGF.EmitBlock(ThenBlock);
-  BodyOpGen();
+  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, BodyOpGen);
   // Emit the rest of bblocks/branches
   CGF.EmitBranch(ContBlock);
   CGF.EmitBlock(ContBlock, true);
 }
 
 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
-                                       const std::function<void()> &MasterOpGen,
+                                       const RegionCodeGenTy &MasterOpGen,
                                        SourceLocation Loc) {
   // if(__kmpc_master(ident_t *, gtid)) {
   //   MasterOpGen();
   //   __kmpc_end_master(ident_t *, gtid);
   // }
   // Prepare arguments and build a call to __kmpc_master
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
   auto *IsMaster =
       CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args);
-  emitIfStmt(CGF, IsMaster, [&]() -> void {
-    MasterOpGen();
+  emitIfStmt(CGF, IsMaster, [&](CodeGenFunction &CGF) -> void {
+    CodeGenFunction::RunCleanupsScope Scope(CGF);
+    MasterOpGen(CGF);
     // Build a call to __kmpc_end_master.
     // OpenMP [1.2.2 OpenMP Language Terminology]
     // For C/C++, an executable statement, possibly compound, with a single
-    // entry at the top and a single exit at the bottom, or an OpenMP construct.
+    // entry at the top and a single exit at the bottom, or an OpenMP
+    // construct.
     // * Access to the structured block must not be the result of a branch.
     // * The point of exit cannot be a branch out of the structured block.
     // * The point of entry must not be a call to setjmp().
     // * longjmp() and throw() must not violate the entry/exit criteria.
     // * An expression statement, iteration statement, selection statement, or
     // try block is considered to be a structured block if the corresponding
     // compound statement obtained by enclosing it in { and } would be a
     // structured block.
     // It is analyzed in Sema, so we can just call __kmpc_end_master() on
     // fallthrough rather than pushing a normal cleanup for it.
-    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_master), Args);
+    // Build a call to __kmpc_end_critical
+    CGF.EHStack.pushCleanup<CallEndCleanup>(
+        NormalAndEHCleanup, [this, Args](CodeGenFunction &CGF) {
+          CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_master),
+                              Args);
+        });
   });
 }
 
 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
                                         SourceLocation Loc) {
   // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
   llvm::Value *Args[] = {
       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
for (auto *E : AssignmentOps) {
     CGF.EmitIgnoredExpr(E);
   }
   Scope.ForceCleanup();
   CGF.FinishFunction();
   return Fn;
 }
 
 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
-                                       const std::function<void()> &SingleOpGen,
+                                       const RegionCodeGenTy &SingleOpGen,
                                        SourceLocation Loc,
                                        ArrayRef<const Expr *> CopyprivateVars,
                                        ArrayRef<const Expr *> SrcExprs,
                                        ArrayRef<const Expr *> DstExprs,
                                        ArrayRef<const Expr *> AssignmentOps) {
   assert(CopyprivateVars.size() == SrcExprs.size() &&
          CopyprivateVars.size() == DstExprs.size() &&
          CopyprivateVars.size() == AssignmentOps.size());
if (!CopyprivateVars.empty()) {
     auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
     DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
     CGF.InitTempAlloca(DidIt, CGF.Builder.getInt32(0));
   }
   // Prepare arguments and build a call to __kmpc_single
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
   auto *IsSingle =
       CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args);
-  emitIfStmt(CGF, IsSingle, [&]() -> void {
-    SingleOpGen();
+  emitIfStmt(CGF, IsSingle, [&](CodeGenFunction &CGF) -> void {
+    CodeGenFunction::RunCleanupsScope Scope(CGF);
+    SingleOpGen(CGF);
     if (DidIt) {
       // did_it = 1;
       CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt,
                                      DidIt->getAlignment());
     }
     // Build a call to __kmpc_end_single.
     // OpenMP [1.2.2 OpenMP Language Terminology]
     // For C/C++, an executable statement, possibly compound, with a single
     // entry at the top and a single exit at the bottom, or an OpenMP construct.
     // * Access to the structured block must not be the result of a branch.
     // * The point of exit cannot be a branch out of the structured block.
     // * The point of entry must not be a call to setjmp().
     // * longjmp() and throw() must not violate the entry/exit criteria.
     // * An expression statement, iteration statement, selection statement, or
     // try block is considered to be a structured block if the corresponding
     // compound statement obtained by enclosing it in { and } would be a
     // structured block.
     // It is analyzed in Sema, so we can just call __kmpc_end_single() on
     // fallthrough rather than pushing a normal cleanup for it.
-    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_single), Args);
+    CGF.EHStack.pushCleanup<CallEndCleanup>(
+        NormalAndEHCleanup, [this, Args](CodeGenFunction &CGF) {
+          CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_end_single),
+                              Args);
+        });
   });
   // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
   // <copy_func>, did_it);
   if (DidIt) {
     llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
     auto CopyprivateArrayTy =
         C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
                                /*IndexTypeQuals=*/0);
void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc,
   }
 }
 
 void CGOpenMPRuntime::emitForFinish(CodeGenFunction &CGF, SourceLocation Loc,
                                     OpenMPScheduleClauseKind ScheduleKind) {
   assert((ScheduleKind == OMPC_SCHEDULE_static ||
           ScheduleKind == OMPC_SCHEDULE_unknown) &&
          "Non-static schedule kinds are not yet implemented");
+  (void)ScheduleKind;
   // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
                          getThreadID(CGF, Loc)};
   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
                       Args);
 }
 
 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
void CGOpenMPRuntime::emitTaskCall(
   // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
   // *new_task);
   llvm::Value *TaskArgs[] = {emitUpdateLocation(CGF, Loc),
                              getThreadID(CGF, Loc), NewTask};
   // TODO: add check for untied tasks.
   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
 }
 
-InlinedOpenMPRegionRAII::InlinedOpenMPRegionRAII(
-    CodeGenFunction &CGF, const OMPExecutableDirective &D)
-    : CGF(CGF) {
-  CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(D, CGF.CapturedStmtInfo);
-  // 1.2.2 OpenMP Language Terminology
-  // Structured block - An executable statement with a single entry at the
-  // top and a single exit at the bottom.
-  // The point of exit cannot be a branch out of the structured block.
-  // longjmp() and throw() must not violate the entry/exit criteria.
-  CGF.EHStack.pushTerminate();
-}
-
-InlinedOpenMPRegionRAII::~InlinedOpenMPRegionRAII() {
-  CGF.EHStack.popTerminate();
-  auto *OldCSI =
-      cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
-  delete CGF.CapturedStmtInfo;
-  CGF.CapturedStmtInfo = OldCSI;
+void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
+                                           const RegionCodeGenTy &CodeGen) {
+  InlinedOpenMPRegionRAII Region(CGF, CodeGen);
+  CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
 }
 

cfe/trunk/lib/CodeGen/CGStmtOpenMP.cpp

 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtOpenMP.h"
 using namespace clang;
 using namespace CodeGen;
 
 //===----------------------------------------------------------------------===//
 //                              OpenMP Directive Emission
 //===----------------------------------------------------------------------===//
-namespace {
-/// \brief RAII for inlined OpenMP regions (like 'omp for', 'omp simd', 'omp
-/// critical' etc.). Helps to generate proper debug info and provides correct
-/// code generation for such constructs.
-class InlinedOpenMPRegionScopeRAII {
-  InlinedOpenMPRegionRAII Region;
-  CodeGenFunction::LexicalScope DirectiveScope;
-
-public:
-  InlinedOpenMPRegionScopeRAII(CodeGenFunction &CGF,
-                               const OMPExecutableDirective &D)
-      : Region(CGF, D), DirectiveScope(CGF, D.getSourceRange()) {}
-};
-} // namespace
-
 /// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen
 /// function. Here is the logic:
 /// if (Cond) {
 ///   CodeGen(true);
 /// } else {
 ///   CodeGen(false);
 /// }
 static void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
for (auto IInit : C->private_copies()) {
       // Silence the warning about unused variable.
       (void)IsRegistered;
       ++IRef;
     }
   }
 }
 
 /// \brief Emits code for OpenMP parallel directive in the parallel region.
-static void EmitOMPParallelCall(CodeGenFunction &CGF,
-                                const OMPParallelDirective &S,
+static void emitOMPParallelCall(CodeGenFunction &CGF,
+                                const OMPExecutableDirective &S,
                                 llvm::Value *OutlinedFn,
                                 llvm::Value *CapturedStruct) {
   if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) {
     CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
     auto NumThreadsClause = cast<OMPNumThreadsClause>(C);
     auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
                                          /*IgnoreResultAssign*/ true);
     CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
         CGF, NumThreads, NumThreadsClause->getLocStart());
   }
   CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
                                               CapturedStruct);
 }
 
-void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
+static void emitCommonOMPParallelDirective(CodeGenFunction &CGF,
+                                           const OMPExecutableDirective &S,
+                                           const RegionCodeGenTy &CodeGen) {
   auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
-  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
-  auto OutlinedFn = CGM.getOpenMPRuntime().emitOutlinedFunction(
-      S, *CS->getCapturedDecl()->param_begin());
+  auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS);
+  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
+      S, *CS->getCapturedDecl()->param_begin(), CodeGen);
   if (auto C = S.getSingleClause(/*K*/ OMPC_if)) {
     auto Cond = cast<OMPIfClause>(C)->getCondition();
-    EmitOMPIfClause(*this, Cond, [&](bool ThenBlock) {
+    EmitOMPIfClause(CGF, Cond, [&](bool ThenBlock) {
       if (ThenBlock)
-        EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct);
+        emitOMPParallelCall(CGF, S, OutlinedFn, CapturedStruct);
       else
-        CGM.getOpenMPRuntime().emitSerialCall(*this, S.getLocStart(),
+        CGF.CGM.getOpenMPRuntime().emitSerialCall(CGF, S.getLocStart(),
                                                   OutlinedFn, CapturedStruct);
     });
   } else
-    EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct);
+    emitOMPParallelCall(CGF, S, OutlinedFn, CapturedStruct);
+}
+
+void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
+  LexicalScope Scope(*this, S.getSourceRange());
+  // Emit parallel region as a standalone region.
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+    OMPPrivateScope PrivateScope(CGF);
+    CGF.EmitOMPPrivateClause(S, PrivateScope);
+    CGF.EmitOMPFirstprivateClause(S, PrivateScope);
+    if (PrivateScope.Privatize())
+      // Emit implicit barrier to synchronize threads and avoid data races.
+      CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
+                                                 OMPD_unknown);
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+    // Emit implicit barrier at the end of the 'parallel' directive.
+    CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
+                                               OMPD_unknown);
+  };
+  emitCommonOMPParallelDirective(*this, S, CodeGen);
 }
 
 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S,
                                       bool SeparateIter) {
   RunCleanupsScope BodyScope(*this);
   // Update counters values on current iteration.
   for (auto I : S.updates()) {
     EmitIgnoredExpr(I);
void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S,
   if (SeparateIter) {
     // TODO: Update lastprivates if the SeparateIter flag is true.
     // This will be implemented in a follow-up OMPLastprivateClause patch, but
     // result should be still correct without it, as we do not make these
     // variables private yet.
   }
 }
 
-void CodeGenFunction::EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup,
-                                       const Expr *LoopCond,
-                                       const Expr *IncExpr,
-                                       const std::function<void()> &BodyGen) {
+void CodeGenFunction::EmitOMPInnerLoop(
+    const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
+    const llvm::function_ref<void(CodeGenFunction &)> &BodyGen) {
   auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
   auto Cnt = getPGORegionCounter(&S);
 
   // Start the loop with a block that tests the condition.
   auto CondBlock = createBasicBlock("omp.inner.for.cond");
   EmitBlock(CondBlock);
   LoopStack.push(CondBlock);
 
void CodeGenFunction::EmitOMPInnerLoop(
 
   EmitBlock(LoopBody);
   Cnt.beginRegion(Builder);
 
   // Create a block for the increment.
   auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
 
-  BodyGen();
+  BodyGen(*this);
 
   // Emit "IV = IV + 1" and a back-edge to the condition block.
   EmitBlock(Continue.getBlock());
   EmitIgnoredExpr(IncExpr);
   BreakContinueStack.pop_back();
   EmitBranch(CondBlock);
   LoopStack.pop();
   // Emit the fall-through block.
for (auto *E : Clause->varlists()) {
       assert(IsRegistered && "linear var already registered as private");
       // Silence the warning about unused variable.
       (void)IsRegistered;
     }
   }
 }
 
 void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
     // Pragma 'simd' code depends on presence of 'lastprivate'.
     // If present, we have to separate last iteration of the loop:
     //
     // if (LastIteration != 0) {
     //   for (IV in 0..LastIteration-1) BODY;
     //   BODY with updates of lastprivate vars;
     //   <Final counter/linear vars updates>;
     // }
     //
     // otherwise (when there's no lastprivate):
     //
     //   for (IV in 0..LastIteration) BODY;
     //   <Final counter/linear vars updates>;
     //
 
     // Walk clauses and process safelen/lastprivate.
     bool SeparateIter = false;
-  LoopStack.setParallel();
-  LoopStack.setVectorizerEnable(true);
+    CGF.LoopStack.setParallel();
+    CGF.LoopStack.setVectorizerEnable(true);
     for (auto C : S.clauses()) {
       switch (C->getClauseKind()) {
       case OMPC_safelen: {
-      RValue Len = EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(),
+        RValue Len = CGF.EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(),
                                      AggValueSlot::ignored(), true);
         llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
-      LoopStack.setVectorizerWidth(Val->getZExtValue());
+        CGF.LoopStack.setVectorizerWidth(Val->getZExtValue());
         // In presence of finite 'safelen', it may be unsafe to mark all
         // the memory instructions parallel, because loop-carried
         // dependences of 'safelen' iterations are possible.
-      LoopStack.setParallel(false);
+        CGF.LoopStack.setParallel(false);
         break;
       }
       case OMPC_aligned:
-      EmitOMPAlignedClause(*this, CGM, cast<OMPAlignedClause>(*C));
+        EmitOMPAlignedClause(CGF, CGF.CGM, cast<OMPAlignedClause>(*C));
         break;
       case OMPC_lastprivate:
         SeparateIter = true;
         break;
       default:
         // Not handled yet
         ;
       }
     }
 
-  InlinedOpenMPRegionScopeRAII Region(*this, S);
-
     // Emit inits for the linear variables.
     for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
       for (auto Init : C->inits()) {
         auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
-      EmitVarDecl(*D);
+        CGF.EmitVarDecl(*D);
       }
     }
 
     // Emit the loop iteration variable.
     const Expr *IVExpr = S.getIterationVariable();
     const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
-  EmitVarDecl(*IVDecl);
-  EmitIgnoredExpr(S.getInit());
+    CGF.EmitVarDecl(*IVDecl);
+    CGF.EmitIgnoredExpr(S.getInit());
 
     // Emit the iterations count variable.
-  // If it is not a variable, Sema decided to calculate iterations count on each
+    // If it is not a variable, Sema decided to calculate iterations count on
+    // each
     // iteration (e.g., it is foldable into a constant).
     if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
-    EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
+      CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
       // Emit calculation of the iterations count.
-    EmitIgnoredExpr(S.getCalcLastIteration());
+      CGF.EmitIgnoredExpr(S.getCalcLastIteration());
     }
 
     // Emit the linear steps for the linear clauses.
     // If a step is not constant, it is pre-calculated before the loop.
     for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
       if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
         if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
-        EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
+          CGF.EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
           // Emit calculation of the linear step.
-        EmitIgnoredExpr(CS);
+          CGF.EmitIgnoredExpr(CS);
         }
     }
 
     if (SeparateIter) {
       // Emit: if (LastIteration > 0) - begin.
-    RegionCounter Cnt = getPGORegionCounter(&S);
-    auto ThenBlock = createBasicBlock("simd.if.then");
-    auto ContBlock = createBasicBlock("simd.if.end");
-    EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount());
-    EmitBlock(ThenBlock);
-    Cnt.beginRegion(Builder);
+      RegionCounter Cnt = CGF.getPGORegionCounter(&S);
+      auto ThenBlock = CGF.createBasicBlock("simd.if.then");
+      auto ContBlock = CGF.createBasicBlock("simd.if.end");
+      CGF.EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock,
+                               Cnt.getCount());
+      CGF.EmitBlock(ThenBlock);
+      Cnt.beginRegion(CGF.Builder);
       // Emit 'then' code.
       {
-      OMPPrivateScope LoopScope(*this);
-      EmitPrivateLoopCounters(*this, LoopScope, S.counters());
-      EmitPrivateLinearVars(*this, S, LoopScope);
-      EmitOMPPrivateClause(S, LoopScope);
+        OMPPrivateScope LoopScope(CGF);
+        EmitPrivateLoopCounters(CGF, LoopScope, S.counters());
+        EmitPrivateLinearVars(CGF, S, LoopScope);
+        CGF.EmitOMPPrivateClause(S, LoopScope);
         (void)LoopScope.Privatize();
-      EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
+        CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
                              S.getCond(/*SeparateIter=*/true), S.getInc(),
-                       [&S, this]() {
-                         EmitOMPLoopBody(S);
-                         EmitStopPoint(&S);
+                             [&S](CodeGenFunction &CGF) {
+                               CGF.EmitOMPLoopBody(S);
+                               CGF.EmitStopPoint(&S);
                              });
-      EmitOMPLoopBody(S, /* SeparateIter */ true);
+        CGF.EmitOMPLoopBody(S, /* SeparateIter */ true);
       }
-    EmitOMPSimdFinal(S);
+      CGF.EmitOMPSimdFinal(S);
       // Emit: if (LastIteration != 0) - end.
-    EmitBranch(ContBlock);
-    EmitBlock(ContBlock, true);
+      CGF.EmitBranch(ContBlock);
+      CGF.EmitBlock(ContBlock, true);
     } else {
       {
-      OMPPrivateScope LoopScope(*this);
-      EmitPrivateLoopCounters(*this, LoopScope, S.counters());
-      EmitPrivateLinearVars(*this, S, LoopScope);
-      EmitOMPPrivateClause(S, LoopScope);
+        OMPPrivateScope LoopScope(CGF);
+        EmitPrivateLoopCounters(CGF, LoopScope, S.counters());
+        EmitPrivateLinearVars(CGF, S, LoopScope);
+        CGF.EmitOMPPrivateClause(S, LoopScope);
         (void)LoopScope.Privatize();
-      EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
+        CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
                              S.getCond(/*SeparateIter=*/false), S.getInc(),
-                       [&S, this]() {
-                         EmitOMPLoopBody(S);
-                         EmitStopPoint(&S);
+                             [&S](CodeGenFunction &CGF) {
+                               CGF.EmitOMPLoopBody(S);
+                               CGF.EmitStopPoint(&S);
                              });
       }
-    EmitOMPSimdFinal(S);
+      CGF.EmitOMPSimdFinal(S);
     }
+  };
+  CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
 }
 
 void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
                                           const OMPLoopDirective &S,
                                           OMPPrivateScope &LoopScope,
                                           llvm::Value *LB, llvm::Value *UB,
                                           llvm::Value *ST, llvm::Value *IL,
                                           llvm::Value *Chunk) {
void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
   if (Dynamic)
     EmitIgnoredExpr(S.getInit());
 
   // Create a block for the increment.
   auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
 
   EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
-                   S.getCond(/*SeparateIter=*/false), S.getInc(), [&S, this]() {
-                     EmitOMPLoopBody(S);
-                     EmitStopPoint(&S);
+                   S.getCond(/*SeparateIter=*/false), S.getInc(),
+                   [&S](CodeGenFunction &CGF) {
+                     CGF.EmitOMPLoopBody(S);
+                     CGF.EmitStopPoint(&S);
                    });
 
   EmitBlock(Continue.getBlock());
   BreakContinueStack.pop_back();
   if (!Dynamic) {
     // Emit "LB = LB + Stride", "UB = UB + Stride".
     EmitIgnoredExpr(S.getNextLowerBound());
     EmitIgnoredExpr(S.getNextUpperBound());
// Emit 'then' code.
                        ST.getAddress());
         // UB = min(UB, GlobalUB);
         EmitIgnoredExpr(S.getEnsureUpperBound());
         // IV = LB;
         EmitIgnoredExpr(S.getInit());
         // while (idx <= UB) { BODY; ++idx; }
         EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
                          S.getCond(/*SeparateIter=*/false), S.getInc(),
-                         [&S, this]() {
-                           EmitOMPLoopBody(S);
-                           EmitStopPoint(&S);
+                         [&S](CodeGenFunction &CGF) {
+                           CGF.EmitOMPLoopBody(S);
+                           CGF.EmitStopPoint(&S);
                          });
         // Tell the runtime we are done.
         RT.emitForFinish(*this, S.getLocStart(), ScheduleKind);
       } else {
         // Emit the outer loop, which requests its work chunk [LB..UB] from
         // runtime and runs the inner loop to process it.
         EmitOMPForOuterLoop(ScheduleKind, S, LoopScope, LB.getAddress(),
                             UB.getAddress(), ST.getAddress(), IL.getAddress(),
                             Chunk);
       }
     }
     // We're now done with the loop, so jump to the continuation block.
     EmitBranch(ContBlock);
     EmitBlock(ContBlock, true);
   }
 }
 
 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
-  InlinedOpenMPRegionScopeRAII Region(*this, S);
-
-  EmitOMPWorksharingLoop(S);
+  LexicalScope Scope(*this, S.getSourceRange());
+  auto &&CodeGen =
+      [&S](CodeGenFunction &CGF) { CGF.EmitOMPWorksharingLoop(S); };
+  CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
 
   // Emit an implicit barrier at the end.
   if (!S.getSingleClause(OMPC_nowait)) {
     CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
   }
 }
 
 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) {
   llvm_unreachable("CodeGen for 'omp for simd' is not supported yet.");
 }
 
 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
                                 const Twine &Name,
                                 llvm::Value *Init = nullptr) {
   auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
   if (Init)
     CGF.EmitScalarInit(Init, LVal);
   return LVal;
 }
 
 void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
-  InlinedOpenMPRegionScopeRAII Region(*this, S);
-
+  LexicalScope Scope(*this, S.getSourceRange());
   auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
   auto *CS = dyn_cast<CompoundStmt>(Stmt);
   if (CS && CS->size() > 1) {
-    auto &C = CGM.getContext();
+    auto &&CodeGen = [&S, CS](CodeGenFunction &CGF) {
+      auto &C = CGF.CGM.getContext();
       auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
       // Emit helper vars inits.
-    LValue LB = createSectionLVal(*this, KmpInt32Ty, ".omp.sections.lb.",
-                                  Builder.getInt32(0));
-    auto *GlobalUBVal = Builder.getInt32(CS->size() - 1);
+      LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
+                                    CGF.Builder.getInt32(0));
+      auto *GlobalUBVal = CGF.Builder.getInt32(CS->size() - 1);
       LValue UB =
-        createSectionLVal(*this, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
-    LValue ST = createSectionLVal(*this, KmpInt32Ty, ".omp.sections.st.",
-                                  Builder.getInt32(1));
-    LValue IL = createSectionLVal(*this, KmpInt32Ty, ".omp.sections.il.",
-                                  Builder.getInt32(0));
+          createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
+      LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
+                                    CGF.Builder.getInt32(1));
+      LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
+                                    CGF.Builder.getInt32(0));
       // Loop counter.
-    LValue IV = createSectionLVal(*this, KmpInt32Ty, ".omp.sections.iv.");
+      LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
       OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
-    OpaqueValueMapping OpaqueIV(*this, &IVRefExpr, IV);
+      OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
       OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
-    OpaqueValueMapping OpaqueUB(*this, &UBRefExpr, UB);
+      OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
       // Generate condition for loop.
       BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
-                        OK_Ordinary, S.getLocStart(), /*fpContractable=*/false);
+                          OK_Ordinary, S.getLocStart(),
+                          /*fpContractable=*/false);
       // Increment for loop counter.
-    UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
-                      S.getLocStart());
-    auto BodyGen = [this, CS, &S, &IV]() {
+      UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue,
+                        OK_Ordinary, S.getLocStart());
+      auto BodyGen = [CS, &S, &IV](CodeGenFunction &CGF) {
         // Iterate through all sections and emit a switch construct:
         // switch (IV) {
         //   case 0:
         //     <SectionStmt[0]>;
         //     break;
         // ...
         //   case <NumSection> - 1:
         //     <SectionStmt[<NumSection> - 1]>;
         //     break;
         // }
         // .omp.sections.exit:
-      auto *ExitBB = createBasicBlock(".omp.sections.exit");
-      auto *SwitchStmt = Builder.CreateSwitch(
-          EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
+        auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
+        auto *SwitchStmt = CGF.Builder.CreateSwitch(
+            CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
             CS->size());
         unsigned CaseNumber = 0;
         for (auto C = CS->children(); C; ++C, ++CaseNumber) {
-        auto CaseBB = createBasicBlock(".omp.sections.case");
-        EmitBlock(CaseBB);
-        SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
-        EmitStmt(*C);
-        EmitBranch(ExitBB);
+          auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
+          CGF.EmitBlock(CaseBB);
+          SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
+          CGF.EmitStmt(*C);
+          CGF.EmitBranch(ExitBB);
         }
-      EmitBlock(ExitBB, /*IsFinished=*/true);
+        CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
       };
       // Emit static non-chunked loop.
-    CGM.getOpenMPRuntime().emitForInit(
-        *this, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32,
+      CGF.CGM.getOpenMPRuntime().emitForInit(
+          CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32,
           /*IVSigned=*/true, IL.getAddress(), LB.getAddress(), UB.getAddress(),
           ST.getAddress());
       // UB = min(UB, GlobalUB);
-    auto *UBVal = EmitLoadOfScalar(UB, S.getLocStart());
-    auto *MinUBGlobalUB = Builder.CreateSelect(
-        Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
-    EmitStoreOfScalar(MinUBGlobalUB, UB);
+      auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
+      auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
+          CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
+      CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
       // IV = LB;
-    EmitStoreOfScalar(EmitLoadOfScalar(LB, S.getLocStart()), IV);
+      CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
       // while (idx <= UB) { BODY; ++idx; }
-    EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen);
+      CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen);
       // Tell the runtime we are done.
-    CGM.getOpenMPRuntime().emitForFinish(*this, S.getLocStart(),
+      CGF.CGM.getOpenMPRuntime().emitForFinish(CGF, S.getLocStart(),
                                                OMPC_SCHEDULE_static);
+    };
+
+    CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
   } else {
-    // If only one section is found - no need to generate loop, emit as a single
+    // If only one section is found - no need to generate loop, emit as a
+    // single
     // region.
-    CGM.getOpenMPRuntime().emitSingleRegion(*this, [&]() -> void {
-      InlinedOpenMPRegionScopeRAII Region(*this, S);
-      EmitStmt(Stmt);
-      EnsureInsertPoint();
-    }, S.getLocStart(), llvm::None, llvm::None, llvm::None, llvm::None);
+    auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+      CGF.EmitStmt(
+          cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+      CGF.EnsureInsertPoint();
+    };
+    CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
+                                            llvm::None, llvm::None, llvm::None,
+                                            llvm::None);
   }
 
   // Emit an implicit barrier at the end.
   if (!S.getSingleClause(OMPC_nowait)) {
     CGM.getOpenMPRuntime().emitBarrierCall(
         *this, S.getLocStart(),
         (CS && CS->size() > 1) ? OMPD_sections : OMPD_single);
   }
 }
 
 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
-  InlinedOpenMPRegionScopeRAII Region(*this, S);
-  EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
-  EnsureInsertPoint();
+  LexicalScope Scope(*this, S.getSourceRange());
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+    CGF.EnsureInsertPoint();
+  };
+  CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
 }
 
 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
   llvm::SmallVector<const Expr *, 8> CopyprivateVars;
   llvm::SmallVector<const Expr *, 8> SrcExprs;
   llvm::SmallVector<const Expr *, 8> DstExprs;
   llvm::SmallVector<const Expr *, 8> AssignmentOps;
-  // Check if there are any 'copyprivate' clauses associated with this 'single'
+  // Check if there are any 'copyprivate' clauses associated with this
+  // 'single'
   // construct.
   auto CopyprivateFilter = [](const OMPClause *C) -> bool {
     return C->getClauseKind() == OMPC_copyprivate;
   };
   // Build a list of copyprivate variables along with helper expressions
   // (<source>, <destination>, <destination>=<source> expressions)
   typedef OMPExecutableDirective::filtered_clause_iterator<decltype(
       CopyprivateFilter)> CopyprivateIter;
   for (CopyprivateIter I(S.clauses(), CopyprivateFilter); I; ++I) {
     auto *C = cast<OMPCopyprivateClause>(*I);
     CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
     SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
     DstExprs.append(C->destination_exprs().begin(),
                     C->destination_exprs().end());
     AssignmentOps.append(C->assignment_ops().begin(),
                          C->assignment_ops().end());
   }
+  LexicalScope Scope(*this, S.getSourceRange());
   // Emit code for 'single' region along with 'copyprivate' clauses
-  CGM.getOpenMPRuntime().emitSingleRegion(*this, [&]() -> void {
-    InlinedOpenMPRegionScopeRAII Region(*this, S);
-    EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
-    EnsureInsertPoint();
-  }, S.getLocStart(), CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+    CGF.EnsureInsertPoint();
+  };
+  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
+                                          CopyprivateVars, SrcExprs, DstExprs,
+                                          AssignmentOps);
   // Emit an implicit barrier at the end.
   if (!S.getSingleClause(OMPC_nowait)) {
     CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_single);
   }
 }
 
 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
-  CGM.getOpenMPRuntime().emitMasterRegion(*this, [&]() -> void {
-    InlinedOpenMPRegionScopeRAII Region(*this, S);
-    EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
-    EnsureInsertPoint();
-  }, S.getLocStart());
+  LexicalScope Scope(*this, S.getSourceRange());
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+    CGF.EnsureInsertPoint();
+  };
+  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
 }
 
 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
+  LexicalScope Scope(*this, S.getSourceRange());
+  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+    CGF.EnsureInsertPoint();
+  };
   CGM.getOpenMPRuntime().emitCriticalRegion(
-      *this, S.getDirectiveName().getAsString(), [&]() -> void {
-        InlinedOpenMPRegionScopeRAII Region(*this, S);
-        EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
-        EnsureInsertPoint();
-      }, S.getLocStart());
+      *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart());
 }
 
 void
 CodeGenFunction::EmitOMPParallelForDirective(const OMPParallelForDirective &) {
   llvm_unreachable("CodeGen for 'omp parallel for' is not supported yet.");
 }
 
 void CodeGenFunction::EmitOMPParallelForSimdDirective(
     const OMPParallelForSimdDirective &) {
   llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet.");
 }
 
 void CodeGenFunction::EmitOMPParallelSectionsDirective(
     const OMPParallelSectionsDirective &) {
   llvm_unreachable("CodeGen for 'omp parallel sections' is not supported yet.");
 }
 
 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
   // Emit outlined function for task construct.
+  LexicalScope Scope(*this, S.getSourceRange());
   auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
   auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
   auto *I = CS->getCapturedDecl()->param_begin();
+  auto *PartId = std::next(I);
   // The first function argument for tasks is a thread id, the second one is a
   // part id (0 for tied tasks, >=0 for untied task).
+  auto &&CodeGen = [PartId, &S](CodeGenFunction &CGF) {
+    if (*PartId) {
+      // TODO: emit code for untied tasks.
+    }
+    CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+  };
   auto OutlinedFn =
-      CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, *std::next(I));
+      CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen);
   // Check if we should emit tied or untied task.
   bool Tied = !S.getSingleClause(OMPC_untied);
   // Check if the task is final
   llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
   if (auto *Clause = S.getSingleClause(OMPC_final)) {
     // If the condition constant folds and can be elided, try to avoid emitting
     // the condition and the dead arm of the if/else.
     auto *Cond = cast<OMPFinalClause>(Clause)->getCondition();
if (C->getClauseKind() != OMPC_seq_cst) {
       break;
     }
   }
 
   const auto *CS =
       S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
   if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
     enterFullExpression(EWC);
-  InlinedOpenMPRegionScopeRAII Region(*this, S);
 
-  EmitOMPAtomicExpr(*this, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(),
+  LexicalScope Scope(*this, S.getSourceRange());
+  auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) {
+    EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(),
                       S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart());
+  };
+  CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
 }
 
 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) {
   llvm_unreachable("CodeGen for 'omp target' is not supported yet.");
 }
 
 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) {
   llvm_unreachable("CodeGen for 'omp teams' is not supported yet.");
 }
 

cfe/trunk/lib/CodeGen/CodeGenFunction.h

explicit CGCapturedStmtInfo(const CapturedStmt &S,
           CaptureFields[I->getCapturedVar()] = *Field;
       }
     }
 
     virtual ~CGCapturedStmtInfo();
 
     CapturedRegionKind getKind() const { return Kind; }
 
-    void setContextValue(llvm::Value *V) { ThisValue = V; }
+    virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
     // \brief Retrieve the value of the context parameter.
     virtual llvm::Value *getContextValue() const { return ThisValue; }
 
     /// \brief Lookup the captured field decl for a variable.
     virtual const FieldDecl *lookup(const VarDecl *VD) const {
       return CaptureFields.lookup(VD);
     }
 
#endif
   void EmitOMPTargetDirective(const OMPTargetDirective &S);
   void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
 
 private:
 
   /// Helpers for the OpenMP loop directives.
   void EmitOMPLoopBody(const OMPLoopDirective &Directive,
                        bool SeparateIter = false);
-  void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup,
-                        const Expr *LoopCond, const Expr *IncExpr,
-                        const std::function<void()> &BodyGen);
+  void
+  EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
+                   const Expr *IncExpr,
+                   const llvm::function_ref<void(CodeGenFunction &)> &BodyGen);
   void EmitOMPSimdFinal(const OMPLoopDirective &S);
   void EmitOMPWorksharingLoop(const OMPLoopDirective &S);
   void EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
                            const OMPLoopDirective &S,
                            OMPPrivateScope &LoopScope, llvm::Value *LB,
                            llvm::Value *UB, llvm::Value *ST, llvm::Value *IL,
                            llvm::Value *Chunk);
 

cfe/trunk/test/OpenMP/critical_codegen.cpp

#pragma omp critical
   // TERM_DEBUG:     unwind label %[[TERM_LPAD:.+]],
   // TERM_DEBUG-NOT: __kmpc_global_thread_num
   // TERM_DEBUG:     call void @__kmpc_end_critical({{.+}}), !dbg [[DBG_LOC_END:![0-9]+]]
   // TERM_DEBUG:     [[TERM_LPAD]]
   // TERM_DEBUG:     call void @__clang_call_terminate
   // TERM_DEBUG:     unreachable
   foo();
 }
-// TERM_DEBUG-DAG: [[DBG_LOC_START]] = !MDLocation(line: 44,
-// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: 44,
+// TERM_DEBUG-DAG: [[DBG_LOC_START]] = !MDLocation(line: [[@LINE-12]],
+// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: [[@LINE-3]],
 #endif

cfe/trunk/test/OpenMP/master_codegen.cpp

#pragma omp master
   // TERM_DEBUG:     unwind label %[[TERM_LPAD:.+]],
   // TERM_DEBUG-NOT: __kmpc_global_thread_num
   // TERM_DEBUG:     call void @__kmpc_end_master({{.+}}), !dbg [[DBG_LOC_END:![0-9]+]]
   // TERM_DEBUG:     [[TERM_LPAD]]
   // TERM_DEBUG:     call void @__clang_call_terminate
   // TERM_DEBUG:     unreachable
   foo();
 }
-// TERM_DEBUG-DAG: [[DBG_LOC_START]] = !MDLocation(line: 52,
-// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: 52,
+// TERM_DEBUG-DAG: [[DBG_LOC_START]] = !MDLocation(line: [[@LINE-12]],
+// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: [[@LINE-3]],
 
 #endif

cfe/trunk/test/OpenMP/parallel_codegen.cpp

#pragma omp parallel
   return tmain(argv);
 }
 
 // CHECK-LABEL: define {{[a-z]*[ ]?i32}} @main({{i32[ ]?[a-z]*}} %argc, i8** %argv)
 // CHECK:       [[AGG_CAPTURED:%.+]] = alloca %struct.anon
 // CHECK:       [[ARGC_REF:%.+]] = getelementptr inbounds %struct.anon, %struct.anon* [[AGG_CAPTURED]], i32 0, i32 0
 // CHECK-NEXT:  store i32* {{%[a-z0-9.]+}}, i32** [[ARGC_REF]]
 // CHECK-NEXT:  [[BITCAST:%.+]] = bitcast %struct.anon* [[AGG_CAPTURED]] to i8*
-// CHECK-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[DEF_LOC_2]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon*)* @.omp_outlined. to void (i32*, i32*, ...)*), i8* [[BITCAST]])
+// CHECK-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[DEF_LOC_2]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon*)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8* [[BITCAST]])
 // CHECK-NEXT:  [[ARGV:%.+]] = load i8**, i8*** {{%[a-z0-9.]+}}
 // CHECK-NEXT:  [[RET:%.+]] = call {{[a-z]*[ ]?i32}} [[TMAIN:@.+tmain.+]](i8** [[ARGV]])
 // CHECK-NEXT:  ret i32 [[RET]]
 // CHECK-NEXT:  }
 // CHECK-DEBUG-LABEL: define i32 @main(i32 %argc, i8** %argv)
 // CHECK-DEBUG-DAG:   [[AGG_CAPTURED:%.+]] = alloca %struct.anon
 // CHECK-DEBUG-DAG:   [[LOC_2_ADDR:%.+]] = alloca %ident_t
 // CHECK-DEBUG:       [[KMPC_LOC_VOIDPTR:%.+]] = bitcast %ident_t* [[LOC_2_ADDR]] to i8*
 // CHECK-DEBUG-NEXT:  [[KMPC_DEFAULT_LOC_VOIDPTR:%.+]] = bitcast %ident_t* [[DEF_LOC_2]] to i8*
 // CHECK-DEBUG-NEXT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[KMPC_LOC_VOIDPTR]], i8* [[KMPC_DEFAULT_LOC_VOIDPTR]], i64 ptrtoint (%ident_t* getelementptr (%ident_t, %ident_t* null, i32 1) to i64), i32 8, i1 false)
 // CHECK-DEBUG:       [[ARGC_REF:%.+]] = getelementptr inbounds %struct.anon, %struct.anon* [[AGG_CAPTURED]], i32 0, i32 0
 // CHECK-DEBUG-NEXT:  store i32* {{%[a-z0-9.]+}}, i32** [[ARGC_REF]]
 // CHECK-DEBUG-NEXT:  [[KMPC_LOC_PSOURCE_REF:%.+]] = getelementptr inbounds %ident_t, %ident_t* [[LOC_2_ADDR]], i32 0, i32 4
 // CHECK-DEBUG-NEXT:  store i8* getelementptr inbounds ([{{.+}} x i8], [{{.+}} x i8]* [[LOC1]], i32 0, i32 0), i8** [[KMPC_LOC_PSOURCE_REF]]
 // CHECK-DEBUG-NEXT:  [[BITCAST:%.+]] = bitcast %struct.anon* [[AGG_CAPTURED]] to i8*
-// CHECK-DEBUG-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[LOC_2_ADDR]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon*)* @.omp_outlined. to void (i32*, i32*, ...)*), i8* [[BITCAST]])
+// CHECK-DEBUG-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[LOC_2_ADDR]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon*)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8* [[BITCAST]])
 // CHECK-DEBUG-NEXT:  [[ARGV:%.+]] = load i8**, i8*** {{%[a-z0-9.]+}}
 // CHECK-DEBUG-NEXT:  [[RET:%.+]] = call i32 [[TMAIN:@.+tmain.+]](i8** [[ARGV]])
 // CHECK-DEBUG-NEXT:  ret i32 [[RET]]
 // CHECK-DEBUG-NEXT:  }
 
-// CHECK-LABEL: define internal void @.omp_outlined.(i32* %.global_tid., i32* %.bound_tid., %struct.anon* %__context)
+// CHECK:       define internal void [[OMP_OUTLINED]](i32* %.global_tid., i32* %.bound_tid., %struct.anon* %__context)
 // CHECK:       #[[FN_ATTRS:[0-9]+]]
 // CHECK:       [[CONTEXT_ADDR:%.+]] = alloca %struct.anon*
 // CHECK:       store %struct.anon* %__context, %struct.anon** [[CONTEXT_ADDR]]
 // CHECK:       [[CONTEXT_PTR:%.+]] = load %struct.anon*, %struct.anon** [[CONTEXT_ADDR]]
 // CHECK-NEXT:  [[ARGC_PTR_REF:%.+]] = getelementptr inbounds %struct.anon, %struct.anon* [[CONTEXT_PTR]], i32 0, i32 0
 // CHECK-NEXT:  [[ARGC_REF:%.+]] = load i32*, i32** [[ARGC_PTR_REF]]
 // CHECK-NEXT:  [[ARGC:%.+]] = load i32, i32* [[ARGC_REF]]
 // CHECK-NEXT:  invoke void [[FOO:@.+foo.+]](i32{{[ ]?[a-z]*}} [[ARGC]])
+// CHECK:       call {{.+}} @__kmpc_cancel_barrier(
 // CHECK:       ret void
 // CHECK:       call void @{{.+terminate.*|abort}}(
 // CHECK-NEXT:  unreachable
 // CHECK-NEXT:  }
-// CHECK-DEBUG-LABEL: define internal void @.omp_outlined.(i32* %.global_tid., i32* %.bound_tid., %struct.anon* %__context)
+// CHECK-DEBUG:       define internal void [[OMP_OUTLINED]](i32* %.global_tid., i32* %.bound_tid., %struct.anon* %__context)
 // CHECK-DEBUG:       #[[FN_ATTRS:[0-9]+]]
 // CHECK-DEBUG:       [[CONTEXT_ADDR:%.+]] = alloca %struct.anon*
 // CHECK-DEBUG:       store %struct.anon* %__context, %struct.anon** [[CONTEXT_ADDR]]
 // CHECK-DEBUG:       [[CONTEXT_PTR:%.+]] = load %struct.anon*, %struct.anon** [[CONTEXT_ADDR]]
 // CHECK-DEBUG-NEXT:  [[ARGC_PTR_REF:%.+]] = getelementptr inbounds %struct.anon, %struct.anon* [[CONTEXT_PTR]], i32 0, i32 0
 // CHECK-DEBUG-NEXT:  [[ARGC_REF:%.+]] = load i32*, i32** [[ARGC_PTR_REF]]
 // CHECK-DEBUG-NEXT:  [[ARGC:%.+]] = load i32, i32* [[ARGC_REF]]
 // CHECK-DEBUG-NEXT:  invoke void [[FOO:@.+foo.+]](i32 [[ARGC]])
+// CHECK-DEBUG:       call {{.+}} @__kmpc_cancel_barrier(
 // CHECK-DEBUG:       ret void
 // CHECK-DEBUG:       call void @{{.+terminate.*|abort}}(
 // CHECK-DEBUG-NEXT:  unreachable
 // CHECK-DEBUG-NEXT:  }
 
 // CHECK-DAG: define linkonce_odr void [[FOO]]({{i32[ ]?[a-z]*}} %argc)
 // CHECK-DAG: declare void @__kmpc_fork_call(%ident_t*, i32, void (i32*, i32*, ...)*, ...)
 // CHECK-DEBUG-DAG: define linkonce_odr void [[FOO]](i32 %argc)
 // CHECK-DEBUG-DAG: declare void @__kmpc_fork_call(%ident_t*, i32, void (i32*, i32*, ...)*, ...)
 
 // CHECK:       define linkonce_odr {{[a-z]*[ ]?i32}} [[TMAIN]](i8** %argc)
 // CHECK:       [[AGG_CAPTURED:%.+]] = alloca %struct.anon.0
 // CHECK:       [[ARGC_REF:%.+]] = getelementptr inbounds %struct.anon.0, %struct.anon.0* [[AGG_CAPTURED]], i32 0, i32 0
 // CHECK-NEXT:  store i8*** {{%[a-z0-9.]+}}, i8**** [[ARGC_REF]]
 // CHECK-NEXT:  [[BITCAST:%.+]] = bitcast %struct.anon.0* [[AGG_CAPTURED]] to i8*
-// CHECK-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[DEF_LOC_2]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon.0*)* @.omp_outlined.1 to void (i32*, i32*, ...)*), i8* [[BITCAST]])
+// CHECK-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[DEF_LOC_2]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon.0*)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8* [[BITCAST]])
 // CHECK-NEXT:  ret i32 0
 // CHECK-NEXT:  }
 // CHECK-DEBUG:       define linkonce_odr i32 [[TMAIN]](i8** %argc)
 // CHECK-DEBUG-DAG:   [[AGG_CAPTURED:%.+]] = alloca %struct.anon.0
 // CHECK-DEBUG-DAG:   [[LOC_2_ADDR:%.+]] = alloca %ident_t
 // CHECK-DEBUG:       [[KMPC_LOC_VOIDPTR:%.+]] = bitcast %ident_t* [[LOC_2_ADDR]] to i8*
 // CHECK-DEBUG-NEXT:  [[KMPC_DEFAULT_LOC_VOIDPTR:%.+]] = bitcast %ident_t* [[DEF_LOC_2]] to i8*
 // CHECK-DEBUG-NEXT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[KMPC_LOC_VOIDPTR]], i8* [[KMPC_DEFAULT_LOC_VOIDPTR]], i64 ptrtoint (%ident_t* getelementptr (%ident_t, %ident_t* null, i32 1) to i64), i32 8, i1 false)
 // CHECK-DEBUG:       [[ARGC_REF:%.+]] = getelementptr inbounds %struct.anon.0, %struct.anon.0* [[AGG_CAPTURED]], i32 0, i32 0
 // CHECK-DEBUG-NEXT:  store i8*** {{%[a-z0-9.]+}}, i8**** [[ARGC_REF]]
 // CHECK-DEBUG-NEXT:  [[KMPC_LOC_PSOURCE_REF:%.+]] = getelementptr inbounds %ident_t, %ident_t* [[LOC_2_ADDR]], i32 0, i32 4
 // CHECK-DEBUG-NEXT:  store i8* getelementptr inbounds ([{{.+}} x i8], [{{.+}} x i8]* [[LOC2]], i32 0, i32 0), i8** [[KMPC_LOC_PSOURCE_REF]]
 // CHECK-DEBUG-NEXT:  [[BITCAST:%.+]] = bitcast %struct.anon.0* [[AGG_CAPTURED]] to i8*
-// CHECK-DEBUG-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[LOC_2_ADDR]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon.0*)* @.omp_outlined.1 to void (i32*, i32*, ...)*), i8* [[BITCAST]])
+// CHECK-DEBUG-NEXT:  call void (%ident_t*, i32, void (i32*, i32*, ...)*, ...)* @__kmpc_fork_call(%ident_t* [[LOC_2_ADDR]], i32 1, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, %struct.anon.0*)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8* [[BITCAST]])
 // CHECK-DEBUG-NEXT:  ret i32 0
 // CHECK-DEBUG-NEXT:  }
 
-// CHECK-LABEL: define internal void @.omp_outlined.1(i32* %.global_tid., i32* %.bound_tid., %struct.anon.0* %__context)
+// CHECK:       define internal void [[OMP_OUTLINED]](i32* %.global_tid., i32* %.bound_tid., %struct.anon.0* %__context)
 // CHECK:       [[CONTEXT_ADDR:%.+]] = alloca %struct.anon.0*
 // CHECK:       store %struct.anon.0* %__context, %struct.anon.0** [[CONTEXT_ADDR]]
 // CHECK:       [[CONTEXT_PTR:%.+]] = load %struct.anon.0*, %struct.anon.0** [[CONTEXT_ADDR]]
 // CHECK-NEXT:  [[ARGC_PTR_REF:%.+]] = getelementptr inbounds %struct.anon.0, %struct.anon.0* [[CONTEXT_PTR]], i32 0, i32 0
 // CHECK-NEXT:  [[ARGC_REF:%.+]] = load i8***, i8**** [[ARGC_PTR_REF]]
 // CHECK-NEXT:  [[ARGC:%.+]] = load i8**, i8*** [[ARGC_REF]]
 // CHECK-NEXT:  invoke void [[FOO1:@.+foo.+]](i8** [[ARGC]])
+// CHECK:       call {{.+}} @__kmpc_cancel_barrier(
 // CHECK:       ret void
 // CHECK:       call void @{{.+terminate.*|abort}}(
 // CHECK-NEXT:  unreachable
 // CHECK-NEXT:  }
-// CHECK-DEBUG-LABEL: define internal void @.omp_outlined.1(i32* %.global_tid., i32* %.bound_tid., %struct.anon.0* %__context)
+// CHECK-DEBUG:       define internal void [[OMP_OUTLINED]](i32* %.global_tid., i32* %.bound_tid., %struct.anon.0* %__context)
 // CHECK-DEBUG:       [[CONTEXT_ADDR:%.+]] = alloca %struct.anon.0*
 // CHECK-DEBUG:       store %struct.anon.0* %__context, %struct.anon.0** [[CONTEXT_ADDR]]
 // CHECK-DEBUG:       [[CONTEXT_PTR:%.+]] = load %struct.anon.0*, %struct.anon.0** [[CONTEXT_ADDR]]
 // CHECK-DEBUG-NEXT:  [[ARGC_PTR_REF:%.+]] = getelementptr inbounds %struct.anon.0, %struct.anon.0* [[CONTEXT_PTR]], i32 0, i32 0
 // CHECK-DEBUG-NEXT:  [[ARGC_REF:%.+]] = load i8***, i8**** [[ARGC_PTR_REF]]
 // CHECK-DEBUG-NEXT:  [[ARGC:%.+]] = load i8**, i8*** [[ARGC_REF]]
 // CHECK-DEBUG-NEXT:  invoke void [[FOO1:@.+foo.+]](i8** [[ARGC]])
+// CHECK-DEBUG:       call {{.+}} @__kmpc_cancel_barrier(
 // CHECK-DEBUG:       ret void
 // CHECK-DEBUG:       call void @{{.+terminate.*|abort}}(
 // CHECK-DEBUG-NEXT:  unreachable
 // CHECK-DEBUG-NEXT:  }
 
 // CHECK: define linkonce_odr void [[FOO1]](i8** %argc)
 // CHECK-DEBUG: define linkonce_odr void [[FOO1]](i8** %argc)
 
 // CHECK: attributes #[[FN_ATTRS]] = {{.+}} nounwind
 // CHECK-DEBUG: attributes #[[FN_ATTRS]] = {{.+}} nounwind
 
 #endif

cfe/trunk/test/OpenMP/sections_codegen.cpp

 // CHECK-NEXT:  br i1 [[BOOLRES]], label %[[THEN:.+]], label %[[END:.+]]
 // CHECK:       [[THEN]]
 // CHECK-NEXT:  invoke void @{{.*}}foo{{.*}}()
 // CHECK-NEXT:  unwind label %[[TERM_LPAD:.+]]
 // CHECK:       call void @__kmpc_end_single(
 // CHECK-NEXT:  br label %[[END]]
 // CHECK:       [[END]]
 // CHECK-NEXT:  call i32 @__kmpc_cancel_barrier(%{{.+}}* [[IMPLICIT_BARRIER_SINGLE_LOC]],
+// CHECK-NEXT:  call i32 @__kmpc_cancel_barrier(
 // CHECK-NEXT:  ret
 // CHECK:       [[TERM_LPAD]]
 // CHECK:       call void @__clang_call_terminate(i8*
 // CHECK-NEXT:  unreachable
 
 #endif

cfe/trunk/test/OpenMP/single_codegen.cpp

#pragma omp single
   // TERM_DEBUG-NOT: __kmpc_global_thread_num
   // TERM_DEBUG:     call void @__kmpc_end_single({{.+}}), !dbg [[DBG_LOC_END:![0-9]+]]
   // TERM_DEBUG:     [[TERM_LPAD]]
   // TERM_DEBUG:     call void @__clang_call_terminate
   // TERM_DEBUG:     unreachable
   foo();
 }
 // TERM_DEBUG-DAG: [[DBG_LOC_START]] = !MDLocation(line: [[@LINE-12]],
-// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: [[@LINE-13]],
+// TERM_DEBUG-DAG: [[DBG_LOC_END]] = !MDLocation(line: [[@LINE-3]],
 
 #endif