[CodeGen] #pragma clang transform

[Meinersbur]

Summary

Emit LLVM-IR metadata for #pragma clang transform.

For a full description, see D69088.

[Meinersbur]

• Switch to Monorepo
• Rebase to latest D69091
• Accommodate revert of r374288 (vectorize_width implies vectorize(enable))

[Meinersbur]

ping

[Meinersbur]

• Remove handling of OpenMP simd and LoopHints by CGTransform

lib/CodeGen/CGBlocks.cpp

auto GenVoidPtrTy =
       IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
   LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
   auto GenVoidPtrSize = CharUnits::fromQuantity(
       CGM.getTarget().getPointerWidth(
           CGM.getContext().getTargetAddressSpace(GenVoidPtrAddr)) /
       8);
   // Using the computed layout, generate the actual block function.
   bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
-  CodeGenFunction BlockCGF{CGM, true};
+  CodeGenFunction BlockCGF{CGM, true, ParentFn};
   BlockCGF.SanOpts = SanOpts;
   auto *InvokeFn = BlockCGF.GenerateBlockFunction(
       CurGD, blockInfo, LocalDeclMap, isLambdaConv, blockInfo.CanBeGlobal);
   auto *blockFn = llvm::ConstantExpr::getPointerCast(InvokeFn, GenVoidPtrTy);
 
   // If there is nothing to capture, we can emit this as a global block.
   if (blockInfo.CanBeGlobal)
     return CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression);
if (BuildGlobalBlock) {
     buildGlobalBlock(CGM, blockInfo,
                      llvm::ConstantExpr::getPointerCast(fn, GenVoidPtrTy));
   }
 
   // Begin generating the function.
   StartFunction(blockDecl, fnType->getReturnType(), fn, fnInfo, args,
                 blockDecl->getLocation(),
                 blockInfo.getBlockExpr()->getBody()->getBeginLoc());
+  HandleCodeTransformations(blockDecl->getBody());
 
   // Okay.  Undo some of what StartFunction did.
 
   // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
   // won't delete the dbg.declare intrinsics for captured variables.
   llvm::Value *BlockPointerDbgLoc = BlockPointer;
   if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
     // Allocate a stack slot for it, so we can point the debugger to it

lib/CodeGen/CGCXX.cpp

 llvm::Function *CodeGenModule::codegenCXXStructor(GlobalDecl GD) {
   const CGFunctionInfo &FnInfo = getTypes().arrangeCXXStructorDeclaration(GD);
   auto *Fn = cast<llvm::Function>(
       getAddrOfCXXStructor(GD, &FnInfo, /*FnType=*/nullptr,
                            /*DontDefer=*/true, ForDefinition));
 
   setFunctionLinkage(GD, Fn);
 
-  CodeGenFunction(*this).GenerateCode(GD, Fn, FnInfo);
+  CodeGenFunction(*this, false, cast<FunctionDecl>(GD.getDecl()))
+      .GenerateCode(GD, Fn, FnInfo);
   setNonAliasAttributes(GD, Fn);
   SetLLVMFunctionAttributesForDefinition(cast<CXXMethodDecl>(GD.getDecl()), Fn);
   return Fn;
 }
 
 llvm::FunctionCallee CodeGenModule::getAddrAndTypeOfCXXStructor(
     GlobalDecl GD, const CGFunctionInfo *FnInfo, llvm::FunctionType *FnType,
     bool DontDefer, ForDefinition_t IsForDefinition) {

lib/CodeGen/CGException.cpp

void CodeGenFunction::startOutlinedSEHHelper(CodeGenFunction &ParentCGF,
   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
   llvm::Function *Fn = llvm::Function::Create(
       FnTy, llvm::GlobalValue::InternalLinkage, Name.str(), &CGM.getModule());
 
   IsOutlinedSEHHelper = true;
 
   StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args,
                 OutlinedStmt->getBeginLoc(), OutlinedStmt->getBeginLoc());
+  LoopStack.initAsOutlined(ParentCGF.LoopStack);
   CurSEHParent = ParentCGF.CurSEHParent;
 
   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FnInfo, CurFn);
   EmitCapturedLocals(ParentCGF, OutlinedStmt, IsFilter);
 }
 
 /// Create a stub filter function that will ultimately hold the code of the
 /// filter expression. The EH preparation passes in LLVM will outline the code

lib/CodeGen/CGLoopInfo.h

 //===---- CGLoopInfo.h - LLVM CodeGen for loop metadata -*- C++ -*---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This is the internal state used for llvm translation for loop statement
 // metadata.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H
 #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H
 
-#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class BasicBlock;
 class Instruction;
 class MDNode;
+class LLVMContext;
 } // end namespace llvm
 
 namespace clang {
-class Attr;
 class ASTContext;
-namespace CodeGen {
-
-/// Attributes that may be specified on loops.
-struct LoopAttributes {
-  explicit LoopAttributes(bool IsParallel = false);
-  void clear();
-
-  /// Generate llvm.loop.parallel metadata for loads and stores.
-  bool IsParallel;
-
-  /// State of loop vectorization or unrolling.
-  enum LVEnableState { Unspecified, Enable, Disable, Full };
-
-  /// Value for llvm.loop.vectorize.enable metadata.
-  LVEnableState VectorizeEnable;
-
-  /// Value for llvm.loop.unroll.* metadata (enable, disable, or full).
-  LVEnableState UnrollEnable;
-
-  /// Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
-  LVEnableState UnrollAndJamEnable;
-
-  /// Value for llvm.loop.vectorize.predicate metadata
-  LVEnableState VectorizePredicateEnable;
-
-  /// Value for llvm.loop.vectorize.width metadata.
-  unsigned VectorizeWidth;
-
-  /// Value for llvm.loop.interleave.count metadata.
-  unsigned InterleaveCount;
-
-  /// llvm.unroll.
-  unsigned UnrollCount;
+class Stmt;
 
-  /// llvm.unroll.
-  unsigned UnrollAndJamCount;
-
-  /// Value for llvm.loop.distribute.enable metadata.
-  LVEnableState DistributeEnable;
-
-  /// Value for llvm.loop.pipeline.disable metadata.
-  bool PipelineDisabled;
-
-  /// Value for llvm.loop.pipeline.iicount metadata.
-  unsigned PipelineInitiationInterval;
-};
+namespace CodeGen {
+class CGTransformedTree;
+class CGDebugInfo;
 
 /// Information used when generating a structured loop.
 class LoopInfo {
 public:
   /// Construct a new LoopInfo for the loop with entry Header.
-  LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs,
-           const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
-           LoopInfo *Parent);
+  LoopInfo(llvm::BasicBlock *Header, CGTransformedTree *TreeNode);
 
   /// Get the loop id metadata for this loop.
-  llvm::MDNode *getLoopID() const { return TempLoopID.get(); }
+  llvm::MDNode *getLoopID() const { return LoopMD; }
 
   /// Get the header block of this loop.
   llvm::BasicBlock *getHeader() const { return Header; }
 
-  /// Get the set of attributes active for this loop.
-  const LoopAttributes &getAttributes() const { return Attrs; }
 
   /// Return this loop's access group or nullptr if it does not have one.
   llvm::MDNode *getAccessGroup() const { return AccGroup; }
 
-  /// Create the loop's metadata. Must be called after its nested loops have
-  /// been processed.
-  void finish();
 
 private:
-  /// Loop ID metadata.
-  llvm::TempMDTuple TempLoopID;
+  /// The metadata node containing this loop's properties. It is assigned to the
+  /// terminators of all loop latches.
+  llvm::MDNode *LoopMD = nullptr;
+
   /// Header block of this loop.
   llvm::BasicBlock *Header;
-  /// The attributes for this loop.
-  LoopAttributes Attrs;
-  /// The access group for memory accesses parallel to this loop.
+
+  /// The metadata node to be assigned to all memory accesses within the loop.
   llvm::MDNode *AccGroup = nullptr;
-  /// Start location of this loop.
-  llvm::DebugLoc StartLoc;
-  /// End location of this loop.
-  llvm::DebugLoc EndLoc;
-  /// The next outer loop, or nullptr if this is the outermost loop.
-  LoopInfo *Parent;
-  /// If this loop has unroll-and-jam metadata, this can be set by the inner
-  /// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner
-  /// metadata.
-  llvm::MDNode *UnrollAndJamInnerFollowup = nullptr;
-
-  /// Create a LoopID without any transformations.
-  llvm::MDNode *
-  createLoopPropertiesMetadata(llvm::ArrayRef<llvm::Metadata *> LoopProperties);
-
-  /// Create a LoopID for transformations.
-  ///
-  /// The methods call each other in case multiple transformations are applied
-  /// to a loop. The transformation first to be applied will use LoopID of the
-  /// next transformation in its followup attribute.
-  ///
-  /// @param Attrs             The loop's transformations.
-  /// @param LoopProperties    Non-transformation properties such as debug
-  ///                          location, parallel accesses and disabled
-  ///                          transformations. These are added to the returned
-  ///                          LoopID.
-  /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
-  ///                          at least one transformation.
-  ///
-  /// @return A LoopID (metadata node) that can be used for the llvm.loop
-  ///         annotation or followup-attribute.
-  /// @{
-  llvm::MDNode *
-  createPipeliningMetadata(const LoopAttributes &Attrs,
-                           llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                           bool &HasUserTransforms);
-  llvm::MDNode *
-  createPartialUnrollMetadata(const LoopAttributes &Attrs,
-                              llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                              bool &HasUserTransforms);
-  llvm::MDNode *
-  createUnrollAndJamMetadata(const LoopAttributes &Attrs,
-                             llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                             bool &HasUserTransforms);
-  llvm::MDNode *
-  createLoopVectorizeMetadata(const LoopAttributes &Attrs,
-                              llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                              bool &HasUserTransforms);
-  llvm::MDNode *
-  createLoopDistributeMetadata(const LoopAttributes &Attrs,
-                               llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                               bool &HasUserTransforms);
-  llvm::MDNode *
-  createFullUnrollMetadata(const LoopAttributes &Attrs,
-                           llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                           bool &HasUserTransforms);
-  /// @}
-
-  /// Create a LoopID for this loop, including transformation-unspecific
-  /// metadata such as debug location.
-  ///
-  /// @param Attrs             This loop's attributes and transformations.
-  /// @param LoopProperties    Additional non-transformation properties to add
-  ///                          to the LoopID, such as transformation-specific
-  ///                          metadata that are not covered by @p Attrs.
-  /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
-  ///                          at least one transformation.
-  ///
-  /// @return A LoopID (metadata node) that can be used for the llvm.loop
-  ///         annotation.
-  llvm::MDNode *createMetadata(const LoopAttributes &Attrs,
-                               llvm::ArrayRef<llvm::Metadata *> LoopProperties,
-                               bool &HasUserTransforms);
 };
 
 /// A stack of loop information corresponding to loop nesting levels.
 /// This stack can be used to prepare attributes which are applied when a loop
 /// is emitted.
 class LoopInfoStack {
   LoopInfoStack(const LoopInfoStack &) = delete;
   void operator=(const LoopInfoStack &) = delete;
 
 public:
   LoopInfoStack() {}
+  ~LoopInfoStack();
 
-  /// Begin a new structured loop. The set of staged attributes will be
-  /// applied to the loop and then cleared.
-  void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc,
-            const llvm::DebugLoc &EndLoc);
-
-  /// Begin a new structured loop. Stage attributes from the Attrs list.
-  /// The staged attributes are applied to the loop and then cleared.
-  void push(llvm::BasicBlock *Header, clang::ASTContext &Ctx,
-            llvm::ArrayRef<const Attr *> Attrs, const llvm::DebugLoc &StartLoc,
-            const llvm::DebugLoc &EndLoc);
+  void initAsOutlined(LoopInfoStack &ParentLIS) {
+    StmtToTree = ParentLIS.StmtToTree;
+  }
 
-  /// End the current loop.
-  void pop();
+  void initBuild(ASTContext &ASTCtx, llvm::LLVMContext &LLVMCtx,
+                 CGDebugInfo *DbgInfo, Stmt *Body);
 
-  /// Return the top loop id metadata.
-  llvm::MDNode *getCurLoopID() const { return getInfo().getLoopID(); }
+  /// Begin a new structured loop.
+  void push(llvm::BasicBlock *Header, const Stmt *LoopStmt);
 
-  /// Return true if the top loop is parallel.
-  bool getCurLoopParallel() const {
-    return hasInfo() ? getInfo().getAttributes().IsParallel : false;
-  }
+  /// End the current loop.
+  void pop();
 
   /// Function called by the CodeGenFunction when an instruction is
   /// created.
   void InsertHelper(llvm::Instruction *I) const;
 
-  /// Set the next pushed loop as parallel.
-  void setParallel(bool Enable = true) { StagedAttrs.IsParallel = Enable; }
-
-  /// Set the next pushed loop 'vectorize.enable'
-  void setVectorizeEnable(bool Enable = true) {
-    StagedAttrs.VectorizeEnable =
-        Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
-  }
-
-  /// Set the next pushed loop as a distribution candidate.
-  void setDistributeState(bool Enable = true) {
-    StagedAttrs.DistributeEnable =
-        Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
-  }
-
-  /// Set the next pushed loop unroll state.
-  void setUnrollState(const LoopAttributes::LVEnableState &State) {
-    StagedAttrs.UnrollEnable = State;
-  }
-
-  /// Set the next pushed vectorize predicate state.
-  void setVectorizePredicateState(const LoopAttributes::LVEnableState &State) {
-    StagedAttrs.VectorizePredicateEnable = State;
-  }
-
-  /// Set the next pushed loop unroll_and_jam state.
-  void setUnrollAndJamState(const LoopAttributes::LVEnableState &State) {
-    StagedAttrs.UnrollAndJamEnable = State;
-  }
-
-  /// Set the vectorize width for the next loop pushed.
-  void setVectorizeWidth(unsigned W) { StagedAttrs.VectorizeWidth = W; }
-
-  /// Set the interleave count for the next loop pushed.
-  void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; }
-
-  /// Set the unroll count for the next loop pushed.
-  void setUnrollCount(unsigned C) { StagedAttrs.UnrollCount = C; }
-
-  /// \brief Set the unroll count for the next loop pushed.
-  void setUnrollAndJamCount(unsigned C) { StagedAttrs.UnrollAndJamCount = C; }
-
-  /// Set the pipeline disabled state.
-  void setPipelineDisabled(bool S) { StagedAttrs.PipelineDisabled = S; }
-
-  /// Set the pipeline initiation interval.
-  void setPipelineInitiationInterval(unsigned C) {
-    StagedAttrs.PipelineInitiationInterval = C;
-  }
-
 private:
   /// Returns true if there is LoopInfo on the stack.
   bool hasInfo() const { return !Active.empty(); }
   /// Return the LoopInfo for the current loop. HasInfo should be called
   /// first to ensure LoopInfo is present.
   const LoopInfo &getInfo() const { return *Active.back(); }
-  /// The set of attributes that will be applied to the next pushed loop.
-  LoopAttributes StagedAttrs;
   /// Stack of active loops.
   llvm::SmallVector<std::unique_ptr<LoopInfo>, 4> Active;
+
+  llvm::SmallVector<CGTransformedTree *, 64> AllNodes;
+  llvm::DenseMap<Stmt *, CGTransformedTree *> StmtToTree;
+
+  CGTransformedTree *TransformedStructure = nullptr;
 };
 
 } // end namespace CodeGen
 } // end namespace clang
 
 #endif

lib/CodeGen/CGLoopInfo.cpp

 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CGLoopInfo.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Attr.h"
+#include "CGTransform.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Metadata.h"
+
 using namespace clang::CodeGen;
 using namespace llvm;
 
-MDNode *
-LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
-  LLVMContext &Ctx = Header->getContext();
-  SmallVector<Metadata *, 4> NewLoopProperties;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  NewLoopProperties.push_back(TempNode.get());
-  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-
-  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
-  LoopID->replaceOperandWith(0, LoopID);
-  return LoopID;
-}
-
-MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
-                                           ArrayRef<Metadata *> LoopProperties,
-                                           bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.PipelineDisabled)
-    Enabled = false;
-  else if (Attrs.PipelineInitiationInterval != 0)
-    Enabled = true;
-
-  if (Enabled != true) {
-    SmallVector<Metadata *, 4> NewLoopProperties;
-    if (Enabled == false) {
-      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-      NewLoopProperties.push_back(
-          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
-                            ConstantAsMetadata::get(ConstantInt::get(
-                                llvm::Type::getInt1Ty(Ctx), 1))}));
-      LoopProperties = NewLoopProperties;
-    }
-    return createLoopPropertiesMetadata(LoopProperties);
-  }
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-
-  if (Attrs.PipelineInitiationInterval > 0) {
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
-        ConstantAsMetadata::get(ConstantInt::get(
-            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  // No follow-up: This is the last transformation.
-
-  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *
-LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
-                                      ArrayRef<Metadata *> LoopProperties,
-                                      bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.UnrollEnable == LoopAttributes::Disable)
-    Enabled = false;
-  else if (Attrs.UnrollEnable == LoopAttributes::Full)
-    Enabled = None;
-  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
-           Attrs.UnrollCount != 0)
-    Enabled = true;
-
-  if (Enabled != true) {
-    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
-    // if unrolling is disabled.
-    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
-  }
-
-  SmallVector<Metadata *, 4> FollowupLoopProperties;
-
-  // Apply all loop properties to the unrolled loop.
-  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-
-  // Don't unroll an already unrolled loop.
-  FollowupLoopProperties.push_back(
-      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
-
-  bool FollowupHasTransforms = false;
-  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
-                                              FollowupHasTransforms);
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-
-  // Setting unroll.count
-  if (Attrs.UnrollCount > 0) {
-    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
-                        ConstantAsMetadata::get(ConstantInt::get(
-                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  // Setting unroll.full or unroll.disable
-  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
-    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  if (FollowupHasTransforms)
-    Args.push_back(MDNode::get(
-        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
-
-  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *
-LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
-                                     ArrayRef<Metadata *> LoopProperties,
-                                     bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
-    Enabled = false;
-  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
-           Attrs.UnrollAndJamCount != 0)
-    Enabled = true;
-
-  if (Enabled != true) {
-    SmallVector<Metadata *, 4> NewLoopProperties;
-    if (Enabled == false) {
-      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-      NewLoopProperties.push_back(MDNode::get(
-          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
-      LoopProperties = NewLoopProperties;
-    }
-    return createPartialUnrollMetadata(Attrs, LoopProperties,
-                                       HasUserTransforms);
-  }
-
-  SmallVector<Metadata *, 4> FollowupLoopProperties;
-  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-  FollowupLoopProperties.push_back(
-      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
-
-  bool FollowupHasTransforms = false;
-  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
-                                                 FollowupHasTransforms);
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-
-  // Setting unroll_and_jam.count
-  if (Attrs.UnrollAndJamCount > 0) {
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
-        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
-                                                 Attrs.UnrollAndJamCount))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
-    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  if (FollowupHasTransforms)
-    Args.push_back(MDNode::get(
-        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
-              Followup}));
-
-  if (UnrollAndJamInnerFollowup)
-    Args.push_back(MDNode::get(
-        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
-              UnrollAndJamInnerFollowup}));
-
-  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *
-LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
-                                      ArrayRef<Metadata *> LoopProperties,
-                                      bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
-    Enabled = false;
-  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
-           Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
-           Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
-    Enabled = true;
-
-  if (Enabled != true) {
-    SmallVector<Metadata *, 4> NewLoopProperties;
-    if (Enabled == false) {
-      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-      NewLoopProperties.push_back(
-          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
-                            ConstantAsMetadata::get(ConstantInt::get(
-                                llvm::Type::getInt1Ty(Ctx), 0))}));
-      LoopProperties = NewLoopProperties;
-    }
-    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
-  }
-
-  // Apply all loop properties to the vectorized loop.
-  SmallVector<Metadata *, 4> FollowupLoopProperties;
-  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-
-  // Don't vectorize an already vectorized loop.
-  FollowupLoopProperties.push_back(
-      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
-
-  bool FollowupHasTransforms = false;
-  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
-                                                FollowupHasTransforms);
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-
-  // Setting vectorize.predicate
-  bool IsVectorPredicateEnabled = false;
-  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
-      Attrs.VectorizeEnable != LoopAttributes::Disable &&
-      Attrs.VectorizeWidth < 1) {
-
-    IsVectorPredicateEnabled =
-        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
-
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
-        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
-                                                 IsVectorPredicateEnabled))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  // Setting vectorize.width
-  if (Attrs.VectorizeWidth > 0) {
-    // This implies vectorize.enable = true, but only add it when it is not
-    // already enabled.
-    if (Attrs.VectorizeEnable != LoopAttributes::Enable)
-      Args.push_back(
-          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
-                            ConstantAsMetadata::get(ConstantInt::get(
-                                llvm::Type::getInt1Ty(Ctx), 1))}));
-
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.vectorize.width"),
-        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
-                                                 Attrs.VectorizeWidth))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  // Setting interleave.count
-  if (Attrs.InterleaveCount > 0) {
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.interleave.count"),
-        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
-                                                 Attrs.InterleaveCount))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  // Setting vectorize.enable
-  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
-      IsVectorPredicateEnabled) {
-    Metadata *Vals[] = {
-        MDString::get(Ctx, "llvm.loop.vectorize.enable"),
-        ConstantAsMetadata::get(ConstantInt::get(
-            llvm::Type::getInt1Ty(Ctx),
-            IsVectorPredicateEnabled
-                ? true
-                : (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
-    Args.push_back(MDNode::get(Ctx, Vals));
-  }
-
-  if (FollowupHasTransforms)
-    Args.push_back(MDNode::get(
-        Ctx,
-        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
-
-  MDNode *LoopID = MDNode::get(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *
-LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
-                                       ArrayRef<Metadata *> LoopProperties,
-                                       bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.DistributeEnable == LoopAttributes::Disable)
-    Enabled = false;
-  if (Attrs.DistributeEnable == LoopAttributes::Enable)
-    Enabled = true;
-
-  if (Enabled != true) {
-    SmallVector<Metadata *, 4> NewLoopProperties;
-    if (Enabled == false) {
-      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-      NewLoopProperties.push_back(
-          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
-                            ConstantAsMetadata::get(ConstantInt::get(
-                                llvm::Type::getInt1Ty(Ctx), 0))}));
-      LoopProperties = NewLoopProperties;
-    }
-    return createLoopVectorizeMetadata(Attrs, LoopProperties,
-                                       HasUserTransforms);
-  }
-
-  bool FollowupHasTransforms = false;
-  MDNode *Followup =
-      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-
-  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
-                      ConstantAsMetadata::get(ConstantInt::get(
-                          llvm::Type::getInt1Ty(Ctx),
-                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
-  Args.push_back(MDNode::get(Ctx, Vals));
-
-  if (FollowupHasTransforms)
-    Args.push_back(MDNode::get(
-        Ctx,
-        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
-
-  MDNode *LoopID = MDNode::get(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
-                                           ArrayRef<Metadata *> LoopProperties,
-                                           bool &HasUserTransforms) {
-  LLVMContext &Ctx = Header->getContext();
-
-  Optional<bool> Enabled;
-  if (Attrs.UnrollEnable == LoopAttributes::Disable)
-    Enabled = false;
-  else if (Attrs.UnrollEnable == LoopAttributes::Full)
-    Enabled = true;
-
-  if (Enabled != true) {
-    SmallVector<Metadata *, 4> NewLoopProperties;
-    if (Enabled == false) {
-      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
-      NewLoopProperties.push_back(
-          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
-      LoopProperties = NewLoopProperties;
-    }
-    return createLoopDistributeMetadata(Attrs, LoopProperties,
-                                        HasUserTransforms);
-  }
-
-  SmallVector<Metadata *, 4> Args;
-  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
-  Args.push_back(TempNode.get());
-  Args.append(LoopProperties.begin(), LoopProperties.end());
-  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
-
-  // No follow-up: there is no loop after full unrolling.
-  // TODO: Warn if there are transformations after full unrolling.
-
-  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
-  LoopID->replaceOperandWith(0, LoopID);
-  HasUserTransforms = true;
-  return LoopID;
-}
-
-MDNode *LoopInfo::createMetadata(
-    const LoopAttributes &Attrs,
-    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
-    bool &HasUserTransforms) {
-  SmallVector<Metadata *, 3> LoopProperties;
-
-  // If we have a valid start debug location for the loop, add it.
-  if (StartLoc) {
-    LoopProperties.push_back(StartLoc.getAsMDNode());
-
-    // If we also have a valid end debug location for the loop, add it.
-    if (EndLoc)
-      LoopProperties.push_back(EndLoc.getAsMDNode());
-  }
-
-  assert(!!AccGroup == Attrs.IsParallel &&
-         "There must be an access group iff the loop is parallel");
-  if (Attrs.IsParallel) {
-    LLVMContext &Ctx = Header->getContext();
-    LoopProperties.push_back(MDNode::get(
-        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
-  }
-
-  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
-                        AdditionalLoopProperties.end());
-  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
-}
-
-LoopAttributes::LoopAttributes(bool IsParallel)
-    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
-      UnrollEnable(LoopAttributes::Unspecified),
-      UnrollAndJamEnable(LoopAttributes::Unspecified),
-      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
-      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
-      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
-      PipelineInitiationInterval(0) {}
-
-void LoopAttributes::clear() {
-  IsParallel = false;
-  VectorizeWidth = 0;
-  InterleaveCount = 0;
-  UnrollCount = 0;
-  UnrollAndJamCount = 0;
-  VectorizeEnable = LoopAttributes::Unspecified;
-  UnrollEnable = LoopAttributes::Unspecified;
-  UnrollAndJamEnable = LoopAttributes::Unspecified;
-  VectorizePredicateEnable = LoopAttributes::Unspecified;
-  DistributeEnable = LoopAttributes::Unspecified;
-  PipelineDisabled = false;
-  PipelineInitiationInterval = 0;
-}
-
-LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
-                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
-                   LoopInfo *Parent)
-    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
-      Parent(Parent) {
-
-  if (Attrs.IsParallel) {
-    // Create an access group for this loop.
-    LLVMContext &Ctx = Header->getContext();
-    AccGroup = MDNode::getDistinct(Ctx, {});
-  }
-
-  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
-      Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
-      Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
-      Attrs.PipelineInitiationInterval == 0 &&
-      Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
-      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
-      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
-      Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
-      Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
-      !EndLoc)
-    return;
-
-  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
+LoopInfo::LoopInfo(llvm::BasicBlock *Header, CGTransformedTree *TreeNode)
+    : Header(Header) {
+  if (TreeNode) {
+    LoopMD = TreeNode->makeLoopID(Header->getContext(), false);
+    AccGroup = TreeNode->getAccessGroupOrNull();
   }
-
-void LoopInfo::finish() {
-  // We did not annotate the loop body instructions because there are no
-  // attributes for this loop.
-  if (!TempLoopID)
-    return;
-
-  MDNode *LoopID;
-  LoopAttributes CurLoopAttr = Attrs;
-  LLVMContext &Ctx = Header->getContext();
-
-  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
-                 Parent->Attrs.UnrollAndJamCount != 0)) {
-    // Parent unroll-and-jams this loop.
-    // Split the transformations in those that happens before the unroll-and-jam
-    // and those after.
-
-    LoopAttributes BeforeJam, AfterJam;
-
-    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
-
-    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
-    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
-    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
-    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
-    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
-
-    switch (Attrs.UnrollEnable) {
-    case LoopAttributes::Unspecified:
-    case LoopAttributes::Disable:
-      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
-      AfterJam.UnrollEnable = Attrs.UnrollEnable;
-      break;
-    case LoopAttributes::Full:
-      BeforeJam.UnrollEnable = LoopAttributes::Full;
-      break;
-    case LoopAttributes::Enable:
-      AfterJam.UnrollEnable = LoopAttributes::Enable;
-      break;
-    }
 
-    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
-    AfterJam.UnrollCount = Attrs.UnrollCount;
-    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
-    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
-
-    // If this loop is subject of an unroll-and-jam by the parent loop, and has
-    // an unroll-and-jam annotation itself, we have to decide whether to first
-    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
-    // UnrollAndJam pass processes loops from inner to outer, so we apply the
-    // inner first.
-    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
-    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
-
-    // Set the inner followup metadata to process by the outer loop. Only
-    // consider the first inner loop.
-    if (!Parent->UnrollAndJamInnerFollowup) {
-      // Splitting the attributes into a BeforeJam and an AfterJam part will
-      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
-      // to be forwarded to the AfterJam part. We detect the situation here and
-      // add it manually.
-      SmallVector<Metadata *, 1> BeforeLoopProperties;
-      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
-          BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
-          BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
-        BeforeLoopProperties.push_back(
-            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
-
-      bool InnerFollowupHasTransform = false;
-      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
-                                             InnerFollowupHasTransform);
-      if (InnerFollowupHasTransform)
-        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
-    }
-
-    CurLoopAttr = BeforeJam;
-  }
-
-  bool HasUserTransforms = false;
-  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
-  TempLoopID->replaceAllUsesWith(LoopID);
-}
-
-void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
-                         const llvm::DebugLoc &EndLoc) {
-  Active.emplace_back(
-      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
-                   Active.empty() ? nullptr : Active.back().get()));
-  // Clear the attributes so nested loops do not inherit them.
-  StagedAttrs.clear();
-}
-
-void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
-                         ArrayRef<const clang::Attr *> Attrs,
-                         const llvm::DebugLoc &StartLoc,
-                         const llvm::DebugLoc &EndLoc) {
-
-  // Identify loop hint attributes from Attrs.
-  for (const auto *Attr : Attrs) {
-    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
-    const OpenCLUnrollHintAttr *OpenCLHint =
-        dyn_cast<OpenCLUnrollHintAttr>(Attr);
-
-    // Skip non loop hint attributes
-    if (!LH && !OpenCLHint) {
-      continue;
-    }
-
-    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
-    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
-    unsigned ValueInt = 1;
-    // Translate opencl_unroll_hint attribute argument to
-    // equivalent LoopHintAttr enums.
-    // OpenCL v2.0 s6.11.5:
-    // 0 - enable unroll (no argument).
-    // 1 - disable unroll.
-    // other positive integer n - unroll by n.
-    if (OpenCLHint) {
-      ValueInt = OpenCLHint->getUnrollHint();
-      if (ValueInt == 0) {
-        State = LoopHintAttr::Enable;
-      } else if (ValueInt != 1) {
-        Option = LoopHintAttr::UnrollCount;
-        State = LoopHintAttr::Numeric;
-      }
-    } else if (LH) {
-      auto *ValueExpr = LH->getValue();
-      if (ValueExpr) {
-        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
-        ValueInt = ValueAPS.getSExtValue();
-      }
-
-      Option = LH->getOption();
-      State = LH->getState();
-    }
-    switch (State) {
-    case LoopHintAttr::Disable:
-      switch (Option) {
-      case LoopHintAttr::Vectorize:
-        // Disable vectorization by specifying a width of 1.
-        setVectorizeWidth(1);
-        break;
-      case LoopHintAttr::Interleave:
-        // Disable interleaving by speciyfing a count of 1.
-        setInterleaveCount(1);
-        break;
-      case LoopHintAttr::Unroll:
-        setUnrollState(LoopAttributes::Disable);
-        break;
-      case LoopHintAttr::UnrollAndJam:
-        setUnrollAndJamState(LoopAttributes::Disable);
-        break;
-      case LoopHintAttr::VectorizePredicate:
-        setVectorizePredicateState(LoopAttributes::Disable);
-        break;
-      case LoopHintAttr::Distribute:
-        setDistributeState(false);
-        break;
-      case LoopHintAttr::PipelineDisabled:
-        setPipelineDisabled(true);
-        break;
-      case LoopHintAttr::UnrollCount:
-      case LoopHintAttr::UnrollAndJamCount:
-      case LoopHintAttr::VectorizeWidth:
-      case LoopHintAttr::InterleaveCount:
-      case LoopHintAttr::PipelineInitiationInterval:
-        llvm_unreachable("Options cannot be disabled.");
-        break;
-      }
-      break;
-    case LoopHintAttr::Enable:
-      switch (Option) {
-      case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
-        setVectorizeEnable(true);
-        break;
-      case LoopHintAttr::Unroll:
-        setUnrollState(LoopAttributes::Enable);
-        break;
-      case LoopHintAttr::UnrollAndJam:
-        setUnrollAndJamState(LoopAttributes::Enable);
-        break;
-      case LoopHintAttr::VectorizePredicate:
-        setVectorizePredicateState(LoopAttributes::Enable);
-        break;
-      case LoopHintAttr::Distribute:
-        setDistributeState(true);
-        break;
-      case LoopHintAttr::UnrollCount:
-      case LoopHintAttr::UnrollAndJamCount:
-      case LoopHintAttr::VectorizeWidth:
-      case LoopHintAttr::InterleaveCount:
-      case LoopHintAttr::PipelineDisabled:
-      case LoopHintAttr::PipelineInitiationInterval:
-        llvm_unreachable("Options cannot enabled.");
-        break;
-      }
-      break;
-    case LoopHintAttr::AssumeSafety:
-      switch (Option) {
-      case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
-        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
-        setParallel(true);
-        setVectorizeEnable(true);
-        break;
-      case LoopHintAttr::Unroll:
-      case LoopHintAttr::UnrollAndJam:
-      case LoopHintAttr::VectorizePredicate:
-      case LoopHintAttr::UnrollCount:
-      case LoopHintAttr::UnrollAndJamCount:
-      case LoopHintAttr::VectorizeWidth:
-      case LoopHintAttr::InterleaveCount:
-      case LoopHintAttr::Distribute:
-      case LoopHintAttr::PipelineDisabled:
-      case LoopHintAttr::PipelineInitiationInterval:
-        llvm_unreachable("Options cannot be used to assume mem safety.");
-        break;
-      }
-      break;
-    case LoopHintAttr::Full:
-      switch (Option) {
-      case LoopHintAttr::Unroll:
-        setUnrollState(LoopAttributes::Full);
-        break;
-      case LoopHintAttr::UnrollAndJam:
-        setUnrollAndJamState(LoopAttributes::Full);
-        break;
-      case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
-      case LoopHintAttr::UnrollCount:
-      case LoopHintAttr::UnrollAndJamCount:
-      case LoopHintAttr::VectorizeWidth:
-      case LoopHintAttr::InterleaveCount:
-      case LoopHintAttr::Distribute:
-      case LoopHintAttr::PipelineDisabled:
-      case LoopHintAttr::PipelineInitiationInterval:
-      case LoopHintAttr::VectorizePredicate:
-        llvm_unreachable("Options cannot be used with 'full' hint.");
-        break;
-      }
-      break;
-    case LoopHintAttr::Numeric:
-      switch (Option) {
-      case LoopHintAttr::VectorizeWidth:
-        setVectorizeWidth(ValueInt);
-        break;
-      case LoopHintAttr::InterleaveCount:
-        setInterleaveCount(ValueInt);
-        break;
-      case LoopHintAttr::UnrollCount:
-        setUnrollCount(ValueInt);
-        break;
-      case LoopHintAttr::UnrollAndJamCount:
-        setUnrollAndJamCount(ValueInt);
-        break;
-      case LoopHintAttr::PipelineInitiationInterval:
-        setPipelineInitiationInterval(ValueInt);
-        break;
-      case LoopHintAttr::Unroll:
-      case LoopHintAttr::UnrollAndJam:
-      case LoopHintAttr::VectorizePredicate:
-      case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
-      case LoopHintAttr::Distribute:
-      case LoopHintAttr::PipelineDisabled:
-        llvm_unreachable("Options cannot be assigned a value.");
-        break;
-      }
-      break;
-    }
-  }
+LoopInfoStack::~LoopInfoStack() {
+  for (auto N : AllNodes)
+    delete N;
+
+void LoopInfoStack::initBuild(clang::ASTContext &ASTCtx,
+                              llvm::LLVMContext &LLVMCtx, CGDebugInfo *DbgInfo,
+                              clang::Stmt *Body) {
+  CGTransformedTreeBuilder Builder(ASTCtx, LLVMCtx, AllNodes, DbgInfo);
+  TransformedStructure = Builder.computeTransformedStructure(Body, StmtToTree);
 
-  /// Stage the attributes.
-  push(Header, StartLoc, EndLoc);
+void LoopInfoStack::push(BasicBlock *Header, const clang::Stmt *LoopStmt) {
+  auto Loop = getAssociatedLoop(LoopStmt);
+  auto TreeNode = StmtToTree.lookup(Loop);
+  Active.emplace_back(new LoopInfo(Header, TreeNode));
 }
 
 void LoopInfoStack::pop() {
   assert(!Active.empty() && "No active loops to pop");
-  Active.back()->finish();
   Active.pop_back();
 }
 
 void LoopInfoStack::InsertHelper(Instruction *I) const {
   if (I->mayReadOrWriteMemory()) {
     SmallVector<Metadata *, 4> AccessGroups;
     for (const auto &AL : Active) {
       // Here we assume that every loop that has an access group is parallel.

lib/CodeGen/CGNonTrivialStruct.cpp

getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
     CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F);
     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
     FunctionDecl *FD = FunctionDecl::Create(
         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
         II, Ctx.getFunctionType(Ctx.VoidTy, llvm::None, {}), nullptr,
         SC_PrivateExtern, false, false);
     CodeGenFunction NewCGF(CGM);
+    NewCGF.LoopStack.initAsOutlined(NewCGF.LoopStack);
     setCGF(&NewCGF);
     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
 
     for (unsigned I = 0; I < N; ++I) {
       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
       Addrs[I] = Address(V, Alignments[I]);
     }
 

lib/CodeGen/CGOpenMPRuntime.h

protected:
   /// \param CodeGen Lambda codegen specific to an accelerator device.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
                                                 StringRef ParentName,
                                                 llvm::Function *&OutlinedFn,
                                                 llvm::Constant *&OutlinedFnID,
                                                 bool IsOffloadEntry,
-                                                const RegionCodeGenTy &CodeGen);
+                                                const RegionCodeGenTy &CodeGen,
+                                                const FunctionDecl *ParentFn);
 
   /// Emits code for OpenMP 'if' clause using specified \a CodeGen
   /// function. Here is the logic:
   /// if (Cond) {
   ///   ThenGen();
   /// } else {
   ///   ElseGen();
   /// }
private:
 
   /// Returns __tgt_bin_desc type.
   QualType getTgtBinaryDescriptorQTy();
 
   /// Start scanning from statement \a S and and emit all target regions
   /// found along the way.
   /// \param S Starting statement.
   /// \param ParentName Name of the function declaration that is being scanned.
-  void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
+  void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName,
+                                     const FunctionDecl *ParentFn);
 
   /// Build type kmp_routine_entry_t (if not built yet).
   void emitKmpRoutineEntryT(QualType KmpInt32Ty);
 
   /// Returns pointer to kmpc_micro type.
   llvm::Type *getKmpc_MicroPointerTy();
 
   /// Returns specified OpenMP runtime function.
public:
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
   virtual llvm::Function *emitParallelOutlinedFunction(
       const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn);
 
   /// Emits outlined function for the specified OpenMP teams directive
   /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
   virtual llvm::Function *emitTeamsOutlinedFunction(
       const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn);
 
   /// Emits outlined function for the OpenMP task directive \a D. This
   /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
   /// TaskT).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param PartIDVar Variable for partition id in the current OpenMP untied
   /// task region.
   /// \param TaskTVar Variable for task_t argument.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
   /// \param Tied true if task is generated for tied task, false otherwise.
   /// \param NumberOfParts Number of parts in untied task. Ignored for tied
   /// tasks.
   ///
   virtual llvm::Function *emitTaskOutlinedFunction(
       const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
       const VarDecl *PartIDVar, const VarDecl *TaskTVar,
       OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
-      bool Tied, unsigned &NumberOfParts);
+      bool Tied, unsigned &NumberOfParts, const FunctionDecl *ParentFn);
 
   /// Cleans up references to the objects in finished function.
   ///
   virtual void functionFinished(CodeGenFunction &CGF);
 
   /// Emits code for parallel or serial call of the \a OutlinedFn with
   /// variables captured in a record which address is stored in \a
   /// CapturedStruct.
public:
   /// \param CodeGen Code generation sequence for the \a D directive.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
                                           StringRef ParentName,
                                           llvm::Function *&OutlinedFn,
                                           llvm::Constant *&OutlinedFnID,
                                           bool IsOffloadEntry,
-                                          const RegionCodeGenTy &CodeGen);
+                                          const RegionCodeGenTy &CodeGen,
+                                          const FunctionDecl *ParentFn);
 
   /// Emit the target offloading code associated with \a D. The emitted
   /// code attempts offloading the execution to the device, an the event of
   /// a failure it executes the host version outlined in \a OutlinedFn.
   /// \param D Directive to emit.
   /// \param OutlinedFn Host version of the code to be offloaded.
   /// \param OutlinedFnID ID of host version of the code to be offloaded.
   /// \param IfCond Expression evaluated in if clause associated with the target
public:
   /// Emits outlined function for the specified OpenMP parallel directive
   /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
-  llvm::Function *
-  emitParallelOutlinedFunction(const OMPExecutableDirective &D,
-                               const VarDecl *ThreadIDVar,
-                               OpenMPDirectiveKind InnermostKind,
-                               const RegionCodeGenTy &CodeGen) override;
+  llvm::Function *emitParallelOutlinedFunction(
+      const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn) override;
 
   /// Emits outlined function for the specified OpenMP teams directive
   /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
-  llvm::Function *
-  emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
-                            const VarDecl *ThreadIDVar,
-                            OpenMPDirectiveKind InnermostKind,
-                            const RegionCodeGenTy &CodeGen) override;
+  llvm::Function *emitTeamsOutlinedFunction(
+      const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn) override;
 
   /// Emits outlined function for the OpenMP task directive \a D. This
   /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
   /// TaskT).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param PartIDVar Variable for partition id in the current OpenMP untied
   /// task region.
   /// \param TaskTVar Variable for task_t argument.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
   /// \param Tied true if task is generated for tied task, false otherwise.
   /// \param NumberOfParts Number of parts in untied task. Ignored for tied
   /// tasks.
   ///
   llvm::Function *emitTaskOutlinedFunction(
       const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
       const VarDecl *PartIDVar, const VarDecl *TaskTVar,
       OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
-      bool Tied, unsigned &NumberOfParts) override;
+      bool Tied, unsigned &NumberOfParts,
+      const FunctionDecl *ParentFn) override;
 
   /// Emits code for parallel or serial call of the \a OutlinedFn with
   /// variables captured in a record which address is stored in \a
   /// CapturedStruct.
   /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
   /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
   /// \param CapturedVars A pointer to the record with the references to
   /// variables used in \a OutlinedFn function.
public:
   /// \param CodeGen Code generation sequence for the \a D directive.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
                                   StringRef ParentName,
                                   llvm::Function *&OutlinedFn,
                                   llvm::Constant *&OutlinedFnID,
                                   bool IsOffloadEntry,
-                                  const RegionCodeGenTy &CodeGen) override;
+                                  const RegionCodeGenTy &CodeGen,
+                                  const FunctionDecl *ParentFn) override;
 
   /// Emit the target offloading code associated with \a D. The emitted
   /// code attempts offloading the execution to the device, an the event of
   /// a failure it executes the host version outlined in \a OutlinedFn.
   /// \param D Directive to emit.
   /// \param OutlinedFn Host version of the code to be offloaded.
   /// \param OutlinedFnID ID of host version of the code to be offloaded.
   /// \param IfCond Expression evaluated in if clause associated with the target

lib/CodeGen/CGOpenMPRuntime.cpp

if (I != UDRMap.end())
     return I->second;
   emitUserDefinedReduction(/*CGF=*/nullptr, D);
   return UDRMap.lookup(D);
 }
 
 static llvm::Function *emitParallelOrTeamsOutlinedFunction(
     CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
     const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
-    const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
+    const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   assert(ThreadIDVar->getType()->isPointerType() &&
          "thread id variable must be of type kmp_int32 *");
-  CodeGenFunction CGF(CGM, true);
+  CodeGenFunction CGF(CGM, true, ParentFn);
   bool HasCancel = false;
   if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
     HasCancel = OPD->hasCancel();
   else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
     HasCancel = OPSD->hasCancel();
   else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
     HasCancel = OPFD->hasCancel();
   else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
static llvm::Function *emitParallelOrTeamsOutlinedFunction(
   CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
                                     HasCancel, OutlinedHelperName);
   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
   return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
 }
 
 llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
   return emitParallelOrTeamsOutlinedFunction(
-      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
+      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen,
+      ParentFn);
 }
 
 llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
   return emitParallelOrTeamsOutlinedFunction(
-      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
+      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen,
+      ParentFn);
 }
 
 llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
-    bool Tied, unsigned &NumberOfParts) {
+    bool Tied, unsigned &NumberOfParts, const FunctionDecl *ParentFn) {
   auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
                                               PrePostActionTy &) {
     llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
     llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
     llvm::Value *TaskArgs[] = {
         UpLoc, ThreadID,
         CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
                                     TaskTVar->getType()->castAs<PointerType>())
             .getPointer()};
     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
   };
   CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
                                                             UntiedCodeGen);
   CodeGen.setAction(Action);
   assert(!ThreadIDVar->getType()->isPointerType() &&
          "thread id variable must be of type kmp_int32 for tasks");
   const OpenMPDirectiveKind Region =
       isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
                                                       : OMPD_task;
   const CapturedStmt *CS = D.getCapturedStmt(Region);
   const auto *TD = dyn_cast<OMPTaskDirective>(&D);
-  CodeGenFunction CGF(CGM, true);
+  CodeGenFunction CGF(CGM, true, ParentFn);
   CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
                                         InnermostKind,
                                         TD ? TD->hasCancel() : false, Action);
   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
   llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
   if (!Tied)
     NumberOfParts = Action.getNumberOfParts();
   return Res;
if (IfCond) {
       ThenRCG(CGF);
     }
   }
 }
 
 void CGOpenMPRuntime::emitTargetOutlinedFunction(
     const OMPExecutableDirective &D, StringRef ParentName,
     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
-    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
+    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   assert(!ParentName.empty() && "Invalid target region parent name!");
   HasEmittedTargetRegion = true;
   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
-                                   IsOffloadEntry, CodeGen);
+                                   IsOffloadEntry, CodeGen, ParentFn);
 }
 
 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
     const OMPExecutableDirective &D, StringRef ParentName,
     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
-    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
+    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   // Create a unique name for the entry function using the source location
   // information of the current target region. The name will be something like:
   //
   // __omp_offloading_DD_FFFF_PP_lBB
   //
   // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
   // mangled name of the function that encloses the target region and BB is the
   // line number of the target region.
 
   unsigned DeviceID;
   unsigned FileID;
   unsigned Line;
   getTargetEntryUniqueInfo(CGM.getContext(), D.getBeginLoc(), DeviceID, FileID,
                            Line);
   SmallString<64> EntryFnName;
   {
     llvm::raw_svector_ostream OS(EntryFnName);
     OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
        << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
   }
 
   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
 
-  CodeGenFunction CGF(CGM, true);
+  CodeGenFunction CGF(CGM, true, ParentFn);
   CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
 
   OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
 
   // If this target outline function is not an offload entry, we don't need to
   // register it.
   if (!IsOffloadEntry)
if (IfCond) {
       ThenRCG(CGF);
     }
   } else {
     RegionCodeGenTy ElseRCG(TargetElseGen);
     ElseRCG(CGF);
   }
 }
 
-void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
-                                                    StringRef ParentName) {
+void CGOpenMPRuntime::scanForTargetRegionsFunctions(
+    const Stmt *S, StringRef ParentName, const FunctionDecl *ParentFn) {
   if (!S)
     return;
 
   // Codegen OMP target directives that offload compute to the device.
   bool RequiresDeviceCodegen =
       isa<OMPExecutableDirective>(S) &&
       isOpenMPTargetExecutionDirective(
           cast<OMPExecutableDirective>(S)->getDirectiveKind());
if (RequiresDeviceCodegen) {
     // Is this a target region that should not be emitted as an entry point? If
     // so just signal we are done with this target region.
     if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
                                                             ParentName, Line))
       return;
 
     switch (E.getDirectiveKind()) {
     case OMPD_target:
-      CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
-                                                   cast<OMPTargetDirective>(E));
+      CodeGenFunction::EmitOMPTargetDeviceFunction(
+          CGM, ParentName, cast<OMPTargetDirective>(E), ParentFn);
       break;
     case OMPD_target_parallel:
       CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
-          CGM, ParentName, cast<OMPTargetParallelDirective>(E));
+          CGM, ParentName, cast<OMPTargetParallelDirective>(E), ParentFn);
       break;
     case OMPD_target_teams:
       CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
-          CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
+          CGM, ParentName, cast<OMPTargetTeamsDirective>(E), ParentFn);
       break;
     case OMPD_target_teams_distribute:
       CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
-          CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
+          CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E),
+          ParentFn);
       break;
     case OMPD_target_teams_distribute_simd:
       CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
-          CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
+          CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E),
+          ParentFn);
       break;
     case OMPD_target_parallel_for:
       CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
-          CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
+          CGM, ParentName, cast<OMPTargetParallelForDirective>(E), ParentFn);
       break;
     case OMPD_target_parallel_for_simd:
       CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
-          CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
+          CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E),
+          ParentFn);
       break;
     case OMPD_target_simd:
       CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
-          CGM, ParentName, cast<OMPTargetSimdDirective>(E));
+          CGM, ParentName, cast<OMPTargetSimdDirective>(E), ParentFn);
       break;
     case OMPD_target_teams_distribute_parallel_for:
       CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
           CGM, ParentName,
-          cast<OMPTargetTeamsDistributeParallelForDirective>(E));
+          cast<OMPTargetTeamsDistributeParallelForDirective>(E), ParentFn);
       break;
     case OMPD_target_teams_distribute_parallel_for_simd:
       CodeGenFunction::
           EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
               CGM, ParentName,
-              cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
+              cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E),
+              ParentFn);
       break;
     case OMPD_parallel:
     case OMPD_for:
     case OMPD_parallel_for:
     case OMPD_parallel_sections:
     case OMPD_for_simd:
     case OMPD_parallel_for_simd:
     case OMPD_cancel:
if (RequiresDeviceCodegen) {
     return;
   }
 
   if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
     if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
       return;
 
     scanForTargetRegionsFunctions(
-        E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
+        E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName, ParentFn);
     return;
   }
 
   // If this is a lambda function, look into its body.
   if (const auto *L = dyn_cast<LambdaExpr>(S))
     S = L->getBody();
 
   // Keep looking for target regions recursively.
   for (const Stmt *II : S->children())
-    scanForTargetRegionsFunctions(II, ParentName);
+    scanForTargetRegionsFunctions(II, ParentName, ParentFn);
 }
 
 bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
   // If emitting code for the host, we do not process FD here. Instead we do
   // the normal code generation.
   if (!CGM.getLangOpts().OpenMPIsDevice) {
     if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) {
       Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
           OMPDeclareTargetDeclAttr::getDeviceType(FD);
       // Do not emit device_type(nohost) functions for the host.
       if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
         return true;
     }
     return false;
   }
 
   const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());
   StringRef Name = CGM.getMangledName(GD);
   // Try to detect target regions in the function.
   if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {
-    scanForTargetRegionsFunctions(FD->getBody(), Name);
+    scanForTargetRegionsFunctions(FD->getBody(), Name, FD);
     Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
         OMPDeclareTargetDeclAttr::getDeviceType(FD);
     // Do not emit device_type(nohost) functions for the host.
     if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_Host)
       return true;
   }
 
   // Do not to emit function if it is not marked as declare target.
   return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
          AlreadyEmittedTargetFunctions.count(Name) == 0;
 }
 
 bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
   if (!CGM.getLangOpts().OpenMPIsDevice)
     return false;
 
   // Check if there are Ctors/Dtors in this declaration and look for target
   // regions in it. We use the complete variant to produce the kernel name
   // mangling.
   QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
   if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
     for (const CXXConstructorDecl *Ctor : RD->ctors()) {
       StringRef ParentName =
           CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
-      scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
+      scanForTargetRegionsFunctions(Ctor->getBody(), ParentName, Ctor);
     }
     if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
       StringRef ParentName =
           CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
-      scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
+      scanForTargetRegionsFunctions(Dtor->getBody(), ParentName, Dtor);
     }
   }
 
   // Do not to emit variable if it is not marked as declare target.
   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
           cast<VarDecl>(GD.getDecl()));
   if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
if (tryEmitDeclareVariant(NewGD, GD, Orig, IsForDefinition)) {
     return true;
   }
   DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD)));
   return true;
 }
 
 llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
-    bool Tied, unsigned &NumberOfParts) {
+    bool Tied, unsigned &NumberOfParts, const FunctionDecl *ParentFn) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
                                            SourceLocation Loc,
                                            llvm::Function *OutlinedFn,
                                            ArrayRef<llvm::Value *> CapturedVars,
                                            const Expr *IfCond) {
void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
                                          SourceLocation Loc, const Expr *IfCond,
                                          OpenMPDirectiveKind CancelRegion) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
     const OMPExecutableDirective &D, StringRef ParentName,
     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
-    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
+    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 void CGOpenMPSIMDRuntime::emitTargetCall(
     CodeGenFunction &CGF, const OMPExecutableDirective &D,
     llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
     const Expr *Device,
     llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,

lib/CodeGen/CGOpenMPRuntimeNVPTX.h

private:
   /// \param OutlinedFn Outlined function value to be defined by this call.
   /// \param OutlinedFnID Outlined function ID value to be defined by this call.
   /// \param IsOffloadEntry True if the outlined function is an offload entry.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
                          llvm::Function *&OutlinedFn,
                          llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
-                         const RegionCodeGenTy &CodeGen);
+                         const RegionCodeGenTy &CodeGen,
+                         const FunctionDecl *ParentFn);
 
   /// Emit outlined function specialized for the Single Program
   /// Multiple Data programming model for applicable target directives on the
   /// NVPTX device.
   /// \param D Directive to emit.
   /// \param ParentName Name of the function that encloses the target region.
   /// \param OutlinedFn Outlined function value to be defined by this call.
   /// \param OutlinedFnID Outlined function ID value to be defined by this call.
   /// \param IsOffloadEntry True if the outlined function is an offload entry.
   /// \param CodeGen Object containing the target statements.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
                       llvm::Function *&OutlinedFn,
                       llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
-                      const RegionCodeGenTy &CodeGen);
+                      const RegionCodeGenTy &CodeGen,
+                      const FunctionDecl *ParentFn);
 
   /// Emit outlined function for 'target' directive on the NVPTX
   /// device.
   /// \param D Directive to emit.
   /// \param ParentName Name of the function that encloses the target region.
   /// \param OutlinedFn Outlined function value to be defined by this call.
   /// \param OutlinedFnID Outlined function ID value to be defined by this call.
   /// \param IsOffloadEntry True if the outlined function is an offload entry.
   /// An outlined function may not be an entry if, e.g. the if clause always
   /// evaluates to false.
   void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
                                   StringRef ParentName,
                                   llvm::Function *&OutlinedFn,
                                   llvm::Constant *&OutlinedFnID,
                                   bool IsOffloadEntry,
-                                  const RegionCodeGenTy &CodeGen) override;
+                                  const RegionCodeGenTy &CodeGen,
+                                  const FunctionDecl *ParentFn) override;
 
   /// Emits code for parallel or serial call of the \a OutlinedFn with
   /// variables captured in a record which address is stored in \a
   /// CapturedStruct.
   /// This call is for the Non-SPMD Execution Mode.
   /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
   /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
   /// \param CapturedVars A pointer to the record with the references to
public:
   //  directive.
   /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
-  llvm::Function *
-  emitParallelOutlinedFunction(const OMPExecutableDirective &D,
-                               const VarDecl *ThreadIDVar,
-                               OpenMPDirectiveKind InnermostKind,
-                               const RegionCodeGenTy &CodeGen) override;
+  llvm::Function *emitParallelOutlinedFunction(
+      const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn) override;
 
   /// Emits inlined function for the specified OpenMP teams
   //  directive.
   /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
   /// kmp_int32 BoundID, struct context_vars*).
   /// \param D OpenMP directive.
   /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
   /// \param InnermostKind Kind of innermost directive (for simple directives it
   /// is a directive itself, for combined - its innermost directive).
   /// \param CodeGen Code generation sequence for the \a D directive.
-  llvm::Function *
-  emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
-                            const VarDecl *ThreadIDVar,
-                            OpenMPDirectiveKind InnermostKind,
-                            const RegionCodeGenTy &CodeGen) override;
+  llvm::Function *emitTeamsOutlinedFunction(
+      const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+      OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+      const FunctionDecl *ParentFn) override;
 
   /// Emits code for teams call of the \a OutlinedFn with
   /// variables captured in a record which address is stored in \a
   /// CapturedStruct.
   /// \param OutlinedFn Outlined function to be run by team masters. Type of
   /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
   /// \param CapturedVars A pointer to the record with the references to
   /// variables used in \a OutlinedFn function.

lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp

llvm_unreachable(
       "Unknown programming model for OpenMP directive on NVPTX target.");
 }
 
 void CGOpenMPRuntimeNVPTX::emitNonSPMDKernel(const OMPExecutableDirective &D,
                                              StringRef ParentName,
                                              llvm::Function *&OutlinedFn,
                                              llvm::Constant *&OutlinedFnID,
                                              bool IsOffloadEntry,
-                                             const RegionCodeGenTy &CodeGen) {
+                                             const RegionCodeGenTy &CodeGen,
+                                             const FunctionDecl *ParentFn) {
   ExecutionRuntimeModesRAII ModeRAII(CurrentExecutionMode);
   EntryFunctionState EST;
   WorkerFunctionState WST(CGM, D.getBeginLoc());
   Work.clear();
   WrapperFunctionsMap.clear();
 
   // Emit target region as a standalone region.
   class NVPTXPrePostActionTy : public PrePostActionTy {
KernelStaticGlobalized = new llvm::GlobalVariable(
         CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
         llvm::GlobalValue::InternalLinkage,
         llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
         "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
         llvm::GlobalValue::NotThreadLocal,
         CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
   }
   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
-                                   IsOffloadEntry, CodeGen);
+                                   IsOffloadEntry, CodeGen, ParentFn);
   IsInTTDRegion = false;
 
   // Now change the name of the worker function to correspond to this target
   // region's entry function.
   WST.WorkerFn->setName(Twine(OutlinedFn->getName(), "_worker"));
 
   // Create the worker function
   emitWorkerFunction(WST);
void CGOpenMPRuntimeNVPTX::emitNonSPMDEntryFooter(CodeGenFunction &CGF,
   EST.ExitBB = nullptr;
 }
 
 void CGOpenMPRuntimeNVPTX::emitSPMDKernel(const OMPExecutableDirective &D,
                                           StringRef ParentName,
                                           llvm::Function *&OutlinedFn,
                                           llvm::Constant *&OutlinedFnID,
                                           bool IsOffloadEntry,
-                                          const RegionCodeGenTy &CodeGen) {
+                                          const RegionCodeGenTy &CodeGen,
+                                          const FunctionDecl *ParentFn) {
   ExecutionRuntimeModesRAII ModeRAII(
       CurrentExecutionMode, RequiresFullRuntime,
       CGM.getLangOpts().OpenMPCUDAForceFullRuntime ||
           !supportsLightweightRuntime(CGM.getContext(), D));
   EntryFunctionState EST;
 
   // Emit target region as a standalone region.
   class NVPTXPrePostActionTy : public PrePostActionTy {
KernelStaticGlobalized = new llvm::GlobalVariable(
         CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
         llvm::GlobalValue::InternalLinkage,
         llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
         "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
         llvm::GlobalValue::NotThreadLocal,
         CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
   }
   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
-                                   IsOffloadEntry, CodeGen);
+                                   IsOffloadEntry, CodeGen, ParentFn);
   IsInTTDRegion = false;
 }
 
 void CGOpenMPRuntimeNVPTX::emitSPMDEntryHeader(
     CodeGenFunction &CGF, EntryFunctionState &EST,
     const OMPExecutableDirective &D) {
   CGBuilderTy &Bld = CGF.Builder;
 
llvm::Metadata *MDVals[] = {
           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
   // Append metadata to nvvm.annotations
   MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
 }
 
 void CGOpenMPRuntimeNVPTX::emitTargetOutlinedFunction(
     const OMPExecutableDirective &D, StringRef ParentName,
     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
-    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
+    bool IsOffloadEntry, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   if (!IsOffloadEntry) // Nothing to do.
     return;
 
   assert(!ParentName.empty() && "Invalid target region parent name!");
 
   bool Mode = supportsSPMDExecutionMode(CGM.getContext(), D);
   if (Mode)
     emitSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
-                   CodeGen);
+                   CodeGen, ParentFn);
   else
     emitNonSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
-                      CodeGen);
+                      CodeGen, ParentFn);
 
   setPropertyExecutionMode(CGM, OutlinedFn->getName(), Mode);
 }
 
 namespace {
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
 /// Enum for accesseing the reserved_2 field of the ident_t struct.
 enum ModeFlagsTy : unsigned {

 
 void CGOpenMPRuntimeNVPTX::emitNumTeamsClause(CodeGenFunction &CGF,
                                               const Expr *NumTeams,
                                               const Expr *ThreadLimit,
                                               SourceLocation Loc) {}
 
 llvm::Function *CGOpenMPRuntimeNVPTX::emitParallelOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   // Emit target region as a standalone region.
   class NVPTXPrePostActionTy : public PrePostActionTy {
     bool &IsInParallelRegion;
     bool PrevIsInParallelRegion;
 
   public:
     NVPTXPrePostActionTy(bool &IsInParallelRegion)
         : IsInParallelRegion(IsInParallelRegion) {}
     void Enter(CodeGenFunction &CGF) override {
       PrevIsInParallelRegion = IsInParallelRegion;
       IsInParallelRegion = true;
     }
     void Exit(CodeGenFunction &CGF) override {
       IsInParallelRegion = PrevIsInParallelRegion;
     }
   } Action(IsInParallelRegion);
   CodeGen.setAction(Action);
   bool PrevIsInTTDRegion = IsInTTDRegion;
   IsInTTDRegion = false;
   bool PrevIsInTargetMasterThreadRegion = IsInTargetMasterThreadRegion;
   IsInTargetMasterThreadRegion = false;
   auto *OutlinedFun =
       cast<llvm::Function>(CGOpenMPRuntime::emitParallelOutlinedFunction(
-          D, ThreadIDVar, InnermostKind, CodeGen));
+          D, ThreadIDVar, InnermostKind, CodeGen, ParentFn));
   if (CGM.getLangOpts().Optimize) {
     OutlinedFun->removeFnAttr(llvm::Attribute::NoInline);
     OutlinedFun->removeFnAttr(llvm::Attribute::OptimizeNone);
     OutlinedFun->addFnAttr(llvm::Attribute::AlwaysInline);
   }
   IsInTargetMasterThreadRegion = PrevIsInTargetMasterThreadRegion;
   IsInTTDRegion = PrevIsInTTDRegion;
   if (getExecutionMode() != CGOpenMPRuntimeNVPTX::EM_SPMD &&
getTeamsReductionVars(ASTContext &Ctx, const OMPExecutableDirective &D,
   for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
     for (const Expr *E : C->privates())
       Vars.push_back(getPrivateItem(E));
   }
 }
 
 llvm::Function *CGOpenMPRuntimeNVPTX::emitTeamsOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
-    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
+    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
+    const FunctionDecl *ParentFn) {
   SourceLocation Loc = D.getBeginLoc();
 
   const RecordDecl *GlobalizedRD = nullptr;
   llvm::SmallVector<const ValueDecl *, 4> LastPrivatesReductions;
   llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
   // Globalize team reductions variable unconditionally in all modes.
   if (getExecutionMode() != CGOpenMPRuntimeNVPTX::EM_SPMD)
     getTeamsReductionVars(CGM.getContext(), D, LastPrivatesReductions);
public:
     }
     void Exit(CodeGenFunction &CGF) override {
       static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime())
           .emitGenericVarsEpilog(CGF);
     }
   } Action(Loc, GlobalizedRD, MappedDeclsFields);
   CodeGen.setAction(Action);
   llvm::Function *OutlinedFun = CGOpenMPRuntime::emitTeamsOutlinedFunction(
-      D, ThreadIDVar, InnermostKind, CodeGen);
+      D, ThreadIDVar, InnermostKind, CodeGen, ParentFn);
   if (CGM.getLangOpts().Optimize) {
     OutlinedFun->removeFnAttr(llvm::Attribute::NoInline);
     OutlinedFun->removeFnAttr(llvm::Attribute::OptimizeNone);
     OutlinedFun->addFnAttr(llvm::Attribute::AlwaysInline);
   }
 
   return OutlinedFun;
 }

lib/CodeGen/CGStmt.cpp

case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
     EmitOMPTargetTeamsDistributeParallelForSimdDirective(
         cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
     break;
   case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
     EmitOMPTargetTeamsDistributeSimdDirective(
         cast<OMPTargetTeamsDistributeSimdDirective>(*S));
     break;
   case Stmt::TransformExecutableDirectiveClass:
-    llvm_unreachable("not implemented");
+    EmitTransformExecutableDirective(cast<TransformExecutableDirective>(*S));
     break;
   }
 }
 
 bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) {
   switch (S->getStmtClass()) {
   default: return false;
   case Stmt::NullStmtClass: break;

 
 void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
                                     ArrayRef<const Attr *> WhileAttrs) {
   // Emit the header for the loop, which will also become
   // the continue target.
   JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
   EmitBlock(LoopHeader.getBlock());
 
-  const SourceRange &R = S.getSourceRange();
-  LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), WhileAttrs,
-                 SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(LoopHeader.getBlock(), &S);
 
   // Create an exit block for when the condition fails, which will
   // also become the break target.
   JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
 
   // Store the blocks to use for break and continue.
   BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
 
void CodeGenFunction::EmitDoStmt(const DoStmt &S,
   EmitBlockWithFallThrough(LoopBody, &S);
   {
     RunCleanupsScope BodyScope(*this);
     EmitStmt(S.getBody());
   }
 
   EmitBlock(LoopCond.getBlock());
 
-  const SourceRange &R = S.getSourceRange();
-  LoopStack.push(LoopBody, CGM.getContext(), DoAttrs,
-                 SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(LoopBody, &S);
 
   // C99 6.8.5.2: "The evaluation of the controlling expression takes place
   // after each execution of the loop body."
 
   // Evaluate the conditional in the while header.
   // C99 6.8.5p2/p4: The first substatement is executed if the expression
   // compares unequal to 0.  The condition must be a scalar type.
   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
void CodeGenFunction::EmitForStmt(const ForStmt &S,
 
   // Start the loop with a block that tests the condition.
   // If there's an increment, the continue scope will be overwritten
   // later.
   JumpDest Continue = getJumpDestInCurrentScope("for.cond");
   llvm::BasicBlock *CondBlock = Continue.getBlock();
   EmitBlock(CondBlock);
 
-  const SourceRange &R = S.getSourceRange();
-  LoopStack.push(CondBlock, CGM.getContext(), ForAttrs,
-                 SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(CondBlock, &S);
 
   // If the for loop doesn't have an increment we can just use the
   // condition as the continue block.  Otherwise we'll need to create
   // a block for it (in the current scope, i.e. in the scope of the
   // condition), and that we will become our continue block.
   if (S.getInc())
     Continue = getJumpDestInCurrentScope("for.inc");
 
CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   EmitStmt(S.getEndStmt());
 
   // Start the loop with a block that tests the condition.
   // If there's an increment, the continue scope will be overwritten
   // later.
   llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
   EmitBlock(CondBlock);
 
-  const SourceRange &R = S.getSourceRange();
-  LoopStack.push(CondBlock, CGM.getContext(), ForAttrs,
-                 SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(CondBlock, &S);
 
   // If there are any cleanups between here and the loop-exit scope,
   // create a block to stage a loop exit along.
   llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
   if (ForScope.requiresCleanups())
     ExitBlock = createBasicBlock("for.cond.cleanup");
 
   // The loop body, consisting of the specified body and the loop variable.
llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
                            CapturedStmtInfo->getHelperName(), &CGM.getModule());
   CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
   if (CD->isNothrow())
     F->addFnAttr(llvm::Attribute::NoUnwind);
 
   // Generate the function.
   StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
                 CD->getBody()->getBeginLoc());
+  HandleCodeTransformations(&S);
   // Set the context parameter in CapturedStmtInfo.
   Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
   CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
 
   // Initialize variable-length arrays.
   LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
                                            Ctx.getTagDeclType(RD));
   for (auto *FD : RD->fields()) {

lib/CodeGen/CGStmtOpenMP.cpp

static llvm::Function *emitOutlinedFunctionPrologue(
   CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
   if (CD->isNothrow())
     F->setDoesNotThrow();
   F->setDoesNotRecurse();
 
   // Generate the function.
   CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
                     FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
+  CGF.HandleCodeTransformations(CD->getBody());
   unsigned Cnt = CD->getContextParamPosition();
   I = FO.S->captures().begin();
   for (const FieldDecl *FD : RD->fields()) {
     // Do not map arguments if we emit function with non-original types.
     Address LocalAddr(Address::invalid());
     if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
       LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
                                                              TargetArgs[Cnt]);
CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
   (void)LocalScope.ForceCleanup();
   FinishFunction(CD->getBodyRBrace());
   if (!NeedWrapperFunction)
     return F;
 
   FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
                             /*RegisterCastedArgsOnly=*/true,
                             CapturedStmtInfo->getHelperName());
-  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
+  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true, ParentFn);
   WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
   Args.clear();
   LocalAddrs.clear();
   VLASizes.clear();
   llvm::Function *WrapperF =
       emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
                                    WrapperCGF.CXXThisValue, WrapperFO);
   llvm::SmallVector<llvm::Value *, 4> CallArgs;

 
 static void emitCommonOMPParallelDirective(
     CodeGenFunction &CGF, const OMPExecutableDirective &S,
     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
     const CodeGenBoundParametersTy &CodeGenBoundParameters) {
   const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
   llvm::Function *OutlinedFn =
       CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
-          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
+          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen,
+          CGF.getParentFn());
   if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
     CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
     llvm::Value *NumThreads =
         CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
                            /*IgnoreResultAssign=*/true);
     CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
         CGF, NumThreads, NumThreadsClause->getBeginLoc());
   }
void CodeGenFunction::EmitOMPInnerLoop(
     const Expr *IncExpr,
     const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
     const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
   auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
 
   // Start the loop with a block that tests the condition.
   auto CondBlock = createBasicBlock("omp.inner.for.cond");
   EmitBlock(CondBlock);
-  const SourceRange R = S.getSourceRange();
-  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(CondBlock, &S);
 
   // If there are any cleanups between here and the loop-exit scope,
   // create a block to stage a loop exit along.
   llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
   if (RequiresCleanup)
     ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
 
   llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
for (const Expr *E : C->varlists()) {
       } else {
         EmitVarDecl(*PrivateVD);
       }
       ++CurPrivate;
     }
   }
 }
 
-static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
-                                     const OMPExecutableDirective &D,
-                                     bool IsMonotonic) {
-  if (!CGF.HaveInsertPoint())
-    return;
-  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
-    RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
-                                 /*ignoreResult=*/true);
-    auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
-    CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
-    // In presence of finite 'safelen', it may be unsafe to mark all
-    // the memory instructions parallel, because loop-carried
-    // dependences of 'safelen' iterations are possible.
-    if (!IsMonotonic)
-      CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
-  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
-    RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
-                                 /*ignoreResult=*/true);
-    auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
-    CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
-    // In presence of finite 'safelen', it may be unsafe to mark all
-    // the memory instructions parallel, because loop-carried
-    // dependences of 'safelen' iterations are possible.
-    CGF.LoopStack.setParallel(/*Enable=*/false);
-  }
-}
-
 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
                                       bool IsMonotonic) {
-  // Walk clauses and process safelen/lastprivate.
-  LoopStack.setParallel(!IsMonotonic);
-  LoopStack.setVectorizeEnable();
-  emitSimdlenSafelenClause(*this, D, IsMonotonic);
 }
 
 void CodeGenFunction::EmitOMPSimdFinal(
     const OMPLoopDirective &D,
     const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
   if (!HaveInsertPoint())
     return;
   llvm::BasicBlock *DoneBB = nullptr;
void CodeGenFunction::EmitOMPOuterLoop(
   const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
   const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
 
   JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
 
   // Start the loop with a block that tests the condition.
   llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
   EmitBlock(CondBlock);
-  const SourceRange R = S.getSourceRange();
-  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
-                 SourceLocToDebugLoc(R.getEnd()));
+  LoopStack.push(CondBlock, &S);
 
   llvm::Value *BoolCondVal = nullptr;
   if (!DynamicOrOrdered) {
     // UB = min(UB, GlobalUB) or
     // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
     // 'distribute parallel for')
     EmitIgnoredExpr(LoopArgs.EUB);
     // IV = LB
void CodeGenFunction::EmitOMPOuterLoop(
   // LB for loop condition and emitted it above).
   if (DynamicOrOrdered)
     EmitIgnoredExpr(LoopArgs.Init);
 
   // Create a block for the increment.
   JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
 
-  // Generate !llvm.loop.parallel metadata for loads and stores for loops
-  // with dynamic/guided scheduling and without ordered clause.
-  if (!isOpenMPSimdDirective(S.getDirectiveKind()))
-    LoopStack.setParallel(!IsMonotonic);
-  else
+  if (isOpenMPSimdDirective(S.getDirectiveKind()))
     EmitOMPSimdInit(S, IsMonotonic);
 
   SourceLocation Loc = S.getBeginLoc();
 
   // when 'distribute' is not combined with a 'for':
   // while (idx <= UB) { BODY; ++idx; }
   // when 'distribute' is combined with a 'for'
   // (e.g. 'distribute parallel for')
void CodeGenFunction::EmitOMPDistributeSimdDirective(
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
     CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
   };
   OMPLexicalScope Scope(*this, S, OMPD_unknown);
   CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
 }
 
 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
-    CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
+    CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S,
+    const FunctionDecl *ParentFn) {
   // Emit SPMD target parallel for region as a standalone region.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitOMPSimdRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetSimdDirective(
     const OMPTargetSimdDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitOMPSimdRegion(CGF, S, Action);
   };
auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
     }
     (void)InRedScope.Privatize();
 
     Action.Enter(CGF);
     BodyGen(CGF);
   };
   llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
       S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
-      Data.NumberOfParts);
+      Data.NumberOfParts, ParentFn);
   OMPLexicalScope Scope(*this, S, llvm::None,
                         !isOpenMPParallelDirective(S.getDirectiveKind()));
   TaskGen(*this, OutlinedFn, Data);
 }
 
 static ImplicitParamDecl *
 createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
                                   QualType Ty, CapturedDecl *CD,
auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
     }
 
     Action.Enter(CGF);
     OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
     BodyGen(CGF);
   };
   llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
       S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
-      Data.NumberOfParts);
+      Data.NumberOfParts, ParentFn);
   llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
   IntegerLiteral IfCond(getContext(), TrueOrFalse,
                         getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
                         SourceLocation());
 
   CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
                                       SharedsTy, CapturedStruct, &IfCond, Data);
 }
void CodeGenFunction::EmitOMPDistributeDirective(
     const OMPDistributeDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
     CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
   };
   OMPLexicalScope Scope(*this, S, OMPD_unknown);
   CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
 }
 
-static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
-                                                   const CapturedStmt *S) {
-  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
+static llvm::Function *
+emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S,
+                            const FunctionDecl *ParentFn) {
+  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true, ParentFn);
   CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
   CGF.CapturedStmtInfo = &CapStmtInfo;
   llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
   Fn->setDoesNotRecurse();
   return Fn;
 }
 
 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
   if (S.hasClausesOfKind<OMPDependClause>()) {
     assert(!S.getAssociatedStmt() &&
            "No associated statement must be in ordered depend construct.");
     for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
       CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
     return;
   }
   const auto *C = S.getSingleClause<OMPSIMDClause>();
   auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
                                  PrePostActionTy &Action) {
     const CapturedStmt *CS = S.getInnermostCapturedStmt();
     if (C) {
       llvm::SmallVector<llvm::Value *, 16> CapturedVars;
       CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
-      llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
+      llvm::Function *OutlinedFn =
+          emitOutlinedOrderedFunction(CGM, CS, ParentFn);
       CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
                                                       OutlinedFn, CapturedVars);
     } else {
       Action.Enter(CGF);
       CGF.EmitStmt(CS->getCapturedStmt());
     }
   };
   OMPLexicalScope Scope(*this, S, OMPD_unknown);
if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
     ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
   else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
     ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
   else
     ParentName =
         CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
 
   // Emit target region as a standalone region.
-  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
-                                                    IsOffloadEntry, CodeGen);
+  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
+      S, ParentName, Fn, FnID, IsOffloadEntry, CodeGen, CGF.getParentFn());
   OMPLexicalScope Scope(CGF, S, OMPD_task);
   auto &&SizeEmitter =
       [IsOffloadEntry](CodeGenFunction &CGF,
                        const OMPLoopDirective &D) -> llvm::Value * {
     if (IsOffloadEntry) {
       OMPLoopScope(CGF, D);
       // Emit calculation of the iterations count.
       llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
   CGF.EmitOMPPrivateClause(S, PrivateScope);
   (void)PrivateScope.Privatize();
   if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
     CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
 
   CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
 }
 
-void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
-                                                  StringRef ParentName,
-                                                  const OMPTargetDirective &S) {
+void CodeGenFunction::EmitOMPTargetDeviceFunction(
+    CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S,
+    const FunctionDecl *ParentFn) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetRegion(CGF, S, Action);
   };
   emitCommonOMPTargetDirective(*this, S, CodeGen);
 }
 
 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
                                         const OMPExecutableDirective &S,
                                         OpenMPDirectiveKind InnermostKind,
                                         const RegionCodeGenTy &CodeGen) {
   const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
   llvm::Function *OutlinedFn =
       CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
-          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
+          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen,
+          CGF.getParentFn());
 
   const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
   const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
   if (NT || TL) {
     const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
     const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
 
     CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
     CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
   };
   emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
-    CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetTeamsDirective &S) {
+    CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S,
+    const FunctionDecl *ParentFn) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsRegion(CGF, Action, S);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDirective(
     const OMPTargetTeamsDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsRegion(CGF, Action, S);
   };
emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
   };
   emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetTeamsDistributeDirective &S) {
+    const OMPTargetTeamsDistributeDirective &S, const FunctionDecl *ParentFn) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeRegion(CGF, Action, S);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
     const OMPTargetTeamsDistributeDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeRegion(CGF, Action, S);
   };
static void emitTargetTeamsDistributeSimdRegion(
   };
   emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetTeamsDistributeSimdDirective &S) {
+    const OMPTargetTeamsDistributeSimdDirective &S,
+    const FunctionDecl *ParentFn) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
     const OMPTargetTeamsDistributeSimdDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
   };
static void emitTargetTeamsDistributeParallelForRegion(
   emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
                               CodeGenTeams);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetTeamsDistributeParallelForDirective &S) {
+    const OMPTargetTeamsDistributeParallelForDirective &S,
+    const FunctionDecl *ParentFn) {
   // Emit SPMD target teams distribute parallel for region as a standalone
   // region.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
     const OMPTargetTeamsDistributeParallelForDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
   };
static void emitTargetTeamsDistributeParallelForSimdRegion(
   emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
                               CodeGenTeams);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
+    const OMPTargetTeamsDistributeParallelForSimdDirective &S,
+    const FunctionDecl *ParentFn) {
   // Emit SPMD target teams distribute parallel for simd region as a standalone
   // region.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
     const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
   };
static void emitTargetParallelRegion(CodeGenFunction &CGF,
   emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
                                  emitEmptyBoundParameters);
   emitPostUpdateForReductionClause(CGF, S,
                                    [](CodeGenFunction &) { return nullptr; });
 }
 
 void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetParallelDirective &S) {
+    const OMPTargetParallelDirective &S, const FunctionDecl *ParentFn) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetParallelDirective(
     const OMPTargetParallelDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelRegion(CGF, S, Action);
   };
CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
                                emitDispatchForLoopBounds);
   };
   emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
                                  emitEmptyBoundParameters);
 }
 
 void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetParallelForDirective &S) {
+    const OMPTargetParallelForDirective &S, const FunctionDecl *ParentFn) {
   // Emit SPMD target parallel for region as a standalone region.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelForRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetParallelForDirective(
     const OMPTargetParallelForDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelForRegion(CGF, S, Action);
   };
CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
                                emitDispatchForLoopBounds);
   };
   emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
                                  emitEmptyBoundParameters);
 }
 
 void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
     CodeGenModule &CGM, StringRef ParentName,
-    const OMPTargetParallelForSimdDirective &S) {
+    const OMPTargetParallelForSimdDirective &S, const FunctionDecl *ParentFn) {
   // Emit SPMD target parallel for region as a standalone region.
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelForSimdRegion(CGF, S, Action);
   };
   llvm::Function *Fn;
   llvm::Constant *Addr;
   // Emit target region as a standalone region.
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
-      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
+      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen, ParentFn);
   assert(Fn && Addr && "Target device function emission failed.");
 }
 
 void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
     const OMPTargetParallelForSimdDirective &S) {
   auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
     emitTargetParallelForSimdRegion(CGF, S, Action);
   };

lib/CodeGen/CGTransform.h

+//===---- CGTransform.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Emitting metadata for loop transformations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_CGTRANSFORM_H
+#define LLVM_CLANG_LIB_CODEGEN_CGTRANSFORM_H
+
+#include "clang/Analysis/AnalysisTransform.h"
+#include "llvm/IR/DebugLoc.h"
+
+namespace clang {
+namespace CodeGen {
+class CGDebugInfo;
+class CGTransformedTreeBuilder;
+
+class CGTransformedTree : public TransformedTree<CGTransformedTree> {
+  friend class CGTransformedTreeBuilder;
+  friend class TransformedTree<CGTransformedTree>;
+  using BaseTy = TransformedTree<CGTransformedTree>;
+  using NodeTy = CGTransformedTree;
+
+  BaseTy &getBase() { return *this; }
+  const BaseTy &getBase() const { return *this; }
+
+  llvm::DebugLoc BeginLoc;
+  llvm::DebugLoc EndLoc;
+
+  llvm::MDNode *AccessGroup = nullptr;
+  llvm::SmallSetVector<llvm::MDNode *, 2> ParallelAccessGroups;
+
+public:
+  CGTransformedTree(llvm::ArrayRef<NodeTy *> SubLoops, NodeTy *BasedOn,
+                    clang::Stmt *Original, int FollowupRole, int Stage)
+      : TransformedTree(SubLoops, BasedOn, Original, FollowupRole, Stage) {}
+
+  bool IsDefault = true;
+  bool DisableHeuristic = false;
+
+  llvm::SmallVector<llvm::Metadata *, 8> Attributes;
+  llvm::SmallVector<llvm::Metadata *, 4> Transforms;
+  llvm::SmallVector<std::pair<llvm::StringRef, NodeTy *>, 4> FollowupAttributes;
+
+  bool collectLoopProperties(llvm::SmallVectorImpl<llvm::Metadata *> &Props);
+  void addAttribute(bool Inherited, llvm::Metadata *Node);
+
+public:
+  void markNondefault() { IsDefault = false; }
+  void markDisableHeuristic() { DisableHeuristic = true; }
+
+  void addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                    llvm::ArrayRef<llvm::Metadata *> Vals);
+  void addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                    llvm::StringRef Name);
+  void addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                    llvm::StringRef Name, bool Val);
+  void addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                    llvm::StringRef Name, int Val);
+
+  llvm::MDNode *getAccessGroupOrNull() {
+    assert(getOriginal());
+    return AccessGroup;
+  }
+  void
+  getOrCreateAccessGroups(llvm::LLVMContext &LLVMCtx,
+                          llvm::SmallVectorImpl<llvm::MDNode *> &AccessGroups);
+  llvm::ArrayRef<llvm::MDNode *> getParallelAccessGroups() const {
+    return ParallelAccessGroups.getArrayRef();
+  }
+
+  llvm::MDNode *makeLoopID(llvm::LLVMContext &Ctx, bool HasAllDisableNonforced);
+};
+
+class CGTransformedTreeBuilder
+    : public TransformedTreeBuilder<CGTransformedTreeBuilder,
+                                    CGTransformedTree> {
+  using BaseTy =
+      TransformedTreeBuilder<CGTransformedTreeBuilder, CGTransformedTree>;
+  using NodeTy = CGTransformedTree;
+
+  BaseTy &getBase() { return *this; }
+  const BaseTy &getBase() const { return *this; }
+
+  llvm::LLVMContext &LLVMCtx;
+  CGDebugInfo *DbgInfo;
+
+public:
+  CGTransformedTreeBuilder(ASTContext &ASTCtx, llvm::LLVMContext &LLVMCtx,
+                           llvm::SmallVectorImpl<NodeTy *> &AllNodes,
+                           CGDebugInfo *DbgInfo)
+      : TransformedTreeBuilder(ASTCtx, AllNodes), LLVMCtx(LLVMCtx),
+        DbgInfo(DbgInfo) {}
+
+  struct DummyDiag {};
+  DummyDiag Diag(SourceLocation Loc, unsigned DiagID) { return {}; }
+
+  void applyOriginal(CGTransformedTree *L);
+
+  void inheritLoopAttributes(CGTransformedTree *Dst, CGTransformedTree *Src,
+                             bool IsAll, bool IsSuccessor);
+  void markParallel(CGTransformedTree *L);
+
+  void disableUnroll(CGTransformedTree *L) {
+    L->addAttribute(LLVMCtx, true, "llvm.loop.unroll.disable");
+    L->markNondefault();
+  }
+
+  void disableUnrollAndJam(CGTransformedTree *L) {
+    L->addAttribute(LLVMCtx, true, "llvm.loop.unroll_and_jam.disable");
+    L->markNondefault();
+  }
+
+  void disableDistribution(CGTransformedTree *L) {
+    L->addAttribute(LLVMCtx, true, "llvm.loop.distribute.enable", false);
+    L->markNondefault();
+  }
+
+  void disableVectorizeInterleave(CGTransformedTree *L) {
+    L->addAttribute(LLVMCtx, true, "llvm.loop.vectorize.width", 1);
+    L->markNondefault();
+  }
+
+  void disablePipelining(CGTransformedTree *L) {
+    L->addAttribute(LLVMCtx, true, "llvm.loop.pipeline.disable", true);
+    L->markNondefault();
+  }
+
+  void applyUnroll(LoopUnrollingTransform *Trans,
+                   CGTransformedTree *OriginalLoop);
+  void applyUnrollAndJam(LoopUnrollAndJamTransform *Trans,
+                         CGTransformedTree *OuterLoop,
+                         CGTransformedTree *InnerLoop);
+  void applyDistribution(LoopDistributionTransform *Trans,
+                         CGTransformedTree *OriginalLoop);
+  void applyVectorizeInterleave(LoopVectorizationInterleavingTransform *Trans,
+                                CGTransformedTree *MainLoop);
+  void applyPipelining(LoopPipeliningTransform *Trans,
+                       CGTransformedTree *MainLoop);
+};
+
+} // namespace CodeGen
+} // namespace clang
+#endif /* LLVM_CLANG_LIB_CODEGEN_CGTRANSFORM_H */

lib/CodeGen/CGTransform.cpp

+//===---- CGTransform.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Emitting metadata for loop transformations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGTransform.h"
+#include "CGDebugInfo.h"
+#include "CodeGenFunction.h"
+#include "clang/AST/StmtTransform.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Transforms/Utils/UnrollLoop.h"
+
+using namespace clang;
+using namespace clang::CodeGen;
+using namespace llvm;
+
+void CodeGenFunction::HandleCodeTransformations(const Stmt *Body) {
+  if (!getParentFn()) {
+    // Transformations not supported for e.g. Objective-C
+    return;
+  }
+
+  assert(CurFn && "Must be called after StartFunction");
+  assert(Body);
+
+  LoopStack.initBuild(getContext(), getLLVMContext(), DebugInfo,
+                      getParentFn()->getBody());
+}
+
+void CodeGenFunction::EmitTransformExecutableDirective(
+    const TransformExecutableDirective &D) {
+  EmitStmt(D.getAssociated());
+}
+
+bool CGTransformedTree::collectLoopProperties(
+    llvm::SmallVectorImpl<llvm::Metadata *> &Props) {
+  for (Metadata *M : this->Attributes)
+    Props.push_back(M);
+  for (Metadata *M : this->Transforms)
+    Props.push_back(M);
+  return !Props.empty();
+}
+
+void CGTransformedTree::addAttribute(bool Inherited, llvm::Metadata *Node) {
+  if (Inherited)
+    Attributes.push_back(Node);
+  else
+    Transforms.push_back(Node);
+}
+
+void CGTransformedTree::addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                                     llvm::ArrayRef<llvm::Metadata *> Vals) {
+  addAttribute(Inherited, MDNode::get(LLVMCtx, Vals));
+}
+
+void CGTransformedTree::addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                                     llvm::StringRef Name) {
+  addAttribute(LLVMCtx, Inherited, {MDString::get(LLVMCtx, Name)});
+}
+
+void CGTransformedTree::addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                                     llvm::StringRef Name, bool Val) {
+  addAttribute(LLVMCtx, Inherited,
+               {MDString::get(LLVMCtx, Name),
+                ConstantAsMetadata::get(
+                    ConstantInt::get(llvm::Type::getInt1Ty(LLVMCtx), Val))});
+}
+void CGTransformedTree::addAttribute(llvm::LLVMContext &LLVMCtx, bool Inherited,
+                                     llvm::StringRef Name, int Val) {
+  addAttribute(LLVMCtx, Inherited,
+               {MDString::get(LLVMCtx, Name),
+                ConstantAsMetadata::get(
+                    ConstantInt::get(llvm::Type::getInt32Ty(LLVMCtx), Val))});
+}
+
+void CGTransformedTree::getOrCreateAccessGroups(
+    llvm::LLVMContext &LLVMCtx,
+    llvm::SmallVectorImpl<llvm::MDNode *> &AccessGroups) {
+  if (getOriginal()) {
+    if (!AccessGroup)
+      AccessGroup = MDNode::getDistinct(LLVMCtx, {});
+    AccessGroups.push_back(AccessGroup);
+    return;
+  }
+
+  getBasedOn()->getOrCreateAccessGroups(LLVMCtx, AccessGroups);
+}
+
+llvm::MDNode *CGTransformedTree::makeLoopID(llvm::LLVMContext &Ctx,
+                                            bool HasAllDisableNonforced) {
+  if (IsDefault && (!DisableHeuristic || HasAllDisableNonforced))
+    return nullptr;
+
+  SmallVector<Metadata *, 16> Args;
+
+  // Reserve operand 0 for loop id self reference.
+  Args.push_back(nullptr);
+
+  if (BeginLoc) {
+    Args.push_back(BeginLoc.getAsMDNode());
+
+    // If we also have a valid end debug location for the loop, add it.
+    if (EndLoc)
+      Args.push_back(EndLoc.getAsMDNode());
+  }
+
+  if (!ParallelAccessGroups.empty()) {
+    SmallVector<Metadata *, 4> ArgOpts;
+    ArgOpts.reserve(ParallelAccessGroups.size());
+    ArgOpts.push_back(MDString::get(Ctx, "llvm.loop.parallel_accesses"));
+    ArgOpts.insert(ArgOpts.end(), ParallelAccessGroups.begin(),
+                   ParallelAccessGroups.end());
+    Args.push_back(MDNode::get(Ctx, ArgOpts));
+  }
+
+  collectLoopProperties(Args);
+
+  bool AllIsDisableHeuristic = false;
+  bool OtherIsNondefault = false;
+  for (auto P : FollowupAttributes) {
+    int Role = P.second->FollowupRole;
+    if (TransformedBy->isAllRole(Role)) {
+      if (P.second->DisableHeuristic)
+        AllIsDisableHeuristic = true;
+    } else {
+      if (!P.second->IsDefault)
+        OtherIsNondefault = true;
+    }
+  }
+
+  for (auto P : FollowupAttributes) {
+    StringRef FollowupName = P.first;
+    NodeTy *FollowupNode = P.second;
+    llvm::MDNode *FollowupId;
+    if (TransformedBy->isAllRole(FollowupNode->FollowupRole)) {
+      if (OtherIsNondefault)
+        FollowupNode->markNondefault();
+      FollowupId = FollowupNode->makeLoopID(Ctx, false);
+    } else {
+      FollowupId = FollowupNode->makeLoopID(Ctx, AllIsDisableHeuristic);
+    }
+    if (!FollowupId)
+      continue;
+
+    Args.push_back(
+        MDNode::get(Ctx, {MDString::get(Ctx, FollowupName), FollowupId}));
+  }
+
+  if (DisableHeuristic && !HasAllDisableNonforced)
+    Args.push_back(
+        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.disable_nonforced")}));
+
+  // No need for an MDNode if it is empty.
+  if (Args.size() <= 1)
+    return nullptr;
+
+  // Set the first operand to itself.
+  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+  LoopID->replaceOperandWith(0, LoopID);
+  return LoopID;
+}
+
+void CGTransformedTreeBuilder::applyOriginal(CGTransformedTree *L) {
+  if (!DbgInfo)
+    return;
+
+  L->BeginLoc = DbgInfo->SourceLocToDebugLoc(L->getOriginal()->getBeginLoc());
+  L->EndLoc = DbgInfo->SourceLocToDebugLoc(L->getOriginal()->getEndLoc());
+  if (L->BeginLoc || L->EndLoc)
+    L->markNondefault();
+}
+
+void CGTransformedTreeBuilder::inheritLoopAttributes(CGTransformedTree *Dst,
+                                                     CGTransformedTree *Src,
+                                                     bool IsAll,
+                                                     bool IsSuccessor) {
+  Dst->BeginLoc = Src->BeginLoc;
+  Dst->EndLoc = Src->EndLoc;
+
+  for (auto A : Src->Attributes) {
+    // TOOD: Check for duplicates?
+    Dst->Attributes.push_back(A);
+  }
+
+  // We currently assume that every transformation of a parallel loop also
+  // results in a parallel loop.
+  Dst->ParallelAccessGroups.insert(Src->ParallelAccessGroups.begin(),
+                                   Src->ParallelAccessGroups.end());
+}
+
+void CGTransformedTreeBuilder::markParallel(CGTransformedTree *L) {
+  getBase().markParallel(L);
+
+  // Has it already been marked parallel?
+  // Avoid redundant metadata if it was.
+  if (!L->ParallelAccessGroups.empty())
+    return;
+
+  SmallVector<llvm::MDNode *, 8> AccGroups;
+  assert(L->ParallelAccessGroups.empty() &&
+         "Should not have parallel access groups if was empty before");
+
+  L->getOrCreateAccessGroups(LLVMCtx, AccGroups);
+  L->ParallelAccessGroups.insert(AccGroups.begin(), AccGroups.end());
+}
+
+void CGTransformedTreeBuilder::applyUnroll(LoopUnrollingTransform *Trans,
+                                           CGTransformedTree *OriginalLoop) {
+  if (Trans->isExplicitEnable() && !(Trans->isLegacy() && Trans->isFull()))
+    OriginalLoop->addAttribute(LLVMCtx, false, "llvm.loop.unroll.enable");
+
+  if (Trans->isFull()) {
+    OriginalLoop->addAttribute(LLVMCtx, false, "llvm.loop.unroll.full");
+  } else {
+    int UnrollFactor = Trans->getFactor();
+    if (UnrollFactor > 0)
+      OriginalLoop->addAttribute(LLVMCtx, false, "llvm.loop.unroll.count",
+                                 UnrollFactor);
+  }
+
+  for (CGTransformedTree *F : OriginalLoop->Followups) {
+    switch (F->FollowupRole) {
+    case LoopUnrollingTransform::FollowupAll:
+      OriginalLoop->FollowupAttributes.emplace_back(LLVMLoopUnrollFollowupAll,
+                                                    F);
+      if (Trans->isLegacy())
+        F->addAttribute(LLVMCtx, true, "llvm.loop.unroll.disable");
+      else
+        F->markDisableHeuristic();
+      break;
+    case LoopUnrollingTransform::FollowupUnrolled:
+      OriginalLoop->FollowupAttributes.emplace_back(
+          LLVMLoopUnrollFollowupUnrolled, F);
+      break;
+    case LoopUnrollingTransform::FollowupRemainder:
+      OriginalLoop->FollowupAttributes.emplace_back(
+          LLVMLoopUnrollFollowupRemainder, F);
+      break;
+    }
+  }
+
+  OriginalLoop->markNondefault();
+  if (!Trans->isLegacy())
+    OriginalLoop->markDisableHeuristic();
+}
+
+void CGTransformedTreeBuilder::applyUnrollAndJam(
+    LoopUnrollAndJamTransform *Trans, CGTransformedTree *OuterLoop,
+    CGTransformedTree *InnerLoop) {
+  int Factor = Trans->getFactor();
+  if (Factor > 0)
+    OuterLoop->addAttribute(LLVMCtx, false, "llvm.loop.unroll_and_jam.count",
+                            Factor);
+
+  if (Trans->isExplicitEnable())
+    OuterLoop->addAttribute(LLVMCtx, false, "llvm.loop.unroll_and_jam.enable");
+
+  for (CGTransformedTree *F : OuterLoop->Followups) {
+    switch (F->FollowupRole) {
+    case LoopUnrollAndJamTransform::FollowupAll:
+      OuterLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.unroll_and_jam.followup_all", F);
+      if (!Trans->isLegacy())
+        F->markDisableHeuristic();
+      break;
+    case LoopUnrollAndJamTransform::FollowupOuter:
+      OuterLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.unroll_and_jam.followup_outer", F);
+      if (!Trans->isLegacy())
+        F->markDisableHeuristic();
+      if (Trans->isLegacy())
+        F->addAttribute(LLVMCtx, true, "llvm.loop.unroll_and_jam.disable");
+      break;
+    }
+  }
+
+  if (InnerLoop) {
+    for (CGTransformedTree *F : InnerLoop->Followups) {
+      switch (F->FollowupRole) {
+      case LoopUnrollAndJamTransform::FollowupInner:
+        if (!Trans->isLegacy())
+          F->markDisableHeuristic();
+        OuterLoop->FollowupAttributes.emplace_back(
+            "llvm.loop.unroll_and_jam.followup_inner", F);
+        break;
+      }
+    }
+  }
+
+  OuterLoop->markNondefault();
+  if (!Trans->isLegacy())
+    OuterLoop->markDisableHeuristic();
+  if (InnerLoop && !Trans->isLegacy())
+    InnerLoop->markDisableHeuristic();
+}
+
+void CGTransformedTreeBuilder::applyDistribution(
+    LoopDistributionTransform *Trans, CGTransformedTree *OriginalLoop) {
+  OriginalLoop->addAttribute(LLVMCtx, false, "llvm.loop.distribute.enable",
+                             true);
+
+  for (CGTransformedTree *F : OriginalLoop->Followups) {
+    switch (F->FollowupRole) {
+    case LoopDistributionTransform::FollowupAll:
+      OriginalLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.distribute.followup_all", F);
+      if (!Trans->isLegacy())
+        F->markDisableHeuristic();
+      break;
+    }
+  }
+
+  OriginalLoop->markNondefault();
+  if (!Trans->isLegacy())
+    OriginalLoop->markDisableHeuristic();
+}
+
+void CGTransformedTreeBuilder::applyVectorizeInterleave(
+    LoopVectorizationInterleavingTransform *Trans,
+    CGTransformedTree *MainLoop) {
+  bool DisabledVectorization =
+      !Trans->isVectorizationEnabled().getValueOr(true) ||
+      Trans->getWidth() == 1;
+  bool ForcedInterleaving = Trans->isInterleavingEnabled().getValueOr(false);
+  Optional<bool> VecEnabled = Trans->isVectorizationEnabled();
+  if (ForcedInterleaving)
+    VecEnabled = true;
+
+  if (Trans->isPredicateEnabled().hasValue() && !DisabledVectorization) {
+    MainLoop->addAttribute(LLVMCtx, false,
+                           "llvm.loop.vectorize.predicate.enable",
+                           Trans->isPredicateEnabled().getValue());
+    if (Trans->isLegacy())
+      VecEnabled = true;
+  }
+
+  if (Trans->getWidth() > 0) {
+    if (Trans->isLegacy() && !VecEnabled.hasValue())
+      MainLoop->addAttribute(LLVMCtx, false, "llvm.loop.vectorize.enable",
+                             true);
+
+    MainLoop->addAttribute(LLVMCtx, false, "llvm.loop.vectorize.width",
+                           Trans->getWidth());
+  }
+
+  if (Trans->getInterleaveCount() > 0)
+    MainLoop->addAttribute(LLVMCtx, false, "llvm.loop.interleave.count",
+                           Trans->getInterleaveCount());
+
+  if (VecEnabled.hasValue())
+    MainLoop->addAttribute(LLVMCtx, false, "llvm.loop.vectorize.enable",
+                           VecEnabled.getValue());
+
+  for (CGTransformedTree *F : MainLoop->Followups) {
+    switch (F->FollowupRole) {
+    case LoopVectorizationInterleavingTransform ::FollowupAll:
+      if (Trans->isLegacy())
+        F->addAttribute(LLVMCtx, true, "llvm.loop.isvectorized");
+      MainLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.vectorize.followup_all", F);
+      if (!Trans->isLegacy())
+        F->markDisableHeuristic();
+      break;
+    case LoopVectorizationInterleavingTransform ::FollowupVectorized:
+      MainLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.vectorize.followup_vectorized", F);
+      break;
+    case LoopVectorizationInterleavingTransform ::FollowupEpilogue:
+      MainLoop->FollowupAttributes.emplace_back(
+          "llvm.loop.vectorize.followup_epilogue", F);
+      break;
+    }
+  }
+
+  MainLoop->markNondefault();
+  if (!Trans->isLegacy())
+    MainLoop->markDisableHeuristic();
+}
+
+void CGTransformedTreeBuilder::applyPipelining(LoopPipeliningTransform *Trans,
+                                               CGTransformedTree *MainLoop) {
+  if (Trans->getInitiationInterval() > 0)
+    MainLoop->addAttribute(LLVMCtx, false,
+                           "llvm.loop.pipeline.initiationinterval",
+                           Trans->getInitiationInterval());
+
+  MainLoop->markNondefault();
+}

lib/CodeGen/CMakeLists.txt

add_clang_library(clangCodeGen
   CGObjCMac.cpp
   CGObjCRuntime.cpp
   CGOpenCLRuntime.cpp
   CGOpenMPRuntime.cpp
   CGOpenMPRuntimeNVPTX.cpp
   CGRecordLayoutBuilder.cpp
   CGStmt.cpp
   CGStmtOpenMP.cpp
+  CGTransform.cpp
   CGVTT.cpp
   CGVTables.cpp
   CodeGenABITypes.cpp
   CodeGenAction.cpp
   CodeGenFunction.cpp
   CodeGenModule.cpp
   CodeGenPGO.cpp
   CodeGenTBAA.cpp

lib/CodeGen/CodeGenFunction.h

 class BlockByrefHelpers;
 class BlockByrefInfo;
 class BlockFlags;
 class BlockFieldFlags;
 class RegionCodeGenTy;
 class TargetCodeGenInfo;
 struct OMPTaskDataTy;
 struct CGCoroData;
+class CGTransformedTree;
 
 /// The kind of evaluation to perform on values of a particular
 /// type.  Basically, is the code in CGExprScalar, CGExprComplex, or
 /// CGExprAgg?
 ///
 /// TODO: should vectors maybe be split out into their own thing?
 enum TypeEvaluationKind {
   TEK_Scalar,
private:
   const bool ShouldEmitLifetimeMarkers;
 
   /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
   /// the function metadata.
   void EmitOpenCLKernelMetadata(const FunctionDecl *FD,
                                 llvm::Function *Fn);
 
 public:
-  CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext=false);
+  CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false,
+                  const FunctionDecl *ParentFn = nullptr);
   ~CodeGenFunction();
 
   CodeGenTypes &getTypes() const { return CGM.getTypes(); }
   ASTContext &getContext() const { return CGM.getContext(); }
   CGDebugInfo *getDebugInfo() {
     if (DisableDebugInfo)
       return nullptr;
     return DebugInfo;

 
   void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
                                 llvm::Value *ParentFP,
                                 llvm::Value *EntryEBP);
   llvm::Value *EmitSEHExceptionCode();
   llvm::Value *EmitSEHExceptionInfo();
   llvm::Value *EmitSEHAbnormalTermination();
 
+  void HandleCodeTransformations(const Stmt *Body);
+  void EmitTransformExecutableDirective(const TransformExecutableDirective &D);
+
   /// Emit simple code for OpenMP directives in Simd-only mode.
   void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
 
   /// Scan the outlined statement for captures from the parent function. For
   /// each capture, mark the capture as escaped and emit a call to
   /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
   void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
                           bool IsFilter);

   void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
       const OMPTargetTeamsDistributeParallelForSimdDirective &S);
   void EmitOMPTargetTeamsDistributeSimdDirective(
       const OMPTargetTeamsDistributeSimdDirective &S);
 
   /// Emit device code for the target directive.
   static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
                                           StringRef ParentName,
-                                          const OMPTargetDirective &S);
+                                          const OMPTargetDirective &S,
+                                          const FunctionDecl *ParentFn);
   static void
   EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
-                                      const OMPTargetParallelDirective &S);
+                                      const OMPTargetParallelDirective &S,
+                                      const FunctionDecl *ParentFn);
   /// Emit device code for the target parallel for directive.
   static void EmitOMPTargetParallelForDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetParallelForDirective &S);
+      const OMPTargetParallelForDirective &S, const FunctionDecl *ParentFn);
   /// Emit device code for the target parallel for simd directive.
   static void EmitOMPTargetParallelForSimdDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetParallelForSimdDirective &S);
+      const OMPTargetParallelForSimdDirective &S, const FunctionDecl *ParentFn);
   /// Emit device code for the target teams directive.
-  static void
-  EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
-                                   const OMPTargetTeamsDirective &S);
+  static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM,
+                                               StringRef ParentName,
+                                               const OMPTargetTeamsDirective &S,
+                                               const FunctionDecl *ParentFn);
   /// Emit device code for the target teams distribute directive.
   static void EmitOMPTargetTeamsDistributeDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetTeamsDistributeDirective &S);
+      const OMPTargetTeamsDistributeDirective &S, const FunctionDecl *ParentFn);
   /// Emit device code for the target teams distribute simd directive.
   static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetTeamsDistributeSimdDirective &S);
+      const OMPTargetTeamsDistributeSimdDirective &S,
+      const FunctionDecl *ParentFn);
   /// Emit device code for the target simd directive.
   static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
                                               StringRef ParentName,
-                                              const OMPTargetSimdDirective &S);
+                                              const OMPTargetSimdDirective &S,
+                                              const FunctionDecl *ParentFn);
   /// Emit device code for the target teams distribute parallel for simd
   /// directive.
   static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetTeamsDistributeParallelForSimdDirective &S);
+      const OMPTargetTeamsDistributeParallelForSimdDirective &S,
+      const FunctionDecl *ParentFn);
 
   static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
       CodeGenModule &CGM, StringRef ParentName,
-      const OMPTargetTeamsDistributeParallelForDirective &S);
+      const OMPTargetTeamsDistributeParallelForDirective &S,
+      const FunctionDecl *ParentFn);
   /// Emit inner loop of the worksharing/simd construct.
   ///
   /// \param S Directive, for which the inner loop must be emitted.
   /// \param RequiresCleanup true, if directive has some associated private
   /// variables.
   /// \param LoopCond Bollean condition for loop continuation.
   /// \param IncExpr Increment expression for loop control variable.
   /// \param BodyGen Generator for the inner body of the inner loop.
private:
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
   llvm::Value *EmitX86CpuIs(const CallExpr *E);
   llvm::Value *EmitX86CpuIs(StringRef CPUStr);
   llvm::Value *EmitX86CpuSupports(const CallExpr *E);
   llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
   llvm::Value *EmitX86CpuSupports(uint64_t Mask);
   llvm::Value *EmitX86CpuInit();
   llvm::Value *FormResolverCondition(const MultiVersionResolverOption &RO);
+
+  const FunctionDecl *ParentFn = nullptr;
+
+public:
+  const FunctionDecl *getParentFn() const { return ParentFn; }
 };
 
 inline DominatingLLVMValue::saved_type
 DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
   if (!needsSaving(value)) return saved_type(value, false);
 
   // Otherwise, we need an alloca.
   auto align = CharUnits::fromQuantity(

lib/CodeGen/CodeGenFunction.cpp

 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This coordinates the per-function state used while generating code.
 //
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
 #include "CGBlocks.h"
-#include "CGCleanup.h"
 #include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
+#include "CGCleanup.h"
 #include "CGDebugInfo.h"
 #include "CGOpenMPRuntime.h"
 #include "CodeGenModule.h"
 #include "CodeGenPGO.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtObjC.h"
+#include "clang/AST/StmtOpenMP.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
+
 using namespace clang;
 using namespace CodeGen;
 
 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
 /// markers.
 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
                                       const LangOptions &LangOpts) {
   if (CGOpts.DisableLifetimeMarkers)
     return false;
 
   // Sanitizers may use markers.
   if (CGOpts.SanitizeAddressUseAfterScope ||
       LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
       LangOpts.Sanitize.has(SanitizerKind::Memory))
     return true;
 
   // For now, only in optimized builds.
   return CGOpts.OptimizationLevel != 0;
 }
 
-CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
+CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext,
+                                 const FunctionDecl *ParentFn)
     : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
               CGBuilderInserterTy(this)),
       SanOpts(CGM.getLangOpts().Sanitize), DebugInfo(CGM.getModuleDebugInfo()),
-      PGO(cgm), ShouldEmitLifetimeMarkers(shouldEmitLifetimeMarkers(
-                    CGM.getCodeGenOpts(), CGM.getLangOpts())) {
+      ParentFn(ParentFn), PGO(cgm),
+      ShouldEmitLifetimeMarkers(
+          shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
   if (!suppressNewContext)
     CGM.getCXXABI().getMangleContext().startNewFunction();
 
   llvm::FastMathFlags FMF;
   if (CGM.getLangOpts().FastMath)
     FMF.setFast();
   if (CGM.getLangOpts().FiniteMathOnly) {
     FMF.setNoNaNs();
void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
 
   // Initialize helper which will detect jumps which can cause invalid lifetime
   // markers.
   if (Body && ShouldEmitLifetimeMarkers)
     Bypasses.Init(Body);
 
   // Emit the standard function prologue.
   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
+  if (Body)
+    HandleCodeTransformations(Body);
 
   // Generate the body of the function.
   PGO.assignRegionCounters(GD, CurFn);
   if (isa<CXXDestructorDecl>(FD))
     EmitDestructorBody(Args);
   else if (isa<CXXConstructorDecl>(FD))
     EmitConstructorBody(Args);
   else if (getLangOpts().CUDA &&

lib/CodeGen/CodeGenModule.cpp

void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
   // FIXME: this is redundant with part of setFunctionDefinitionAttributes
   setGVProperties(Fn, GD);
 
   MaybeHandleStaticInExternC(D, Fn);
 
 
   maybeSetTrivialComdat(*D, *Fn);
 
-  CodeGenFunction(*this).GenerateCode(D, Fn, FI);
+  {
+    CodeGenFunction CGF(*this, false, D);
+    CGF.GenerateCode(D, Fn, FI);
+  }
 
   setNonAliasAttributes(GD, Fn);
   SetLLVMFunctionAttributesForDefinition(D, Fn);
 
   if (const ConstructorAttr *CA = D->getAttr<ConstructorAttr>())
     AddGlobalCtor(Fn, CA->getPriority());
   if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
     AddGlobalDtor(Fn, DA->getPriority());

test/CodeGenCXX/pragma-followup_inner.cpp

 
 
 // CHECK-LABEL: define void @followup_inner
 // CHECK: br label %for.cond1, !llvm.loop ![[INNERLOOP_3:[0-9]+]]
 // CHECK: br label %for.cond, !llvm.loop ![[OUTERLOOP_9:[0-9]+]]
 
 // CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
 
-// CHECK-DAG: ![[INNERLOOP_3:[0-9]+]] = distinct !{![[INNERLOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
-// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", !2}
+// CHECK-DAG: ![[INNERLOOP_3:[0-9]+]] = distinct !{![[INNERLOOP_3:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
 // CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
 // CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]]}
+// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2:[0-9]+]]}
 // CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
 
 // CHECK-DAG: ![[OUTERLOOP_9:[0-9]+]] = distinct !{![[OUTERLOOP_9:[0-9]+]], ![[UNROLLANDJAM_COUNT_10:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]]}
 // CHECK-DAG: ![[UNROLLANDJAM_COUNT_10:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
 // CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_inner", !12}
 
 // CHECK-DAG: ![[LOOP_12:[0-9]+]] = distinct !{![[LOOP_12:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_COUNT_13:[0-9]+]], ![[UNROLL_FOLLOWUP_14:[0-9]+]]}
 // CHECK-DAG: ![[ISVECTORIZED_13:[0-9]+]] = !{!"llvm.loop.isvectorized"}
 // CHECK-DAG: ![[UNROLL_COUNT_13:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
 // CHECK-DAG: ![[UNROLL_FOLLOWUP_14:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_15:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_15:[0-9]+]] = distinct !{![[LOOP_15:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_DISABLE_16:[0-9]+]], ![[PIPELINE_17:[0-9]+]]}
 // CHECK-DAG: ![[UNROLL_DISABLE_16:[0-9]+]] = !{!"llvm.loop.unroll.disable"}
 // CHECK-DAG: ![[PIPELINE_17:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}

test/CodeGenCXX/pragma-followup_outer.cpp

 }
 
 
 // CHECK-LABEL: define void @followup_outer
 // CHECK: br label %for.cond, !llvm.loop ![[LOOP_3:[0-9]+]]
 
 // CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
 
-// CHECK-DAG: ![[LOOP_3:[0-9]+]] = distinct !{![[LOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
-// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2]]}
+// CHECK-DAG: ![[LOOP_3:[0-9]+]] = distinct !{![[LOOP_3:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
 // CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
 // CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]], ![[VECTORIZE_FOLLOWUP_9:[0-9]+]]}
+// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2]]}
 // CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
 // CHECK-DAG: ![[VECTORIZE_FOLLOWUP_9:[0-9]+]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_10:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_10:[0-9]+]] = distinct !{![[LOOP_10:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_12:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]]}
 // CHECK-DAG: ![[ISVECTORIZED_11:[0-9]+]] = !{!"llvm.loop.isvectorized"}
 // CHECK-DAG: ![[UNROLLANDJAM_12:[0-9]+]] =  !{!"llvm.loop.unroll_and_jam.enable"}
 // CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_14:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_14:[0-9]+]] = distinct !{![[LOOP_14:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_COUNT_16:[0-9]+]], ![[UNROLL_FOLLOWUP_17:[0-9]+]]}
 // CHECK-DAG: ![[UNROLLANDJAM_DISABLE_15:[0-9]+]]  = !{!"llvm.loop.unroll_and_jam.disable"}
 // CHECK-DAG: ![[UNROLL_COUNT_16:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
 // CHECK-DAG: ![[UNROLL_FOLLOWUP_17:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_18:[0-9]+]]}
 
 // CHECK-DAG: ![[LOOP_18:[0-9]+]] = distinct !{![[LOOP_18:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_DISABLE_19:[0-9]+]], ![[INITIATIONINTERVAL_20:[0-9]+]]}
 // CHECK-DAG: ![[UNROLL_DISABLE_19:[0-9]+]]  = !{!"llvm.loop.unroll.disable"}
 // CHECK-DAG: ![[INITIATIONINTERVAL_20:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}

test/CodeGenCXX/pragma-loop.cpp

 // CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
 // CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
 // CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32}
 
 // CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], ![[VECTORIZE_ENABLE]], ![[WIDTH_10:.*]]}
 // CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}
 
 // CHECK:      ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[WIDTH_1]], ![[VECTORIZE_ENABLE]]}
+// !7, !19
 
 // CHECK-NEXT: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[VECTORIZE_ENABLE]], ![[WIDTH_1]]}
+ // !19, !7
+

test/CodeGenCXX/pragma-transform/pragma-transform-distribute.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_distribute(int *List, int Length) {
+#pragma clang transform distribute
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[DISTRIBUTE_ENABLE:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}

test/CodeGenCXX/pragma-transform/pragma-transform-interleave-factor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_interleave_factor(int *List, int Length) {
+#pragma clang transform interleave factor(2)
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[VECTORIZE_DISABLE:[0-9]+]],  ![[INTERLEAVE_COUNT:[0-9]+]], ![[INTERLEAVE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK-DAG: ![[INTERLEAVE_COUNT]] = !{!"llvm.loop.interleave.count", i32 2}
+// CHECK-DAG: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-interleave-successor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_interleave_successor(int *List, int Length) {
+#pragma clang transform unroll
+#pragma clang transform interleave
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[VECTORIZE_DISABLE:[0-9]+]], ![[INTERLEAVE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[VECTORIZE_FOLLOWUP_VECTORIZED:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK-DAG: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_VECTORIZED]] = !{!"llvm.loop.vectorize.followup_vectorized", ![[LOOP_VECTORIZED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_VECTORIZED]] = distinct !{![[LOOP_VECTORIZED]],  ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", !12}
+
+// CHECK-DAG: ![[LOOP_UNROLLED:[0-9]+]] = distinct !{![[LOOP_UNROLLED:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-interleave.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_interleave(int *List, int Length) {
+#pragma clang transform interleave
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[VECTORIZE_DISABLE:[0-9]+]], ![[INTERLEAVE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK-DAG: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unroll-full.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unroll_full(int *List, int Length) {
+#pragma clang transform unroll full
+  for (int i = 0; i < 4; i++) {
+//  CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FULL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}

test/CodeGenCXX/pragma-transform/pragma-transform-unroll-partial.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unroll_partial(int *List, int Length) {
+#pragma clang transform unroll partial(8)
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FACTOR:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FACTOR]] = !{!"llvm.loop.unroll.count", i32 8}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_UNROLL_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_UNROLL_ALL]] = distinct !{![[LOOP_UNROLL_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unroll-successor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unroll_successor(int *List, int Length) {
+#pragma clang transform distribute
+#pragma clang transform unroll
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]], ![[UNROLL_FOLLOWUP_UNROLLED:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_UNROLL_ALL:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_UNROLLED]] = !{!"llvm.loop.unroll.followup_unrolled", ![[LOOP_UNROLLED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLED]] = distinct !{![[LOOP_UNROLLED]], ![[DISTRIBUTE_ENABLE:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_ALL]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_DISTRIBUTE_ALL:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLL_ALL]] = distinct !{![[LOOP_UNROLL_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[LOOP_DISTRIBUTE_ALL]] = distinct !{![[LOOP_DISTRIBUTE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}

test/CodeGenCXX/pragma-transform/pragma-transform-unroll.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unroll(int *List, int Length) {
+#pragma clang transform unroll
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_UNROLL_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_UNROLL_ALL]] = distinct !{![[LOOP_UNROLL_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unrollandjam-partial.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unrollandjam(int *List, int Length) {
+#pragma clang transform unrollandjam partial(4)
+  for (int i = 0; i < Length; i++) {
+    for (int j = 0; j < Length; j++)  {
+      List[j] += i * 2;
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+    }
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[UNROLLANDJAM_COUNT:[0-9]+]], ![[UNROLLANDJAM_ENABLE:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_OUTER:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_INNER:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_COUNT]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
+// CHECK-DAG: ![[UNROLLANDJAM_ENABLE]] = !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_OUTER]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_UNROLLANDJAM_OUTER:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_INNER]] = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[LOOP_UNROLLANDJAM_INNER:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_OUTER]] = distinct !{![[LOOP_UNROLLANDJAM_OUTER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_INNER]] = distinct !{![[LOOP_UNROLLANDJAM_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unrollandjam-predecessor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unrollandjam_predecessor(int *List, int Length) {
+#pragma clang transform unrollandjam
+  for (int i = 0; i < Length; i++) {
+    #pragma clang transform unroll
+    for (int j = 0; j < Length; j++)  {
+      List[j] += i * 2;
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+    }
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_INNER_UNROLL_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_INNER_UNROLL_ALL]] = distinct !{![[LOOP_INNER_UNROLL_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[UNROLLANDJAM_ENABLE:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_OUTER:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_INNER:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_ENABLE]] = !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_OUTER]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_UNROLLANDJAM_OUTER:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_INNER]] = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[LOOP_UNROLLANDJAM_INNER:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_OUTER]] = distinct !{![[LOOP_UNROLLANDJAM_OUTER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_INNER]] = distinct !{![[LOOP_UNROLLANDJAM_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unrollandjam-successor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unrollandjam_successor(int *List, int Length) {
+#pragma clang transform distribute
+#pragma clang transform unrollandjam
+  for (int i = 0; i < Length; i++) {
+    for (int j = 0; j < Length; j++)  {
+      List[j] += i * 2;
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+    }
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[UNROLLANDJAM_ENABLE:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_OUTER:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_INNER:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_ENABLE]] = !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_OUTER]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_UNROLLANDJAM_OUTER:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_INNER]] = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[LOOP_UNROLLANDJAM_INNER:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_INNER]] = distinct !{![[LOOP_UNROLLANDJAM_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_OUTER]] = distinct !{![[LOOP_UNROLLANDJAM_OUTER]], ![[DISTRIBUTE_ENABLE:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_ALL]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_DISTRIBUTE_FOLLOWUP_ALL:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_DISTRIBUTE_FOLLOWUP_ALL]] = distinct !{![[LOOP_DISTRIBUTE_FOLLOWUP_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-unrollandjam.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_unrollandjam(int *List, int Length) {
+#pragma clang transform unrollandjam
+  for (int i = 0; i < Length; i++) {
+    for (int j = 0; j < Length; j++)  {
+      List[j] += i * 2;
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+    }
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[UNROLLANDJAM_ENABLE:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_OUTER:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUP_INNER:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_ENABLE]] = !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_OUTER]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_UNROLLANDJAM_OUTER:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUP_INNER]] = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[LOOP_UNROLLANDJAM_INNER:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_OUTER]] = distinct !{![[LOOP_UNROLLANDJAM_OUTER]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_UNROLLANDJAM_INNER]] = distinct !{![[LOOP_UNROLLANDJAM_INNER]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-vectorize-successor.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+void pragma_transform_vectorize_successor(int *List, int Length) {
+#pragma clang transform unroll
+#pragma clang transform vectorize
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[INTERLEAVE_DISABLE:[0-9]+]], ![[VECTORIZE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[VECTORIZE_FOLLOWUP_VECTORIZED:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.count", i32 1}
+// CHECK-DAG: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_VECTORIZED]] = !{!"llvm.loop.vectorize.followup_vectorized", ![[LOOP_VECTORIZED:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_VECTORIZED]] = distinct !{![[LOOP_VECTORIZED]],  ![[UNROLL_ENABLE:[0-9]+]], ![[UNROLL_FOLLOWUP_ALL:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_ENABLE]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_ALL:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", !12}
+
+// CHECK-DAG: ![[LOOP_UNROLLED:[0-9]+]] = distinct !{![[LOOP_UNROLLED:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-vectorize-width.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_vectorize(int *List, int Length) {
+#pragma clang transform vectorize width(4)
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[VECTORIZE_WIDTH:[0-9]+]], ![[INTERLEAVE_DISABLE:[0-9]+]], ![[VECTORIZE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_WIDTH]] = !{!"llvm.loop.vectorize.width", i32 4}
+// CHECK-DAG: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.count", i32 1}
+// CHECK-DAG: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}

test/CodeGenCXX/pragma-transform/pragma-transform-vectorize.cpp

+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void pragma_transform_vectorize(int *List, int Length) {
+#pragma clang transform vectorize
+  for (int i = 0; i < Length; i++) {
+// CHECK: br label %{{.*}}, !llvm.loop ![[LOOP:[0-9]+]]
+    List[i] = i * 2;
+  }
+}
+
+
+// CHECK-DAG: ![[LOOP]] = distinct !{![[LOOP]], ![[INTERLEAVE_DISABLE:[0-9]+]], ![[VECTORIZE_ENABLE:[0-9]+]], ![[VECTORIZE_FOLLOWUP_ALL:[0-9]+]], ![[DISABLE_NONFORCED:[0-9]+]]}
+// CHECK-DAG: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.count", i32 1}
+// CHECK-DAG: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_ALL]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_VECTORIZE_ALL:[0-9]+]]}
+// CHECK-DAG: ![[DISABLE_NONFORCED]] = !{!"llvm.loop.disable_nonforced"}
+
+// CHECK-DAG: ![[LOOP_VECTORIZE_ALL]] = distinct !{![[LOOP_VECTORIZE_ALL]], ![[DISABLE_NONFORCED:[0-9]+]]}