Index: cfe/trunk/lib/CodeGen/CGLoopInfo.h =================================================================== --- cfe/trunk/lib/CodeGen/CGLoopInfo.h +++ cfe/trunk/lib/CodeGen/CGLoopInfo.h @@ -78,10 +78,11 @@ public: /// Construct a new LoopInfo for the loop with entry Header. LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, - const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc); + const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, + LoopInfo *Parent); /// Get the loop id metadata for this loop. - llvm::MDNode *getLoopID() const { return LoopID; } + llvm::MDNode *getLoopID() const { return TempLoopID.get(); } /// Get the header block of this loop. llvm::BasicBlock *getHeader() const { return Header; } @@ -92,15 +93,92 @@ /// Return this loop's access group or nullptr if it does not have one. llvm::MDNode *getAccessGroup() const { return AccGroup; } + /// Create the loop's metadata. Must be called after its nested loops have + /// been processed. + void finish(); + private: /// Loop ID metadata. - llvm::MDNode *LoopID; + llvm::TempMDTuple TempLoopID; /// Header block of this loop. llvm::BasicBlock *Header; /// The attributes for this loop. LoopAttributes Attrs; /// The access group for memory accesses parallel to this loop. llvm::MDNode *AccGroup = nullptr; + /// Start location of this loop. + llvm::DebugLoc StartLoc; + /// End location of this loop. + llvm::DebugLoc EndLoc; + /// The next outer loop, or nullptr if this is the outermost loop. + LoopInfo *Parent; + /// If this loop has unroll-and-jam metadata, this can be set by the inner + /// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner + /// metadata. + llvm::MDNode *UnrollAndJamInnerFollowup = nullptr; + + /// Create a LoopID without any transformations. + llvm::MDNode * + createLoopPropertiesMetadata(llvm::ArrayRef LoopProperties); + + /// Create a LoopID for transformations. + /// + /// The methods call each other in case multiple transformations are applied + /// to a loop. The transformation first to be applied will use LoopID of the + /// next transformation in its followup attribute. + /// + /// @param Attrs The loop's transformations. + /// @param LoopProperties Non-transformation properties such as debug + /// location, parallel accesses and disabled + /// transformations. These are added to the returned + /// LoopID. + /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes + /// at least one transformation. + /// + /// @return A LoopID (metadata node) that can be used for the llvm.loop + /// annotation or followup-attribute. + /// @{ + llvm::MDNode * + createPipeliningMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + llvm::MDNode * + createPartialUnrollMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + llvm::MDNode * + createUnrollAndJamMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + llvm::MDNode * + createLoopVectorizeMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + llvm::MDNode * + createLoopDistributeMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + llvm::MDNode * + createFullUnrollMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); + /// @} + + /// Create a LoopID for this loop, including transformation-unspecific + /// metadata such as debug location. + /// + /// @param Attrs This loop's attributes and transformations. + /// @param LoopProperties Additional non-transformation properties to add + /// to the LoopID, such as transformation-specific + /// metadata that are not covered by @p Attrs. + /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes + /// at least one transformation. + /// + /// @return A LoopID (metadata node) that can be used for the llvm.loop + /// annotation. + llvm::MDNode *createMetadata(const LoopAttributes &Attrs, + llvm::ArrayRef LoopProperties, + bool &HasUserTransforms); }; /// A stack of loop information corresponding to loop nesting levels. Index: cfe/trunk/lib/CodeGen/CGLoopInfo.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGLoopInfo.cpp +++ cfe/trunk/lib/CodeGen/CGLoopInfo.cpp @@ -18,138 +18,396 @@ using namespace clang::CodeGen; using namespace llvm; -static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, - const llvm::DebugLoc &StartLoc, - const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) { +MDNode * +LoopInfo::createLoopPropertiesMetadata(ArrayRef LoopProperties) { + LLVMContext &Ctx = Header->getContext(); + SmallVector NewLoopProperties; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + NewLoopProperties.push_back(TempNode.get()); + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); - if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && - Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && - Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && - Attrs.PipelineInitiationInterval == 0 && - Attrs.VectorizeEnable == LoopAttributes::Unspecified && - Attrs.UnrollEnable == LoopAttributes::Unspecified && - Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && - Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && - !EndLoc) - return nullptr; + MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties); + LoopID->replaceOperandWith(0, LoopID); + return LoopID; +} + +MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.PipelineDisabled) + Enabled = false; + else if (Attrs.PipelineInitiationInterval != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 1))})); + LoopProperties = NewLoopProperties; + } + return createLoopPropertiesMetadata(LoopProperties); + } SmallVector Args; - // Reserve operand 0 for loop id self reference. - auto TempNode = MDNode::getTemporary(Ctx, None); + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); - // If we have a valid start debug location for the loop, add it. - if (StartLoc) { - Args.push_back(StartLoc.getAsMDNode()); - - // If we also have a valid end debug location for the loop, add it. - if (EndLoc) - Args.push_back(EndLoc.getAsMDNode()); - } - - // Setting vectorize.width - if (Attrs.VectorizeWidth > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; + if (Attrs.PipelineInitiationInterval > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting interleave.count - if (Attrs.InterleaveCount > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; - Args.push_back(MDNode::get(Ctx, Vals)); - } + // No follow-up: This is the last transformation. + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.UnrollEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollEnable == LoopAttributes::Full) + Enabled = None; + else if (Attrs.UnrollEnable != LoopAttributes::Unspecified || + Attrs.UnrollCount != 0) + Enabled = true; + + if (Enabled != true) { + // createFullUnrollMetadata will already have added llvm.loop.unroll.disable + // if unrolling is disabled. + return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms); + } + + SmallVector FollowupLoopProperties; + + // Apply all loop properties to the unrolled loop. + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + + // Don't unroll an already unrolled loop. + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); // Setting unroll.count if (Attrs.UnrollCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; + llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting unroll_and_jam.count - if (Attrs.UnrollAndJamCount > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))}; + // Setting unroll.full or unroll.disable + if (Attrs.UnrollEnable == LoopAttributes::Enable) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting vectorize.enable - if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == - LoopAttributes::Enable)))}; - Args.push_back(MDNode::get(Ctx, Vals)); - } + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup})); - // Setting unroll.full or unroll.disable - if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { - std::string Name; - if (Attrs.UnrollEnable == LoopAttributes::Enable) - Name = "llvm.loop.unroll.enable"; - else if (Attrs.UnrollEnable == LoopAttributes::Full) - Name = "llvm.loop.unroll.full"; - else - Name = "llvm.loop.unroll.disable"; - Metadata *Vals[] = {MDString::get(Ctx, Name)}; - Args.push_back(MDNode::get(Ctx, Vals)); + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable || + Attrs.UnrollAndJamCount != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back(MDNode::get( + Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); + LoopProperties = NewLoopProperties; + } + return createPartialUnrollMetadata(Attrs, LoopProperties, + HasUserTransforms); } - // Setting unroll_and_jam.full or unroll_and_jam.disable - if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) { - std::string Name; - if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) - Name = "llvm.loop.unroll_and_jam.enable"; - else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full) - Name = "llvm.loop.unroll_and_jam.full"; - else - Name = "llvm.loop.unroll_and_jam.disable"; - Metadata *Vals[] = {MDString::get(Ctx, Name)}; + SmallVector FollowupLoopProperties; + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + // Setting unroll_and_jam.count + if (Attrs.UnrollAndJamCount > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.UnrollAndJamCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == - LoopAttributes::Enable)))}; + if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.IsParallel) { - AccGroup = MDNode::getDistinct(Ctx, {}); + if (FollowupHasTransforms) Args.push_back(MDNode::get( - Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); + Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"), + Followup})); + + if (UnrollAndJamInnerFollowup) + Args.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"), + UnrollAndJamInnerFollowup})); + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.VectorizeEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || + Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 0))})); + LoopProperties = NewLoopProperties; + } + return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms); + } + + // Apply all loop properties to the vectorized loop. + SmallVector FollowupLoopProperties; + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + + // Don't vectorize an already vectorized loop. + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + // Setting vectorize.width + if (Attrs.VectorizeWidth > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.vectorize.width"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.VectorizeWidth))}; + Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.PipelineDisabled) { + // Setting interleave.count + if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.pipeline.disable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))}; + MDString::get(Ctx, "llvm.loop.interleave.count"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.PipelineInitiationInterval > 0) { + // Setting vectorize.enable + if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), + MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; + llvm::Type::getInt1Ty(Ctx), + (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Set the first operand to itself. + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, + {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup})); + + MDNode *LoopID = MDNode::get(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.DistributeEnable == LoopAttributes::Disable) + Enabled = false; + if (Attrs.DistributeEnable == LoopAttributes::Enable) + Enabled = true; + + if (Enabled != true) { + SmallVector NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 0))})); + LoopProperties = NewLoopProperties; + } + return createLoopVectorizeMetadata(Attrs, LoopProperties, + HasUserTransforms); + } + + bool FollowupHasTransforms = false; + MDNode *Followup = + createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms); + + SmallVector Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), + (Attrs.DistributeEnable == LoopAttributes::Enable)))}; + Args.push_back(MDNode::get(Ctx, Vals)); + + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, + {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup})); + MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; return LoopID; } +MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs, + ArrayRef LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional Enabled; + if (Attrs.UnrollEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollEnable == LoopAttributes::Full) + Enabled = true; + + if (Enabled != true) { + SmallVector NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); + LoopProperties = NewLoopProperties; + } + return createLoopDistributeMetadata(Attrs, LoopProperties, + HasUserTransforms); + } + + SmallVector Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))); + + // No follow-up: there is no loop after full unrolling. + // TODO: Warn if there are transformations after full unrolling. + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode *LoopInfo::createMetadata( + const LoopAttributes &Attrs, + llvm::ArrayRef AdditionalLoopProperties, + bool &HasUserTransforms) { + SmallVector LoopProperties; + + // If we have a valid start debug location for the loop, add it. + if (StartLoc) { + LoopProperties.push_back(StartLoc.getAsMDNode()); + + // If we also have a valid end debug location for the loop, add it. + if (EndLoc) + LoopProperties.push_back(EndLoc.getAsMDNode()); + } + + assert(!!AccGroup == Attrs.IsParallel && + "There must be an access group iff the loop is parallel"); + if (Attrs.IsParallel) { + LLVMContext &Ctx = Header->getContext(); + LoopProperties.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); + } + + LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(), + AdditionalLoopProperties.end()); + return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); +} + LoopAttributes::LoopAttributes(bool IsParallel) : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), UnrollEnable(LoopAttributes::Unspecified), @@ -173,15 +431,114 @@ } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, - const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) - : LoopID(nullptr), Header(Header), Attrs(Attrs) { - LoopID = - createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup); + const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, + LoopInfo *Parent) + : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc), + Parent(Parent) { + + if (Attrs.IsParallel) { + // Create an access group for this loop. + LLVMContext &Ctx = Header->getContext(); + AccGroup = MDNode::getDistinct(Ctx, {}); + } + + if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && + Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && + Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && + Attrs.PipelineInitiationInterval == 0 && + Attrs.VectorizeEnable == LoopAttributes::Unspecified && + Attrs.UnrollEnable == LoopAttributes::Unspecified && + Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && + Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && + !EndLoc) + return; + + TempLoopID = MDNode::getTemporary(Header->getContext(), None); +} + +void LoopInfo::finish() { + // We did not annotate the loop body instructions because there are no + // attributes for this loop. + if (!TempLoopID) + return; + + MDNode *LoopID; + LoopAttributes CurLoopAttr = Attrs; + LLVMContext &Ctx = Header->getContext(); + + if (Parent && (Parent->Attrs.UnrollAndJamEnable || + Parent->Attrs.UnrollAndJamCount != 0)) { + // Parent unroll-and-jams this loop. + // Split the transformations in those that happens before the unroll-and-jam + // and those after. + + LoopAttributes BeforeJam, AfterJam; + + BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; + + BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; + BeforeJam.InterleaveCount = Attrs.InterleaveCount; + BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; + BeforeJam.DistributeEnable = Attrs.DistributeEnable; + + switch (Attrs.UnrollEnable) { + case LoopAttributes::Unspecified: + case LoopAttributes::Disable: + BeforeJam.UnrollEnable = Attrs.UnrollEnable; + AfterJam.UnrollEnable = Attrs.UnrollEnable; + break; + case LoopAttributes::Full: + BeforeJam.UnrollEnable = LoopAttributes::Full; + break; + case LoopAttributes::Enable: + AfterJam.UnrollEnable = LoopAttributes::Enable; + break; + } + + AfterJam.UnrollCount = Attrs.UnrollCount; + AfterJam.PipelineDisabled = Attrs.PipelineDisabled; + AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval; + + // If this loop is subject of an unroll-and-jam by the parent loop, and has + // an unroll-and-jam annotation itself, we have to decide whether to first + // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The + // UnrollAndJam pass processes loops from inner to outer, so we apply the + // inner first. + BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount; + BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable; + + // Set the inner followup metadata to process by the outer loop. Only + // consider the first inner loop. + if (!Parent->UnrollAndJamInnerFollowup) { + // Splitting the attributes into a BeforeJam and an AfterJam part will + // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam) + // to be forwarded to the AfterJam part. We detect the situation here and + // add it manually. + SmallVector BeforeLoopProperties; + if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || + BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0) + BeforeLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); + + bool InnerFollowupHasTransform = false; + MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties, + InnerFollowupHasTransform); + if (InnerFollowupHasTransform) + Parent->UnrollAndJamInnerFollowup = InnerFollowup; + } + + CurLoopAttr = BeforeJam; + } + + bool HasUserTransforms = false; + LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms); + TempLoopID->replaceAllUsesWith(LoopID); } void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) { - Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc)); + Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, + Active.empty() ? nullptr : &Active.back())); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); } @@ -364,6 +721,7 @@ void LoopInfoStack::pop() { assert(!Active.empty() && "No active loops to pop"); + Active.back().finish(); Active.pop_back(); } Index: cfe/trunk/test/CodeGenCXX/pragma-followup_inner.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-followup_inner.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-followup_inner.cpp @@ -0,0 +1,42 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s + +extern "C" void followup_inner(int n, int *x) { +#pragma unroll_and_jam(4) + for(int j = 0; j < n; j++) { +#pragma clang loop pipeline_initiation_interval(10) +#pragma clang loop unroll_count(4) +#pragma clang loop vectorize(assume_safety) +#pragma clang loop distribute(enable) + for(int i = 0; i < n; i++) + x[j+i*n] = 10; + + } +} + + +// CHECK-LABEL: define void @followup_inner +// CHECK: br label %for.cond1, !llvm.loop ![[INNERLOOP_3:[0-9]+]] +// CHECK: br label %for.cond, !llvm.loop ![[OUTERLOOP_9:[0-9]+]] + +// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{} + +// CHECK-DAG: ![[INNERLOOP_3:[0-9]+]] = distinct !{![[INNERLOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]} +// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", !2} +// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true} +// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]]} +// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true} + +// CHECK-DAG: ![[OUTERLOOP_9:[0-9]+]] = distinct !{![[OUTERLOOP_9:[0-9]+]], ![[UNROLLANDJAM_COUNT_10:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]]} +// CHECK-DAG: ![[UNROLLANDJAM_COUNT_10:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.count", i32 4} +// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_inner", !12} + +// CHECK-DAG: ![[LOOP_12:[0-9]+]] = distinct !{![[LOOP_12:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_COUNT_13:[0-9]+]], ![[UNROLL_FOLLOWUP_14:[0-9]+]]} +// CHECK-DAG: ![[ISVECTORIZED_13:[0-9]+]] = !{!"llvm.loop.isvectorized"} +// CHECK-DAG: ![[UNROLL_COUNT_13:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4} +// CHECK-DAG: ![[UNROLL_FOLLOWUP_14:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_15:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_15:[0-9]+]] = distinct !{![[LOOP_15:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_DISABLE_16:[0-9]+]], ![[PIPELINE_17:[0-9]+]]} +// CHECK-DAG: ![[UNROLL_DISABLE_16:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[PIPELINE_17:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10} Index: cfe/trunk/test/CodeGenCXX/pragma-followup_outer.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-followup_outer.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-followup_outer.cpp @@ -0,0 +1,41 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s + +extern "C" void followup_outer(int n, int *x) { +#pragma clang loop pipeline_initiation_interval(10) +#pragma clang loop unroll_count(4) +#pragma unroll_and_jam +#pragma clang loop vectorize(assume_safety) +#pragma clang loop distribute(enable) + for(int j = 0; j < n; j++) { + x[j] = 10; + } +} + + +// CHECK-LABEL: define void @followup_outer +// CHECK: br label %for.cond, !llvm.loop ![[LOOP_3:[0-9]+]] + +// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{} + +// CHECK-DAG: ![[LOOP_3:[0-9]+]] = distinct !{![[LOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]} +// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2]]} +// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true} +// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]], ![[VECTORIZE_FOLLOWUP_9:[0-9]+]]} +// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_9:[0-9]+]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_10:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_10:[0-9]+]] = distinct !{![[LOOP_10:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_12:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]]} +// CHECK-DAG: ![[ISVECTORIZED_11:[0-9]+]] = !{!"llvm.loop.isvectorized"} +// CHECK-DAG: ![[UNROLLANDJAM_12:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.enable"} +// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_14:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_14:[0-9]+]] = distinct !{![[LOOP_14:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_COUNT_16:[0-9]+]], ![[UNROLL_FOLLOWUP_17:[0-9]+]]} +// CHECK-DAG: ![[UNROLLANDJAM_DISABLE_15:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.disable"} +// CHECK-DAG: ![[UNROLL_COUNT_16:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4} +// CHECK-DAG: ![[UNROLL_FOLLOWUP_17:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_18:[0-9]+]]} + +// CHECK-DAG: ![[LOOP_18:[0-9]+]] = distinct !{![[LOOP_18:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_DISABLE_19:[0-9]+]], ![[INITIATIONINTERVAL_20:[0-9]+]]} +// CHECK-DAG: ![[UNROLL_DISABLE_19:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[INITIATIONINTERVAL_20:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10} Index: cfe/trunk/test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp @@ -26,7 +26,7 @@ // CHECK: ![[ACCESS_GROUP_2]] = distinct !{} // CHECK: ![[ACCESS_GROUP_LIST_3:[0-9]+]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]} // CHECK: ![[ACCESS_GROUP_4]] = distinct !{} -// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]} +// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]] // CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]} -// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]} +// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]] // CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} Index: cfe/trunk/test/CodeGenCXX/pragma-loop-safety-nested.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-loop-safety-nested.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-loop-safety-nested.cpp @@ -21,7 +21,7 @@ // CHECK: ![[ACCESS_GROUP_2]] = distinct !{} // CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]} // CHECK: ![[ACCESS_GROUP_4]] = distinct !{} -// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]} +// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]] // CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]} -// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]} +// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]] // CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} Index: cfe/trunk/test/CodeGenCXX/pragma-loop-safety-outer.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-loop-safety-outer.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-loop-safety-outer.cpp @@ -18,5 +18,5 @@ // CHECK: ![[ACCESS_GROUP_2]] = distinct !{} // CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], -// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_9:[0-9]+]]} +// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_9:[0-9]+]] // CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} Index: cfe/trunk/test/CodeGenCXX/pragma-loop-safety.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-loop-safety.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-loop-safety.cpp @@ -47,12 +47,12 @@ } // CHECK: ![[ACCESS_GROUP_2]] = distinct !{} -// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]]} +// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]} +// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} +// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"} // CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1} // CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true} -// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"} -// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} // CHECK: ![[ACCESS_GROUP_8]] = distinct !{} -// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]], ![[UNROLL_DISABLE]], ![[PARALLEL_ACCESSES_11:[0-9]+]]} -// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} +// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]} // CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]} +// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} Index: cfe/trunk/test/CodeGenCXX/pragma-loop.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-loop.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-loop.cpp @@ -158,37 +158,60 @@ for_template_constant_expression_test(List, Length); } -// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]], ![[DISTRIBUTE_ENABLE:.*]]} -// CHECK: ![[WIDTH_4]] = !{!"llvm.loop.vectorize.width", i32 4} -// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4} -// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[UNROLL_FULL:.*]]} // CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"} -// CHECK: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true} -// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]} -// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8} + +// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]]} // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"} // CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false} -// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]], ![[INTENABLE_1:.*]]} +// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8} +// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4} + +// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[FOLLOWUP_VECTOR_3:.*]]} +// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]} +// CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} +// CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"} // CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8} + // CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]]} // CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2} // CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2} -// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]} + +// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]} // CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} -// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]} + +// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]} +// CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]} +// CHECK: ![[AFTER_VECTOR_6]] = distinct !{![[AFTER_VECTOR_6]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} + // CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[WIDTH_5:.*]]} // CHECK: ![[WIDTH_5]] = !{!"llvm.loop.vectorize.width", i32 5} + // CHECK: ![[LOOP_8]] = distinct !{![[LOOP_8]], ![[WIDTH_5:.*]]} -// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[UNROLL_8:.*]]} -// CHECK: ![[INTERLEAVE_8]] = !{!"llvm.loop.interleave.count", i32 8} -// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]} -// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]]} -// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[UNROLL_24:.*]]} -// CHECK: ![[WIDTH_6]] = !{!"llvm.loop.vectorize.width", i32 6} -// CHECK: ![[INTERLEAVE_10]] = !{!"llvm.loop.interleave.count", i32 10} + +// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[FOLLOWUP_VECTOR_9:.*]]} +// CHECK: ![[FOLLOWUP_VECTOR_9]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_9:.*]]} +// CHECK: ![[AFTER_VECTOR_9]] = distinct !{![[AFTER_VECTOR_9]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} + +// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_10:.*]]} +// CHECK: ![[FOLLOWUP_VECTOR_10]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_10:.*]]} +// CHECK: ![[AFTER_VECTOR_10]] = distinct !{![[AFTER_VECTOR_10]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} + +// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[FOLLOWUP_VECTOR_11:.*]]} +// CHECK: ![[FOLLOWUP_VECTOR_11]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_11:.*]]} +// CHECK: ![[AFTER_VECTOR_11]] = distinct !{![[AFTER_VECTOR_11]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} + +// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[FOLLOWUP_VECTOR_12:.*]]} +// CHECK: ![[FOLLOWUP_VECTOR_12]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_12:.*]]} +// CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]} // CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24} -// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[UNROLL_32:.*]]} + +// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[FOLLOWUP_VECTOR_13:.*]]} // CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16} +// CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]} +// CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]} // CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32} + // CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], ![[WIDTH_10:.*]]} // CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10} Index: cfe/trunk/test/CodeGenCXX/pragma-unroll-and-jam.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/pragma-unroll-and-jam.cpp +++ cfe/trunk/test/CodeGenCXX/pragma-unroll-and-jam.cpp @@ -51,5 +51,5 @@ // CHECK: ![[UNJ_4]] = !{!"llvm.loop.unroll_and_jam.count", i32 4} // CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[UNJ_DISABLE:.*]]} // CHECK: ![[UNJ_DISABLE]] = !{!"llvm.loop.unroll_and_jam.disable"} -// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNROLL_4:.*]], ![[UNJ_DISABLE:.*]]} +// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNJ_DISABLE:.*]], ![[UNROLL_4:.*]]} // CHECK: ![[UNROLL_4]] = !{!"llvm.loop.unroll.count", i32 4} Index: cfe/trunk/test/OpenMP/simd_metadata.c =================================================================== --- cfe/trunk/test/OpenMP/simd_metadata.c +++ cfe/trunk/test/OpenMP/simd_metadata.c @@ -147,16 +147,16 @@ // CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8:![0-9]+]], [[LOOP_VEC_ENABLE]]} // CHECK: [[LOOP_WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8} // CHECK: ![[ACCESS_GROUP_7]] = distinct !{} -// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_9:[0-9]+]]} +// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], ![[PARALLEL_ACCESSES_9:[0-9]+]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]]} // CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_7]]} // // Metadata for h2: // CHECK: ![[ACCESS_GROUP_10]] = distinct !{} -// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_12:[0-9]+]]} +// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], ![[PARALLEL_ACCESSES_12:[0-9]+]], [[LOOP_VEC_ENABLE]]} // CHECK: ![[PARALLEL_ACCESSES_12]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_10]]} // // Metadata for h3: // CHECK: ![[ACCESS_GROUP_13]] = distinct !{} -// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_15:[0-9]+]]} +// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], ![[PARALLEL_ACCESSES_15:[0-9]+]], [[LOOP_VEC_ENABLE]]} // CHECK: ![[PARALLEL_ACCESSES_15]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_13]]} //