diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -738,11 +738,6 @@ /// Returns true if the recipe may have side-effects. bool mayHaveSideEffects() const; - /// Returns true for PHI-like recipes. - bool isPhi() const { - return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC; - } - /// Returns true if the recipe may read from memory. bool mayReadFromMemory() const; @@ -1134,12 +1129,12 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *B) { return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC && - B->getVPDefID() <= VPDef::VPLastPHISC; + B->getVPDefID() <= VPDef::VPLastHeaderPHISC; } static inline bool classof(const VPValue *V) { auto *B = V->getDefiningRecipe(); return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC && - B->getVPDefID() <= VPRecipeBase::VPLastPHISC; + B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC; } /// Generate the phi nodes. @@ -2722,6 +2717,17 @@ } } // end namespace vputils +struct VPRecipeUtils { + /// Returns true for PHI-like recipes. + static bool isPhi(const VPRecipeBase &R); + + /// Returns true for PHI-like recipes that generate their own backedge + static bool isPhiThatGeneratesBackedge(const VPRecipeBase &R); + + /// Returns true for PHI-like recipes that exists in vector loop header basic + /// block + static bool isHeaderPhi(const VPRecipeBase &R); +}; } // end namespace llvm #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -206,7 +206,7 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() { iterator It = begin(); - while (It != end() && It->isPhi()) + while (It != end() && VPRecipeUtils::isPhi(*It)) It++; return It; } @@ -684,8 +684,7 @@ // phis in the vector loop. VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock(); for (VPRecipeBase &R : Header->phis()) { - // Skip phi-like recipes that generate their backedege values themselves. - if (isa(&R)) + if (VPRecipeUtils::isPhiThatGeneratesBackedge(R)) continue; if (isa(&R) || @@ -713,6 +712,7 @@ continue; } + assert(isa(&R) && "R must be a VPHeaderPHIRecipe"); auto *PhiR = cast(&R); // For canonical IV, first-order recurrences and in-order reduction phis, // only a single part is generated, which provides the last part from the @@ -1050,7 +1050,7 @@ InterleavedAccessInfo &IAI) { if (VPBasicBlock *VPBB = dyn_cast(Block)) { for (VPRecipeBase &VPI : *VPBB) { - if (isa(&VPI)) + if (VPRecipeUtils::isHeaderPhi(VPI)) continue; assert(isa(&VPI) && "Can only handle VPInstructions"); auto *VPInst = cast(&VPI); @@ -1128,3 +1128,16 @@ Preheader->appendRecipe(Step); return Step; } + +bool VPRecipeUtils::isPhi(const VPRecipeBase &R) { + return isa(&R) || isa(&R) || + isa(&R) || isa(&R); +} + +bool VPRecipeUtils::isPhiThatGeneratesBackedge(const VPRecipeBase &R) { + return isa(&R); +} + +bool VPRecipeUtils::isHeaderPhi(const VPRecipeBase &R) { + return isa(&R) || isa(&R); +} diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -590,7 +590,7 @@ assert( SinkCandidate != Previous && "The previous value cannot depend on the users of the recurrence phi."); - if (isa(SinkCandidate) || + if (VPRecipeUtils::isHeaderPhi(*SinkCandidate) || !Seen.insert(SinkCandidate).second || properlyDominates(Previous, SinkCandidate, VPDT)) return; @@ -707,7 +707,8 @@ InsertBlock = new VPBasicBlock(Region->getName() + ".succ"); VPBlockUtils::insertBlockAfter(InsertBlock, Region); } - if (Region || isa(Previous)) + if (Region || isa(Previous) || + isa(Previous)) Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi()); else Builder.setInsertPoint(InsertBlock, std::next(Previous->getIterator())); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -346,21 +346,20 @@ VPWidenMemoryInstructionSC, VPWidenSC, VPWidenSelectSC, - - // Phi-like recipes. Need to be kept together. VPBlendSC, VPPredInstPHISC, - // Header-phi recipes. Need to be kept together. + VPWidenIntOrFpInductionSC, + // START: SubclassID for recipes that inherit VPHeaderPHIRecipe. + // VPHeaderPHIRecipe need to be kept together. VPCanonicalIVPHISC, VPActiveLaneMaskPHISC, VPFirstOrderRecurrencePHISC, VPWidenPHISC, - VPWidenIntOrFpInductionSC, VPWidenPointerInductionSC, VPReductionPHISC, - VPFirstPHISC = VPBlendSC, + // END: SubclassID for recipes that inherit VPHeaderPHIRecipe VPFirstHeaderPHISC = VPCanonicalIVPHISC, - VPLastPHISC = VPReductionPHISC, + VPLastHeaderPHISC = VPReductionPHISC, }; VPDef(const unsigned char SC) : SubclassID(SC) {} diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -143,11 +143,11 @@ const VPRegionBlock *ParentR = VPBB->getParent(); bool IsHeaderVPBB = ParentR && !ParentR->isReplicator() && ParentR->getEntryBasicBlock() == VPBB; - while (RecipeI != End && RecipeI->isPhi()) { + while (RecipeI != End && VPRecipeUtils::isPhi(*RecipeI)) { if (isa(RecipeI)) NumActiveLaneMaskPhiRecipes++; - if (IsHeaderVPBB && !isa(*RecipeI)) { + if (IsHeaderVPBB && !VPRecipeUtils::isHeaderPhi(*RecipeI)) { errs() << "Found non-header PHI recipe in header VPBB"; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) errs() << ": "; @@ -156,7 +156,7 @@ return false; } - if (!IsHeaderVPBB && isa(*RecipeI)) { + if (!IsHeaderVPBB && VPRecipeUtils::isHeaderPhi(*RecipeI)) { errs() << "Found header PHI recipe in non-header VPBB"; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) errs() << ": "; @@ -174,7 +174,7 @@ } while (RecipeI != End) { - if (RecipeI->isPhi() && !isa(&*RecipeI)) { + if (VPRecipeUtils::isPhi(*RecipeI) && !isa(&*RecipeI)) { errs() << "Found phi-like recipe after non-phi recipe"; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -207,7 +207,8 @@ for (const VPUser *U : V->users()) { auto *UI = dyn_cast(U); // TODO: check dominance of incoming values for phis properly. - if (!UI || isa(UI) || isa(UI)) + if (!UI || VPRecipeUtils::isHeaderPhi(*UI) || + isa(UI)) continue; // If the user is in the same block, check it comes after R in the