diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4066,13 +4066,10 @@ // original loop is widened to a vector form so we can use them to construct // the incoming edges. VPBasicBlock *Header = State.Plan->getEntry()->getEntryBasicBlock(); - for (VPRecipeBase &R : *Header) { - if (isa(&R)) - continue; + for (VPRecipeBase &R : Header->phis()) { VPWidenPHIRecipe *PhiR = dyn_cast(&R); if (!PhiR) - break; - + continue; auto *OrigPhi = cast(PhiR->getUnderlyingValue()); if (PhiR->getRecurrenceDescriptor()) { fixReduction(PhiR, State); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -685,6 +685,12 @@ /// Returns true if the recipe may have side-effects. bool mayHaveSideEffects() const; + + /// Returns true for PHI-like recipes. + bool isPhi() const { + return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC || + getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC; + } }; inline bool VPUser::classof(const VPDef *Def) { @@ -1433,7 +1439,7 @@ /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It /// holds a sequence of zero or more VPRecipe's each representing a sequence of -/// output IR instructions. +/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes. class VPBasicBlock : public VPBlockBase { public: using RecipeListTy = iplist; @@ -1511,6 +1517,11 @@ /// Return the position of the first non-phi node recipe in the block. iterator getFirstNonPhi(); + /// Returns an iterator range over the PHI-like recipes in the block. + iterator_range phis() { + return make_range(begin(), getFirstNonPhi()); + } + void dropAllReferences(VPValue *NewValue) override; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -249,10 +249,7 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() { iterator It = begin(); - while (It != end() && (isa(&*It) || - isa(&*It) || - isa(&*It) || - isa(&*It))) + while (It != end() && It->isPhi()) It++; return It; }