diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1139,6 +1139,7 @@ B->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC || B->getVPDefID() == VPRecipeBase::VPReductionPHISC || B->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC || + B->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC || B->getVPDefID() == VPRecipeBase::VPWidenPHISC; } static inline bool classof(const VPValue *V) { @@ -1147,6 +1148,7 @@ V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC || V->getVPValueID() == VPValue::VPVReductionPHISC || V->getVPValueID() == VPValue::VPVWidenIntOrFpInductionSC || + V->getVPValueID() == VPValue::VPVWidenPointerInductionSC || V->getVPValueID() == VPValue::VPVWidenPHISC; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -133,17 +133,38 @@ verifyRegionRec(TopRegion); } -static bool -verifyVPBasicBlock(const VPBasicBlock *VPBB, - DenseMap &BlockNumbering) { - // Verify that phi-like recipes are at the beginning of the block, with no - // other recipes in between. +// Verify that phi-like recipes are at the beginning of \p VPBB, with no +// other recipes in between. Also check that only header blocks contain +// VPHeaderPHIRecipes. +static bool verifyPhiRecipes(const VPBasicBlock *VPBB) { auto RecipeI = VPBB->begin(); auto End = VPBB->end(); unsigned NumActiveLaneMaskPhiRecipes = 0; + const VPRegionBlock *ParentR = VPBB->getParent(); + bool IsHeaderVPBB = ParentR && !ParentR->isReplicator() && + ParentR->getEntryBasicBlock() == VPBB; while (RecipeI != End && RecipeI->isPhi()) { if (isa(RecipeI)) NumActiveLaneMaskPhiRecipes++; + + if (IsHeaderVPBB && !isa(*RecipeI)) { + errs() << "Found non-header PHI recipe in header VPBB"; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + errs() << ": "; + RecipeI->dump(); +#endif + return false; + } + + if (!IsHeaderVPBB && isa(*RecipeI)) { + errs() << "Found header PHI recipe in non-header VPBB"; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + errs() << ": "; + RecipeI->dump(); +#endif + return false; + } + RecipeI++; } @@ -166,6 +187,14 @@ } RecipeI++; } + return true; +} + +static bool +verifyVPBasicBlock(const VPBasicBlock *VPBB, + DenseMap &BlockNumbering) { + if (!verifyPhiRecipes(VPBB)) + return false; // Verify that defs in VPBB dominate all their uses. The current // implementation is still incomplete.