[VPlan] Verify that header only contains header phi recipes.

[fhahn]

Summary

Add verification that VPHeaderPHIRecipes are only in header VPBBs. Also
adds missing checks for VPPointerInductionRecipe to
VPHeaderPHIRecipe::classof.

Split off from D119661.

[Ayal]

Thanks for addressing, adding a couple of minor suggestions.

llvm/lib/Transforms/Vectorize/VPlan.h

1137

Phi-like recipes are kept in order to facilitate checking intervals (and help avoid such missed-case bugs)?

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

136–139

Perhaps worth outlining the first part that verifies phi recipes, breaking the somewhat long method into two?

[fhahn]

llvm/lib/Transforms/Vectorize/VPlan.h

1137

I'll do that separately.

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

136–139

Done!

llvm/lib/Transforms/Vectorize/VPlan.h

if (Start)
       addOperand(Start);
   }
 
 public:
   ~VPHeaderPHIRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *B) {
     return B->getVPDefID() == VPRecipeBase::VPCanonicalIVPHISC ||
            B->getVPDefID() == VPRecipeBase::VPActiveLaneMaskPHISC ||
            B->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC ||
            B->getVPDefID() == VPRecipeBase::VPReductionPHISC ||
            B->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC ||
+           B->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC ||
            B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
   }
   static inline bool classof(const VPValue *V) {
     return V->getVPValueID() == VPValue::VPVCanonicalIVPHISC ||
            V->getVPValueID() == VPValue::VPVActiveLaneMaskPHISC ||
            V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC ||
            V->getVPValueID() == VPValue::VPVReductionPHISC ||
            V->getVPValueID() == VPValue::VPVWidenIntOrFpInductionSC ||
+           V->getVPValueID() == VPValue::VPVWidenPointerInductionSC ||
            V->getVPValueID() == VPValue::VPVWidenPHISC;
   }
 
   /// Generate the phi nodes.
   void execute(VPTransformState &State) override = 0;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

void VPlanVerifier::verifyHierarchicalCFG(
   if (!EnableHCFGVerifier)
     return;
 
   LLVM_DEBUG(dbgs() << "Verifying VPlan H-CFG.\n");
   assert(!TopRegion->getParent() && "VPlan Top Region should have no parent.");
   verifyRegionRec(TopRegion);
 }
 
-static bool
-verifyVPBasicBlock(const VPBasicBlock *VPBB,
-                   DenseMap<const VPBlockBase *, unsigned> &BlockNumbering) {
-  // Verify that phi-like recipes are at the beginning of the block, with no
-  // other recipes in between.
+// Verify that phi-like recipes are at the beginning of \p VPBB, with no
+// other recipes in between. Also check that only header blocks contain
+// VPHeaderPHIRecipes.
+static bool verifyPhiRecipes(const VPBasicBlock *VPBB) {
   auto RecipeI = VPBB->begin();
   auto End = VPBB->end();
   unsigned NumActiveLaneMaskPhiRecipes = 0;
+  const VPRegionBlock *ParentR = VPBB->getParent();
+  bool IsHeaderVPBB = ParentR && !ParentR->isReplicator() &&
+                      ParentR->getEntryBasicBlock() == VPBB;
   while (RecipeI != End && RecipeI->isPhi()) {
     if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
       NumActiveLaneMaskPhiRecipes++;
+
+    if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe>(*RecipeI)) {
+      errs() << "Found non-header PHI recipe in header VPBB";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+      errs() << ": ";
+      RecipeI->dump();
+#endif
+      return false;
+    }
+
+    if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
+      errs() << "Found header PHI recipe in non-header VPBB";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+      errs() << ": ";
+      RecipeI->dump();
+#endif
+      return false;
+    }
+
     RecipeI++;
   }
 
   if (NumActiveLaneMaskPhiRecipes > 1) {
     errs() << "There should be no more than one VPActiveLaneMaskPHIRecipe";
     return false;
   }
 
   while (RecipeI != End) {
     if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
       errs() << "Found phi-like recipe after non-phi recipe";
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
       errs() << ": ";
       RecipeI->dump();
       errs() << "after\n";
       std::prev(RecipeI)->dump();
 #endif
       return false;
     }
     RecipeI++;
   }
+  return true;
+}
+
+static bool
+verifyVPBasicBlock(const VPBasicBlock *VPBB,
+                   DenseMap<const VPBlockBase *, unsigned> &BlockNumbering) {
+  if (!verifyPhiRecipes(VPBB))
+    return false;
 
   // Verify that defs in VPBB dominate all their uses. The current
   // implementation is still incomplete.
   DenseMap<const VPRecipeBase *, unsigned> RecipeNumbering;
   unsigned Cnt = 0;
   for (const VPRecipeBase &R : *VPBB)
     RecipeNumbering[&R] = Cnt++;