[VPlan] Use VPIRFlags for VPWidenGEPRecipe (NFCI).

[fhahn]

Summary

Extend VPIRFlags to also include InBounds and use for VPWidenGEPRecipe.

The last remaining recipe that needs updating for
MayGeneratePoisonRecipes is VPReplicateRecipe.

Depends on D149081.

[Ayal]

LGTM, pending points raised in D149081 review.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

803

Call transferFlags() instead, if only to be consistent and exercised?

[fhahn]

rebased and updated to use transferFlags.

[fhahn]

Rebase after recent changes to D149079.

[fhahn]

Rebased on top of D149079.

[fhahn]

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

803

Updated, thanks!

[Ayal]

LGTM with a couple of minor nits.

llvm/lib/Transforms/Vectorize/VPlan.h

945–946

Just to be consistent, GEP is introduced between Exact and FMF in all that follows.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

838

Simpler in this case to pass GEPFlags.IsInBounds to CreateGEP()?

[Ayal]

(post-commit: note last minor nits, which can be discarded if preferred.)

[fhahn]

In D149082#4329459, @Ayal wrote:

(post-commit: note last minor nits, which can be discarded if preferred.)

Sorry I missed those, should be addressed in c096e91735397e2f2892ede9ece50bf4693d94ac

llvm/lib/Transforms/Vectorize/VPlan.h

#endif
   }
 };
 
 /// Class to record LLVM IR flag for a recipe along with it.
 class VPRecipeWithIRFlags : public VPRecipeBase {
   enum class OperationType : unsigned char {
     OverflowingBinOp,
     PossiblyExactOp,
     FPMathOp,
+    GEPOp,
     Other
   };
   struct WrapFlagsTy {
     char HasNUW : 1;
     char HasNSW : 1;
   };
   struct ExactFlagsTy {
     char IsExact : 1;
   };
+  struct GEPFlagsTy {
+    char IsInBounds : 1;
+  };
   struct FastMathFlagsTy {
     char AllowReassoc : 1;
     char NoNaNs : 1;
     char NoInfs : 1;
     char NoSignedZeros : 1;
     char AllowReciprocal : 1;
     char AllowContract : 1;
     char ApproxFunc : 1;
   };
 
   OperationType OpType;
 
   union {
     WrapFlagsTy WrapFlags;
     ExactFlagsTy ExactFlags;
+    GEPFlagsTy GEPFlags;
     FastMathFlagsTy FMFs;
     unsigned char AllFlags;
   };
 
 public:
   template <typename IterT>
   VPRecipeWithIRFlags(const unsigned char SC, iterator_range<IterT> Operands)
       : VPRecipeBase(SC, Operands) {
     OpType = OperationType::Other;
     AllFlags = 0;
   }
 
   template <typename IterT>
   VPRecipeWithIRFlags(const unsigned char SC, iterator_range<IterT> Operands,
                       Instruction &I)
       : VPRecipeWithIRFlags(SC, Operands) {
     if (auto *Op = dyn_cast<OverflowingBinaryOperator>(&I)) {
       OpType = OperationType::OverflowingBinOp;
       WrapFlags.HasNUW = Op->hasNoUnsignedWrap();
       WrapFlags.HasNSW = Op->hasNoSignedWrap();
     } else if (auto *Op = dyn_cast<PossiblyExactOperator>(&I)) {
       OpType = OperationType::PossiblyExactOp;
       ExactFlags.IsExact = Op->isExact();
+    } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
+      OpType = OperationType::GEPOp;
+      GEPFlags.IsInBounds = GEP->isInBounds();
     } else if (auto *Op = dyn_cast<FPMathOperator>(&I)) {
       OpType = OperationType::FPMathOp;
       FastMathFlags FMF = Op->getFastMathFlags();
       FMFs.AllowReassoc = FMF.allowReassoc();
       FMFs.NoNaNs = FMF.noNaNs();
       FMFs.NoInfs = FMF.noInfs();
       FMFs.NoSignedZeros = FMF.noSignedZeros();
       FMFs.AllowReciprocal = FMF.allowReciprocal();
       FMFs.AllowContract = FMF.allowContract();
       FMFs.ApproxFunc = FMF.approxFunc();
     }
   }
 
   static inline bool classof(const VPRecipeBase *R) {
-    return R->getVPDefID() == VPRecipeBase::VPWidenSC;
+    return R->getVPDefID() == VPRecipeBase::VPWidenSC ||
+           R->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
   }
 
   /// Drop all poison-generating flags.
   void dropPoisonGeneratingFlags() {
     // NOTE: This needs to be kept in-sync with
     // Instruction::dropPoisonGeneratingFlags.
     switch (OpType) {
     case OperationType::OverflowingBinOp:
       WrapFlags.HasNUW = false;
       WrapFlags.HasNSW = false;
       break;
     case OperationType::PossiblyExactOp:
       ExactFlags.IsExact = false;
       break;
+    case OperationType::GEPOp:
+      GEPFlags.IsInBounds = false;
+      break;
     case OperationType::FPMathOp:
       FMFs.NoNaNs = false;
       FMFs.NoInfs = false;
       break;
     case OperationType::Other:
       break;
     }
   }
 
   /// Set the IR flags for \p I.
   void setFlags(Instruction *I) const {
     switch (OpType) {
     case OperationType::OverflowingBinOp:
       I->setHasNoUnsignedWrap(WrapFlags.HasNUW);
       I->setHasNoSignedWrap(WrapFlags.HasNSW);
       break;
     case OperationType::PossiblyExactOp:
       I->setIsExact(ExactFlags.IsExact);
       break;
+    case OperationType::GEPOp:
+      cast<GetElementPtrInst>(I)->setIsInBounds(GEPFlags.IsInBounds);
+      break;
     case OperationType::FPMathOp:
       I->setHasAllowReassoc(FMFs.AllowReassoc);
       I->setHasNoNaNs(FMFs.NoNaNs);
       I->setHasNoInfs(FMFs.NoInfs);
       I->setHasNoSignedZeros(FMFs.NoSignedZeros);
       I->setHasAllowReciprocal(FMFs.AllowReciprocal);
       I->setHasAllowContract(FMFs.AllowContract);
       I->setHasApproxFunc(FMFs.ApproxFunc);
#endif
   }
 
   bool isInvariantCond() const {
     return getCond()->isDefinedOutsideVectorRegions();
   }
 };
 
 /// A recipe for handling GEP instructions.
-class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
+class VPWidenGEPRecipe : public VPRecipeWithIRFlags, public VPValue {
   bool isPointerLoopInvariant() const {
     return getOperand(0)->isDefinedOutsideVectorRegions();
   }
 
   bool isIndexLoopInvariant(unsigned I) const {
     return getOperand(I + 1)->isDefinedOutsideVectorRegions();
   }
 
   bool areAllOperandsInvariant() const {
     return all_of(operands(), [](VPValue *Op) {
       return Op->isDefinedOutsideVectorRegions();
     });
   }
 
 public:
   template <typename IterT>
   VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range<IterT> Operands)
-      : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP) {}
+      : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP),
+        VPValue(this, GEP) {}
 
   ~VPWidenGEPRecipe() override = default;
 
   VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
 
   /// Generate the gep nodes.
   void execute(VPTransformState &State) override;
 

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

if (areAllOperandsInvariant()) {
     // Here, we broadcast a clone of the original.
     //
     // TODO: If at some point we decide to scalarize instructions having
     //       loop-invariant operands, this special case will no longer be
     //       required. We would add the scalarization decision to
     //       collectLoopScalars() and teach getVectorValue() to broadcast
     //       the lane-zero scalar value.
     auto *Clone = State.Builder.Insert(GEP->clone());
+    setFlags(Clone);
     for (unsigned Part = 0; Part < State.UF; ++Part) {
       Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
       State.set(this, EntryPart, Part);
       State.addMetadata(EntryPart, GEP);
     }
   } else {
     // If the GEP has at least one loop-varying operand, we are sure to
     // produce a vector of pointers. But if we are only unrolling, we want
for (unsigned Part = 0; Part < State.UF; ++Part) {
       for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
         VPValue *Operand = getOperand(I);
         if (isIndexLoopInvariant(I - 1))
           Indices.push_back(State.get(Operand, VPIteration(0, 0)));
         else
           Indices.push_back(State.get(Operand, Part));
       }
 
-      // If the GEP instruction is vectorized and was in a basic block that
-      // needed predication, we can't propagate the poison-generating 'inbounds'
-      // flag. The control flow has been linearized and the GEP is no longer
-      // guarded by the predicate, which could make the 'inbounds' properties to
-      // no longer hold.
-      bool IsInBounds =
-          GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;
-
       // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
       // but it should be a vector, otherwise.
       auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
-                                             Indices, "", IsInBounds);
+                                             Indices, "");
+      if (auto *I = dyn_cast<GetElementPtrInst>(NewGEP))
+        setFlags(I);
       assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
              "NewGEP is not a pointer vector");
       State.set(this, NewGEP, Part);
       State.addMetadata(NewGEP, GEP);
     }
   }
 }