diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8791,7 +8791,7 @@ if (auto GEP = dyn_cast(Instr)) return toVPRecipeResult(new VPWidenGEPRecipe( - GEP, make_range(Operands.begin(), Operands.end()), OrigLoop)); + GEP, make_range(Operands.begin(), Operands.end()))); if (auto *SI = dyn_cast(Instr)) { bool InvariantCond = diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -990,25 +990,25 @@ /// A recipe for handling GEP instructions. class VPWidenGEPRecipe : public VPRecipeBase, public VPValue { - bool IsPtrLoopInvariant; - SmallBitVector IsIndexLoopInvariant; + bool isPointerLoopInvariant() const { + return getOperand(0)->isDefinedOutsideVectorRegions(); + } + + bool isIndexLoopInvariant(unsigned I) const { + return getOperand(I + 1)->isDefinedOutsideVectorRegions(); + } + + bool areAllOperandsInvariant() const { + return all_of(operands(), [](VPValue *Op) { + return Op->isDefinedOutsideVectorRegions(); + }); + } public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP), - IsIndexLoopInvariant(GEP->getNumIndices(), false) {} + : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP) {} - template - VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, - Loop *OrigLoop) - : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP), - IsIndexLoopInvariant(GEP->getNumIndices(), false) { - IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); - for (auto Index : enumerate(GEP->indices())) - IsIndexLoopInvariant[Index.index()] = - OrigLoop->isLoopInvariant(Index.value().get()); - } ~VPWidenGEPRecipe() override = default; VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -775,7 +775,7 @@ // is vector-typed. Thus, to keep the representation compact, we only use // vector-typed operands for loop-varying values. - if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) { + if (State.VF.isVector() && areAllOperandsInvariant()) { // If we are vectorizing, but the GEP has only loop-invariant operands, // the GEP we build (by only using vector-typed operands for // loop-varying values) would be a scalar pointer. Thus, to ensure we @@ -805,7 +805,7 @@ for (unsigned Part = 0; Part < State.UF; ++Part) { // The pointer operand of the new GEP. If it's loop-invariant, we // won't broadcast it. - auto *Ptr = IsPtrLoopInvariant + auto *Ptr = isPointerLoopInvariant() ? State.get(getOperand(0), VPIteration(0, 0)) : State.get(getOperand(0), Part); @@ -814,7 +814,7 @@ SmallVector Indices; for (unsigned I = 1, E = getNumOperands(); I < E; I++) { VPValue *Operand = getOperand(I); - if (IsIndexLoopInvariant[I - 1]) + if (isIndexLoopInvariant(I - 1)) Indices.push_back(State.get(Operand, VPIteration(0, 0))); else Indices.push_back(State.get(Operand, Part)); @@ -844,10 +844,9 @@ void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN-GEP "; - O << (IsPtrLoopInvariant ? "Inv" : "Var"); - size_t IndicesNumber = IsIndexLoopInvariant.size(); - for (size_t I = 0; I < IndicesNumber; ++I) - O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]"; + O << (isPointerLoopInvariant() ? "Inv" : "Var"); + for (size_t I = 0; I < getNumOperands() - 1; ++I) + O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]"; O << " "; printAsOperand(O, SlotTracker); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -76,8 +76,8 @@ Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { - NewRecipe = new VPWidenGEPRecipe( - GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop); + NewRecipe = + new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands())); } else if (CallInst *CI = dyn_cast(Inst)) { NewRecipe = new VPWidenCallRecipe(*CI, Plan->mapToVPValues(CI->args()),