diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9659,7 +9659,9 @@ // If the recipe is uniform across all parts (instead of just per VF), only // generate a single instance. if ((isa(UI) || isa(UI)) && - all_of(operands(), [](VPValue *Op) { return !Op->getDef(); })) { + all_of(operands(), [](VPValue *Op) { + return Op->isDefinedOutsideVectorRegions(); + })) { State.ILV->scalarizeInstruction(UI, this, VPIteration(0, 0), IsPredicated, State); if (user_begin() != user_end()) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3055,13 +3055,15 @@ /// Returns true if \p VPV is uniform after vectorization. inline bool isUniformAfterVectorization(VPValue *VPV) { - if (auto *Def = VPV->getDef()) { - if (auto Rep = dyn_cast(Def)) - return Rep->isUniform(); - return false; - } - // A value without a def is external to vplan and thus uniform. - return true; + // A value defined outside the vector region must be uniform after + // vectorization inside a vector region. + if (VPV->isDefinedOutsideVectorRegions()) + return true; + VPDef *Def = VPV->getDef(); + assert(Def && "Must have definition for value defined inside vector region"); + if (auto Rep = dyn_cast(Def)) + return Rep->isUniform(); + return false; } } // end namespace vputils diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -198,6 +198,11 @@ "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); return getUnderlyingValue(); } + + /// Returns true if the VPValue is defined outside any vector regions, i.e. it + /// is a live-in value. + /// TODO: Also handle recipes defined in pre-header blocks. + bool isDefinedOutsideVectorRegions() const { return !getDef(); } }; typedef DenseMap Value2VPValueTy;