diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3989,7 +3989,8 @@ // If the value wasn't vectorized, we must maintain the original scalar // type. The absence of the value from State indicates that it // wasn't vectorized. - VPValue *Def = State.Plan->getVPValue(KV.first); + // FIXME: Should not rely on getVPValue at this point. + VPValue *Def = State.Plan->getVPValue(KV.first, true); if (!State.hasAnyVectorValue(Def)) continue; for (unsigned Part = 0; Part < UF; ++Part) { @@ -4096,7 +4097,8 @@ // If the value wasn't vectorized, we must maintain the original scalar // type. The absence of the value from State indicates that it // wasn't vectorized. - VPValue *Def = State.Plan->getVPValue(KV.first); + // FIXME: Should not rely on getVPValue at this point. + VPValue *Def = State.Plan->getVPValue(KV.first, true); if (!State.hasAnyVectorValue(Def)) continue; for (unsigned Part = 0; Part < UF; ++Part) { @@ -4482,7 +4484,8 @@ Instruction *Cur = Worklist.pop_back_val(); if (isa(Cur)) for (unsigned Part = 0; Part < UF; ++Part) { - Value *V = State.get(State.Plan->getVPValue(Cur), Part); + // FIXME: Should not rely on getVPValue at this point. + Value *V = State.get(State.Plan->getVPValue(Cur, true), Part); cast(V)->dropPoisonGeneratingFlags(); } @@ -4513,11 +4516,12 @@ // Can be a loop invariant incoming value or the last scalar value to be // extracted from the vectorized loop. + // FIXME: Should not rely on getVPValue at this point. Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); Value *lastIncomingValue = OrigLoop->isLoopInvariant(IncomingValue) ? IncomingValue - : State.get(State.Plan->getVPValue(IncomingValue), + : State.get(State.Plan->getVPValue(IncomingValue, true), VPIteration(UF - 1, Lane)); LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock); } @@ -9480,6 +9484,10 @@ } } + // From this point onwards, VPlan-to-VPlan transformations may change the plan + // in ways that accessing values using original IR values is incorrect. + Plan->markIRToVPlanDisallowed(); + VPlanTransforms::sinkScalarOperands(*Plan); VPlanTransforms::mergeReplicateRegions(*Plan); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2100,6 +2100,9 @@ /// Holds the VPLoopInfo analysis for this VPlan. VPLoopInfo VPLInfo; + /// Indicates whether it is safe to use IR references to query VPlan values. + bool IRToVPlanAllowed = true; + public: VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) { if (Entry) @@ -2141,6 +2144,10 @@ return BackedgeTakenCount; } + /// Mark the plan to indicate that using IR references to query VPlan values + /// is not safe any longer. + void markIRToVPlanDisallowed() { IRToVPlanAllowed = false; } + void addVF(ElementCount VF) { VFs.insert(VF); } bool hasVF(ElementCount VF) { return VFs.count(VF); } @@ -2167,14 +2174,23 @@ Value2VPValue[V] = VPV; } - VPValue *getVPValue(Value *V) { + /// Returns the VPValue for \p V. \p OverrideAllowed can be used to disable + /// checking whether it is safe to query VPValues using IR Values. + VPValue *getVPValue(Value *V, bool OverrideAllowed = false) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); + assert((OverrideAllowed || isa(V) || IRToVPlanAllowed) && + "getVPValue used"); return Value2VPValue[V]; } - VPValue *getOrAddVPValue(Value *V) { + /// Gets the VPValue or adds a new one (if none exists yet) for \p V. \p + /// OverrideAllowed can be used to disable checking whether it is safe to + /// query VPValues using IR Values. + VPValue *getOrAddVPValue(Value *V, bool OverrideAllowed = false) { assert(V && "Trying to get or add the VPValue of a null Value"); + assert((OverrideAllowed || isa(V) || IRToVPlanAllowed) && + "getVPValue used"); if (!Value2VPValue.count(V)) addVPValue(V); return getVPValue(V);