diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3989,7 +3989,8 @@ // If the value wasn't vectorized, we must maintain the original scalar // type. The absence of the value from State indicates that it // wasn't vectorized. - VPValue *Def = State.Plan->getVPValue(KV.first); + // FIXME: Should not rely on getVPValue at this point. + VPValue *Def = State.Plan->getVPValue(KV.first, true); if (!State.hasAnyVectorValue(Def)) continue; for (unsigned Part = 0; Part < UF; ++Part) { @@ -4096,7 +4097,8 @@ // If the value wasn't vectorized, we must maintain the original scalar // type. The absence of the value from State indicates that it // wasn't vectorized. - VPValue *Def = State.Plan->getVPValue(KV.first); + // FIXME: Should not rely on getVPValue at this point. + VPValue *Def = State.Plan->getVPValue(KV.first, true); if (!State.hasAnyVectorValue(Def)) continue; for (unsigned Part = 0; Part < UF; ++Part) { @@ -4482,7 +4484,8 @@ Instruction *Cur = Worklist.pop_back_val(); if (isa(Cur)) for (unsigned Part = 0; Part < UF; ++Part) { - Value *V = State.get(State.Plan->getVPValue(Cur), Part); + // FIXME: Should not rely on getVPValue at this point. + Value *V = State.get(State.Plan->getVPValue(Cur, true), Part); cast(V)->dropPoisonGeneratingFlags(); } @@ -4513,11 +4516,12 @@ // Can be a loop invariant incoming value or the last scalar value to be // extracted from the vectorized loop. + // FIXME: Should not rely on getVPValue at this point. Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); Value *lastIncomingValue = OrigLoop->isLoopInvariant(IncomingValue) ? IncomingValue - : State.get(State.Plan->getVPValue(IncomingValue), + : State.get(State.Plan->getVPValue(IncomingValue, true), VPIteration(UF - 1, Lane)); LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock); } @@ -9480,6 +9484,10 @@ } } + // From this point onwards, VPlan-to-VPlan transformations may change the plan + // in ways that accessing values using original IR values is incorrect. + Plan->disableValue2VPValue(); + VPlanTransforms::sinkScalarOperands(*Plan); VPlanTransforms::mergeReplicateRegions(*Plan); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2100,6 +2100,10 @@ /// Holds the VPLoopInfo analysis for this VPlan. VPLoopInfo VPLInfo; + /// Indicates whether it is safe use the Value2VPValue mapping or if the + /// mapping cannot be used any longer, because it is stale. + bool Value2VPValueEnabled = true; + public: VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) { if (Entry) @@ -2141,6 +2145,10 @@ return BackedgeTakenCount; } + /// Mark the plan to indicate that using Value2VPValue is not safe any + /// longer, because it may be stale. + void disableValue2VPValue() { Value2VPValueEnabled = false; } + void addVF(ElementCount VF) { VFs.insert(VF); } bool hasVF(ElementCount VF) { return VFs.count(VF); } @@ -2154,6 +2162,8 @@ void addExternalDef(VPValue *VPVal) { VPExternalDefs.insert(VPVal); } void addVPValue(Value *V) { + assert(Value2VPValueEnabled && + "IR value to VPValue mapping may be out of date!"); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); VPValue *VPV = new VPValue(V); @@ -2162,25 +2172,39 @@ } void addVPValue(Value *V, VPValue *VPV) { + assert(Value2VPValueEnabled && "Value2VPValue mapping may be out of date!"); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); Value2VPValue[V] = VPV; } - VPValue *getVPValue(Value *V) { + /// Returns the VPValue for \p V. \p OverrideAllowed can be used to disable + /// checking whether it is safe to query VPValues using IR Values. + VPValue *getVPValue(Value *V, bool OverrideAllowed = false) { + assert((OverrideAllowed || isa(V) || Value2VPValueEnabled) && + "Value2VPValue mapping may be out of date!"); assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); return Value2VPValue[V]; } - VPValue *getOrAddVPValue(Value *V) { + /// Gets the VPValue or adds a new one (if none exists yet) for \p V. \p + /// OverrideAllowed can be used to disable checking whether it is safe to + /// query VPValues using IR Values. + VPValue *getOrAddVPValue(Value *V, bool OverrideAllowed = false) { + assert((OverrideAllowed || isa(V) || Value2VPValueEnabled) && + "Value2VPValue mapping may be out of date!"); assert(V && "Trying to get or add the VPValue of a null Value"); if (!Value2VPValue.count(V)) addVPValue(V); return getVPValue(V); } - void removeVPValueFor(Value *V) { Value2VPValue.erase(V); } + void removeVPValueFor(Value *V) { + assert(Value2VPValueEnabled && + "IR value to VPValue mapping may be out of date!"); + Value2VPValue.erase(V); + } /// Return the VPLoopInfo analysis for this VPlan. VPLoopInfo &getVPLoopInfo() { return VPLInfo; }