diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -269,6 +269,9 @@ typedef SmallVector PerPartValuesTy; DenseMap PerPartOutput; + + using ScalarsPerPartValuesTy = SmallVector, 2>; + DenseMap PerPartScalars; } Data; /// Get the generated Value for a given VPValue and a given Part. Note that @@ -287,6 +290,21 @@ /// Get the generated Value for a given VPValue and given Part and Lane. Value *get(VPValue *Def, VPIteration Instance); + bool hasVectorValue(VPValue *Def, unsigned Part) { + auto I = Data.PerPartOutput.find(Def); + return I != Data.PerPartOutput.end() && Part < I->second.size() && + I->second[Part]; + } + + bool hasScalarValue(VPValue *Def, VPIteration Instance) { + auto I = Data.PerPartScalars.find(Def); + if (I == Data.PerPartScalars.end()) + return false; + return Instance.Part < I->second.size() && + Instance.Lane < I->second[Instance.Part].size() && + I->second[Instance.Part][Instance.Lane]; + } + /// Set the generated Value for a given VPValue and a given Part. void set(VPValue *Def, Value *V, unsigned Part) { if (!Data.PerPartOutput.count(Def)) { @@ -297,6 +315,17 @@ } void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part); + void set(VPValue *Def, Value *V, const VPIteration &Instance) { + auto Iter = Data.PerPartScalars.insert({Def, {}}); + auto &PerPartVec = Iter.first->second; + while (PerPartVec.size() <= Instance.Part) + PerPartVec.emplace_back(); + auto &Scalars = PerPartVec[Instance.Part]; + while (Scalars.size() <= Instance.Lane) + Scalars.push_back(nullptr); + Scalars[Instance.Lane] = V; + } + /// Hold state information used when constructing the CFG of the output IR, /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks. struct CFGState { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -217,17 +217,20 @@ } Value *VPTransformState::get(VPValue *Def, VPIteration Instance) { + if (!Def->getDef()) + return Def->getLiveInIRValue(); + // For uniform definitions, all lanes produce the same value, so we can always // return the first lane. if (auto *ReplicateR = dyn_cast(Def)) if (ReplicateR->isUniform()) Instance.Lane = 0; - // If the Def is managed directly by VPTransformState, extract the lane from - // the relevant part. Note that currently only VPInstructions and external - // defs are managed by VPTransformState. Other Defs are still created by ILV - // and managed in its ValueMap. For those this method currently just - // delegates the call to ILV below. - if (Data.PerPartOutput.count(Def)) { + + if (hasScalarValue(Def, Instance)) + return Data.PerPartScalars[Def][Instance.Part][Instance.Lane]; + + if (hasVectorValue(Def, Instance.Part)) { + assert(Data.PerPartOutput.count(Def)); auto *VecPart = Data.PerPartOutput[Def][Instance.Part]; if (!VecPart->getType()->isVectorTy()) { assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");