[VPlan] Handle scalarized values in VPTransformState.

[fhahn]

Summary

This patch adds plumbing to handle scalarized values directly in
VPTransformState.

[fhahn]

ping

[fhahn]

ping :)

[fhahn]

ping

[gilr]

llvm/lib/Transforms/Vectorize/VPlan.cpp

243

Why is this commented out?

llvm/lib/Transforms/Vectorize/VPlan.h

273

Better to use using?

[fhahn]

Address Gil's comments, thanks!

[fhahn]

Remove unnecessary variable

llvm/lib/Transforms/Vectorize/VPlan.cpp

243

That was some leftovers from earlier experiments. I removed the line for now.

[gilr]

llvm/lib/Transforms/Vectorize/VPlan.h

318

Implementation?

[fhahn]

Move void set(VPValue *Def, Value *IRDef, Value *V, const VPIteration &Instance); declaration to D92284, where it is actually defined & used.

[fhahn]

llvm/lib/Transforms/Vectorize/VPlan.h

318

Good catch, this accidentally slipped into this patch when I split them up. I moved the declaration to D92284 where it is implemented & used. Thanks!

[gilr]

LGTM, with a nit. Thanks!

llvm/lib/Transforms/Vectorize/VPlan.cpp

233

The assert is missing an "error message", but checking hasVectorValue() seems a bit too defensive. Any reason to assert here? (and not assert at line 230?)

[fhahn]

Thanks Gil! Rebased before committing.

llvm/lib/Transforms/Vectorize/VPlan.h

struct VPTransformState {
 
   struct DataState {
     /// A type for vectorized values in the new loop. Each value from the
     /// original loop, when vectorized, is represented by UF vector values in
     /// the new unrolled loop, where UF is the unroll factor.
     typedef SmallVector<Value *, 2> PerPartValuesTy;
 
     DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
+
+    using ScalarsPerPartValuesTy = SmallVector<SmallVector<Value *, 4>, 2>;
+    DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
   } Data;
 
   /// Get the generated Value for a given VPValue and a given Part. Note that
   /// as some Defs are still created by ILV and managed in its ValueMap, this
   /// method will delegate the call to ILV in such cases in order to provide
   /// callers a consistent API.
   /// \see set.
   Value *get(VPValue *Def, unsigned Part) {
     // If Values have been set for this Def return the one relevant for \p Part.
     if (Data.PerPartOutput.count(Def))
       return Data.PerPartOutput[Def][Part];
     // Def is managed by ILV: bring the Values from ValueMap.
     return Callback.getOrCreateVectorValues(VPValue2Value[Def], Part);
   }
 
   /// Get the generated Value for a given VPValue and given Part and Lane.
   Value *get(VPValue *Def, VPIteration Instance);
 
+  bool hasVectorValue(VPValue *Def, unsigned Part) {
+    auto I = Data.PerPartOutput.find(Def);
+    return I != Data.PerPartOutput.end() && Part < I->second.size() &&
+           I->second[Part];
+  }
+
+  bool hasScalarValue(VPValue *Def, VPIteration Instance) {
+    auto I = Data.PerPartScalars.find(Def);
+    if (I == Data.PerPartScalars.end())
+      return false;
+    return Instance.Part < I->second.size() &&
+           Instance.Lane < I->second[Instance.Part].size() &&
+           I->second[Instance.Part][Instance.Lane];
+  }
+
   /// Set the generated Value for a given VPValue and a given Part.
   void set(VPValue *Def, Value *V, unsigned Part) {
     if (!Data.PerPartOutput.count(Def)) {
       DataState::PerPartValuesTy Entry(UF);
       Data.PerPartOutput[Def] = Entry;
     }
     Data.PerPartOutput[Def][Part] = V;
   }
   void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part);
 
+  void set(VPValue *Def, Value *IRDef, Value *V, const VPIteration &Instance);
+
+  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
+    auto Iter = Data.PerPartScalars.insert({Def, {}});
+    auto &PerPartVec = Iter.first->second;
+    while (PerPartVec.size() <= Instance.Part)
+      PerPartVec.emplace_back();
+    auto &Scalars = PerPartVec[Instance.Part];
+    while (Scalars.size() <= Instance.Lane)
+      Scalars.push_back(nullptr);
+    Scalars[Instance.Lane] = V;
+  }
+
   /// Hold state information used when constructing the CFG of the output IR,
   /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
   struct CFGState {
     /// The previous VPBasicBlock visited. Initially set to null.
     VPBasicBlock *PrevVPBB = nullptr;
 
     /// The previous IR BasicBlock created or used. Initially set to the new
     /// header BasicBlock.

llvm/lib/Transforms/Vectorize/VPlan.cpp

while (It != end() && (isa<VPWidenPHIRecipe>(&*It) ||
                          isa<VPWidenIntOrFpInductionRecipe>(&*It) ||
                          isa<VPPredInstPHIRecipe>(&*It) ||
                          isa<VPWidenCanonicalIVRecipe>(&*It)))
     It++;
   return It;
 }
 
 Value *VPTransformState::get(VPValue *Def, VPIteration Instance) {
+  if (!Def->getDef())
+    return Def->getLiveInIRValue();
+
   // For uniform definitions, all lanes produce the same value, so we can always
   // return the first lane.
   if (auto *ReplicateR = dyn_cast<VPReplicateRecipe>(Def))
     if (ReplicateR->isUniform())
       Instance.Lane = 0;
-  // If the Def is managed directly by VPTransformState, extract the lane from
-  // the relevant part. Note that currently only VPInstructions and external
-  // defs are managed by VPTransformState. Other Defs are still created by ILV
-  // and managed in its ValueMap. For those this method currently just
-  // delegates the call to ILV below.
-  if (Data.PerPartOutput.count(Def)) {
+
+  if (hasScalarValue(Def, Instance))
+    return Data.PerPartScalars[Def][Instance.Part][Instance.Lane];
+
+  if (hasVectorValue(Def, Instance.Part)) {
+    assert(Data.PerPartOutput.count(Def));
     auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
     if (!VecPart->getType()->isVectorTy()) {
       assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
       return VecPart;
     }
     // TODO: Cache created scalar values.
     return Builder.CreateExtractElement(VecPart,
                                         Builder.getInt32(Instance.Lane));
   }
 
   return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance);
 }
 
 BasicBlock *
 VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
   // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
   // Pred stands for Predessor. Prev stands for Previous - last visited/created.
   BasicBlock *PrevBB = CFG.PrevBB;