diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -407,7 +407,9 @@
BasicBlock *createVectorizedLoopSkeleton();
/// Widen a single instruction within the innermost loop.
- void widenInstruction(Instruction &I);
+ void widenInstruction(Instruction &I, VPUser &Operands,
+ VPTransformState &State,
+ std::function<Value *(unsigned)> GetUnderlyingValue);
/// Widen a single call instruction within the innermost loop.
void widenCallInstruction(CallInst &I);
@@ -4225,7 +4227,9 @@
return !CInt || CInt->isZero();
}
-void InnerLoopVectorizer::widenInstruction(Instruction &I) {
+void InnerLoopVectorizer::widenInstruction(
+ Instruction &I, VPUser &User, VPTransformState &State,
+ std::function<Value *(unsigned)> GetUnderlyingValue) {
switch (I.getOpcode()) {
case Instruction::Call:
case Instruction::Br:
@@ -4256,8 +4260,8 @@
for (unsigned Part = 0; Part < UF; ++Part) {
SmallVector<Value *, 2> Ops;
- for (Value *Op : I.operands())
- Ops.push_back(getOrCreateVectorValue(Op, Part));
+ for (VPValue *VPOp : User.operands())
+ Ops.push_back(State.get(VPOp, Part));
Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
@@ -4277,7 +4281,7 @@
// instruction with a scalar condition. Otherwise, use vector-select.
auto *SE = PSE.getSE();
bool InvariantCond =
- SE->isLoopInvariant(PSE.getSCEV(I.getOperand(0)), OrigLoop);
+ SE->isLoopInvariant(PSE.getSCEV(GetUnderlyingValue(0)), OrigLoop);
setDebugLocFromInst(Builder, &I);
// The condition can be loop invariant but still defined inside the
@@ -4285,12 +4289,12 @@
// We have to take the 'vectorized' value and pick the first lane.
// Instcombine will make this a no-op.
- auto *ScalarCond = getOrCreateScalarValue(I.getOperand(0), {0, 0});
+ auto *ScalarCond = State.get(User.getOperand(0), {0, 0});
for (unsigned Part = 0; Part < UF; ++Part) {
- Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part);
- Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part);
- Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part);
+ Value *Cond = State.get(User.getOperand(0), Part);
+ Value *Op0 = State.get(User.getOperand(1), Part);
+ Value *Op1 = State.get(User.getOperand(2), Part);
Value *Sel =
Builder.CreateSelect(InvariantCond ? ScalarCond : Cond, Op0, Op1);
VectorLoopValueMap.setVectorValue(&I, Part, Sel);
@@ -4307,8 +4311,8 @@
auto *Cmp = cast<CmpInst>(&I);
setDebugLocFromInst(Builder, Cmp);
for (unsigned Part = 0; Part < UF; ++Part) {
- Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part);
- Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part);
+ Value *A = State.get(User.getOperand(0), Part);
+ Value *B = State.get(User.getOperand(1), Part);
Value *C = nullptr;
if (FCmp) {
// Propagate fast math flags.
@@ -4345,7 +4349,7 @@
(VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF);
for (unsigned Part = 0; Part < UF; ++Part) {
- Value *A = getOrCreateVectorValue(CI->getOperand(0), Part);
+ Value *A = State.get(User.getOperand(0), Part);
Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
VectorLoopValueMap.setVectorValue(&I, Part, Cast);
addMetadata(Cast, &I);
@@ -6922,7 +6926,8 @@
return new VPWidenCallRecipe(*CI);
}
-VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) {
+VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range,
+ VPlan &Plan) {
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
[&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
@@ -6994,7 +6999,11 @@
return nullptr;
// Success: widen this instruction.
- return new VPWidenRecipe(*I);
+ // Create VPValue operands.
+ auto VPValues = map_range(
+ I->operands(), [&Plan](Value *Op) { return Plan.getOrAddVPValue(Op); });
+ SmallVector<VPValue *, 4> Values(VPValues.begin(), VPValues.end());
+ return new VPWidenRecipe(*I, Values);
}
VPBasicBlock *VPRecipeBuilder::handleReplication(
@@ -7099,7 +7108,7 @@
// Check if Instr is to be widened by a general VPWidenRecipe, after
// having first checked for specific widening recipes.
- if ((Recipe = tryToWiden(Instr, Range))) {
+ if ((Recipe = tryToWiden(Instr, Range, *Plan))) {
setRecipe(Instr, Recipe);
VPBB->appendRecipe(Recipe);
return true;
@@ -7394,7 +7403,12 @@
}
void VPWidenRecipe::execute(VPTransformState &State) {
- State.ILV->widenInstruction(Ingredient);
+ // Helpers to access the underlying value of a VPValue operand.
+ auto GetUnderlyingValue = [this](unsigned OpIdx) {
+ return User.getOperand(OpIdx)->getUnderlyingValue();
+ };
+
+ State.ILV->widenInstruction(Ingredient, User, State, GetUnderlyingValue);
}
void VPWidenGEPRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -116,7 +116,7 @@
/// widened for \p Range.Start, build a new VPWidenRecipe and return it.
/// Range.End may be decreased to ensure same decision from \p Range.Start to
/// \p Range.End.
- VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range);
+ VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range, VPlan &Plan);
/// Create a replicating region for instruction \p I that requires
/// predication. \p PredRecipe is a VPReplicateRecipe holding \p I.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -766,8 +766,11 @@
/// Hold the instruction to be widened.
Instruction &Ingredient;
+ VPUser User;
+
public:
- VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {}
+ VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+ : VPRecipeBase(VPWidenSC), Ingredient(I), User(Operands) {}
~VPWidenRecipe() override = default;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -71,8 +71,14 @@
NewRecipe = new VPWidenPHIRecipe(Phi);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop);
- } else
- NewRecipe = new VPWidenRecipe(*Inst);
+ } else {
+ // Create VPValue operands.
+ auto VPValues = map_range(Inst->operands(), [&Plan](Value *Op) {
+ return Plan->getOrAddVPValue(Op);
+ });
+ SmallVector<VPValue *, 4> Values(VPValues.begin(), VPValues.end());
+ NewRecipe = new VPWidenRecipe(*Inst, Values);
+ }
NewRecipe->insertBefore(Ingredient);
Ingredient->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -43,6 +43,7 @@
friend class VPBasicBlock;
friend class VPInterleavedAccessInfo;
friend class VPSlotTracker;
+ friend class VPWidenRecipe;
private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).