diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -407,7 +407,8 @@ BasicBlock *createVectorizedLoopSkeleton(); /// Widen a single instruction within the innermost loop. - void widenInstruction(Instruction &I); + void widenInstruction(Instruction &I, VPUser &Operands, + VPTransformState &State); /// Widen a single call instruction within the innermost loop. void widenCallInstruction(CallInst &I, VPUser &ArgOperands, @@ -4229,7 +4230,8 @@ return !CInt || CInt->isZero(); } -void InnerLoopVectorizer::widenInstruction(Instruction &I) { +void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User, + VPTransformState &State) { switch (I.getOpcode()) { case Instruction::Call: case Instruction::Br: @@ -4261,8 +4263,8 @@ for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Ops; - for (Value *Op : I.operands()) - Ops.push_back(getOrCreateVectorValue(Op, Part)); + for (VPValue *VPOp : User.operands()) + Ops.push_back(State.get(VPOp, Part)); Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops); @@ -4283,8 +4285,8 @@ auto *Cmp = cast(&I); setDebugLocFromInst(Builder, Cmp); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part); - Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part); + Value *A = State.get(User.getOperand(0), Part); + Value *B = State.get(User.getOperand(1), Part); Value *C = nullptr; if (FCmp) { // Propagate fast math flags. @@ -4321,7 +4323,7 @@ (VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(CI->getOperand(0), Part); + Value *A = State.get(User.getOperand(0), Part); Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy); VectorLoopValueMap.setVectorValue(&I, Part, Cast); addMetadata(Cast, &I); @@ -6963,7 +6965,8 @@ return new VPWidenSelectRecipe(*SI, InvariantCond); } -VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) { +VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range, + VPlan &Plan) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); @@ -7035,7 +7038,10 @@ return nullptr; // Success: widen this instruction. - return new VPWidenRecipe(*I); + // Create VPValue operands. + auto VPValues = map_range( + I->operands(), [&Plan](Value *Op) { return Plan.getOrAddVPValue(Op); }); + return new VPWidenRecipe(*I, VPValues); } VPBasicBlock *VPRecipeBuilder::handleReplication( @@ -7141,7 +7147,7 @@ // Check if Instr is to be widened by a general VPWidenRecipe, after // having first checked for specific widening recipes. - if ((Recipe = tryToWiden(Instr, Range))) { + if ((Recipe = tryToWiden(Instr, Range, *Plan))) { setRecipe(Instr, Recipe); VPBB->appendRecipe(Recipe); return true; @@ -7442,7 +7448,7 @@ } void VPWidenRecipe::execute(VPTransformState &State) { - State.ILV->widenInstruction(Ingredient); + State.ILV->widenInstruction(Ingredient, User, State); } void VPWidenGEPRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -121,7 +121,7 @@ /// widened for \p Range.Start, build a new VPWidenRecipe and return it. /// Range.End may be decreased to ensure same decision from \p Range.Start to /// \p Range.End. - VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range); + VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range, VPlan &Plan); /// Create a replicating region for instruction \p I that requires /// predication. \p PredRecipe is a VPReplicateRecipe holding \p I. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -771,8 +771,12 @@ /// Hold the instruction to be widened. Instruction &Ingredient; + VPUser User; + public: - VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {} + template + VPWidenRecipe(Instruction &I, iterator_range Operands) + : VPRecipeBase(VPWidenSC), Ingredient(I), User(Operands) {} ~VPWidenRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -71,8 +71,13 @@ NewRecipe = new VPWidenPHIRecipe(Phi); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop); - } else - NewRecipe = new VPWidenRecipe(*Inst); + } else { + // Create VPValue operands. + auto VPValues = map_range(Inst->operands(), [&Plan](Value *Op) { + return Plan->getOrAddVPValue(Op); + }); + NewRecipe = new VPWidenRecipe(*Inst, VPValues); + } NewRecipe->insertBefore(Ingredient); Ingredient->eraseFromParent(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -44,6 +44,7 @@ friend class VPBasicBlock; friend class VPInterleavedAccessInfo; friend class VPSlotTracker; + friend class VPWidenRecipe; private: const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).