diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -407,7 +407,8 @@ BasicBlock *createVectorizedLoopSkeleton(); /// Widen a single instruction within the innermost loop. - void widenInstruction(Instruction &I); + void widenInstruction(Instruction &I, VPUser &Operands, + VPTransformState &State); /// Widen a single call instruction within the innermost loop. void widenCallInstruction(CallInst &I, VPUser &ArgOperands, @@ -4231,7 +4232,8 @@ return !CInt || CInt->isZero(); } -void InnerLoopVectorizer::widenInstruction(Instruction &I) { +void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User, + VPTransformState &State) { switch (I.getOpcode()) { case Instruction::Call: case Instruction::Br: @@ -4263,8 +4265,8 @@ for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Ops; - for (Value *Op : I.operands()) - Ops.push_back(getOrCreateVectorValue(Op, Part)); + for (VPValue *VPOp : User.operands()) + Ops.push_back(State.get(VPOp, Part)); Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops); @@ -4285,8 +4287,8 @@ auto *Cmp = cast(&I); setDebugLocFromInst(Builder, Cmp); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part); - Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part); + Value *A = State.get(User.getOperand(0), Part); + Value *B = State.get(User.getOperand(1), Part); Value *C = nullptr; if (FCmp) { // Propagate fast math flags. @@ -4323,7 +4325,7 @@ (VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(CI->getOperand(0), Part); + Value *A = State.get(User.getOperand(0), Part); Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy); VectorLoopValueMap.setVectorValue(&I, Part, Cast); addMetadata(Cast, &I); @@ -7004,7 +7006,10 @@ return nullptr; // Success: widen this instruction. - return new VPWidenRecipe(*I); + // Create VPValue operands. + auto VPValues = map_range( + I->operands(), [&Plan](Value *Op) { return Plan.getOrAddVPValue(Op); }); + return new VPWidenRecipe(*I, VPValues); } VPBasicBlock *VPRecipeBuilder::handleReplication( @@ -7411,7 +7416,7 @@ } void VPWidenRecipe::execute(VPTransformState &State) { - State.ILV->widenInstruction(Ingredient); + State.ILV->widenInstruction(Ingredient, User, State); } void VPWidenGEPRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -768,8 +768,12 @@ /// Hold the instruction to be widened. Instruction &Ingredient; + VPUser User; + public: - VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {} + template + VPWidenRecipe(Instruction &I, iterator_range Operands) + : VPRecipeBase(VPWidenSC), Ingredient(I), User(Operands) {} ~VPWidenRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -71,8 +71,13 @@ NewRecipe = new VPWidenPHIRecipe(Phi); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop); - } else - NewRecipe = new VPWidenRecipe(*Inst); + } else { + // Create VPValue operands. + auto VPValues = map_range(Inst->operands(), [&Plan](Value *Op) { + return Plan->getOrAddVPValue(Op); + }); + NewRecipe = new VPWidenRecipe(*Inst, VPValues); + } NewRecipe->insertBefore(Ingredient); Ingredient->eraseFromParent();