diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -407,7 +407,9 @@ BasicBlock *createVectorizedLoopSkeleton(); /// Widen a single instruction within the innermost loop. - void widenInstruction(Instruction &I); + void widenInstruction(Instruction &I, VPUser &Operands, + VPTransformState &State, + std::function GetUnderlyingValue); /// Widen a single call instruction within the innermost loop. void widenCallInstruction(CallInst &I); @@ -4225,7 +4227,9 @@ return !CInt || CInt->isZero(); } -void InnerLoopVectorizer::widenInstruction(Instruction &I) { +void InnerLoopVectorizer::widenInstruction( + Instruction &I, VPUser &User, VPTransformState &State, + std::function GetUnderlyingValue) { switch (I.getOpcode()) { case Instruction::Call: case Instruction::Br: @@ -4256,8 +4260,8 @@ for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Ops; - for (Value *Op : I.operands()) - Ops.push_back(getOrCreateVectorValue(Op, Part)); + for (VPValue *VPOp : User.operands()) + Ops.push_back(State.get(VPOp, Part)); Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops); @@ -4277,7 +4281,7 @@ // instruction with a scalar condition. Otherwise, use vector-select. auto *SE = PSE.getSE(); bool InvariantCond = - SE->isLoopInvariant(PSE.getSCEV(I.getOperand(0)), OrigLoop); + SE->isLoopInvariant(PSE.getSCEV(GetUnderlyingValue(0)), OrigLoop); setDebugLocFromInst(Builder, &I); // The condition can be loop invariant but still defined inside the @@ -4285,12 +4289,12 @@ // We have to take the 'vectorized' value and pick the first lane. // Instcombine will make this a no-op. - auto *ScalarCond = getOrCreateScalarValue(I.getOperand(0), {0, 0}); + auto *ScalarCond = State.get(User.getOperand(0), {0, 0}); for (unsigned Part = 0; Part < UF; ++Part) { - Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part); - Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part); - Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part); + Value *Cond = State.get(User.getOperand(0), Part); + Value *Op0 = State.get(User.getOperand(1), Part); + Value *Op1 = State.get(User.getOperand(2), Part); Value *Sel = Builder.CreateSelect(InvariantCond ? ScalarCond : Cond, Op0, Op1); VectorLoopValueMap.setVectorValue(&I, Part, Sel); @@ -4307,8 +4311,8 @@ auto *Cmp = cast(&I); setDebugLocFromInst(Builder, Cmp); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part); - Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part); + Value *A = State.get(User.getOperand(0), Part); + Value *B = State.get(User.getOperand(1), Part); Value *C = nullptr; if (FCmp) { // Propagate fast math flags. @@ -4345,7 +4349,7 @@ (VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF); for (unsigned Part = 0; Part < UF; ++Part) { - Value *A = getOrCreateVectorValue(CI->getOperand(0), Part); + Value *A = State.get(User.getOperand(0), Part); Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy); VectorLoopValueMap.setVectorValue(&I, Part, Cast); addMetadata(Cast, &I); @@ -6922,7 +6926,8 @@ return new VPWidenCallRecipe(*CI); } -VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) { +VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range, + VPlan &Plan) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); @@ -6994,7 +6999,11 @@ return nullptr; // Success: widen this instruction. - return new VPWidenRecipe(*I); + // Create VPValue operands. + auto VPValues = map_range( + I->operands(), [&Plan](Value *Op) { return Plan.getOrAddVPValue(Op); }); + SmallVector Values(VPValues.begin(), VPValues.end()); + return new VPWidenRecipe(*I, Values); } VPBasicBlock *VPRecipeBuilder::handleReplication( @@ -7099,7 +7108,7 @@ // Check if Instr is to be widened by a general VPWidenRecipe, after // having first checked for specific widening recipes. - if ((Recipe = tryToWiden(Instr, Range))) { + if ((Recipe = tryToWiden(Instr, Range, *Plan))) { setRecipe(Instr, Recipe); VPBB->appendRecipe(Recipe); return true; @@ -7394,7 +7403,12 @@ } void VPWidenRecipe::execute(VPTransformState &State) { - State.ILV->widenInstruction(Ingredient); + // Helpers to access the underlying value of a VPValue operand. + auto GetUnderlyingValue = [this](unsigned OpIdx) { + return User.getOperand(OpIdx)->getUnderlyingValue(); + }; + + State.ILV->widenInstruction(Ingredient, User, State, GetUnderlyingValue); } void VPWidenGEPRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -116,7 +116,7 @@ /// widened for \p Range.Start, build a new VPWidenRecipe and return it. /// Range.End may be decreased to ensure same decision from \p Range.Start to /// \p Range.End. - VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range); + VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range, VPlan &Plan); /// Create a replicating region for instruction \p I that requires /// predication. \p PredRecipe is a VPReplicateRecipe holding \p I. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -766,8 +766,11 @@ /// Hold the instruction to be widened. Instruction &Ingredient; + VPUser User; + public: - VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {} + VPWidenRecipe(Instruction &I, ArrayRef Operands) + : VPRecipeBase(VPWidenSC), Ingredient(I), User(Operands) {} ~VPWidenRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -71,8 +71,14 @@ NewRecipe = new VPWidenPHIRecipe(Phi); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop); - } else - NewRecipe = new VPWidenRecipe(*Inst); + } else { + // Create VPValue operands. + auto VPValues = map_range(Inst->operands(), [&Plan](Value *Op) { + return Plan->getOrAddVPValue(Op); + }); + SmallVector Values(VPValues.begin(), VPValues.end()); + NewRecipe = new VPWidenRecipe(*Inst, Values); + } NewRecipe->insertBefore(Ingredient); Ingredient->eraseFromParent(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -43,6 +43,7 @@ friend class VPBasicBlock; friend class VPInterleavedAccessInfo; friend class VPSlotTracker; + friend class VPWidenRecipe; private: const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).