diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -412,7 +412,8 @@ std::function GetUnderlyingValue); /// Widen a single call instruction within the innermost loop. - void widenCallInstruction(CallInst &I); + void widenCallInstruction(CallInst &I, VPUser &ArgOperands, + VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. void fixVectorizedLoop(); @@ -4363,7 +4364,8 @@ } // end of switch. } -void InnerLoopVectorizer::widenCallInstruction(CallInst &I) { +void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands, + VPTransformState &State) { // Ignore dbg intrinsics. // TODO: Debug intrinsics should be skipped/handled during VPlan construction // rather than dropping them here. @@ -4392,12 +4394,14 @@ for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Args; - for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) { - Value *Arg = CI->getArgOperand(i); + for (auto &I : enumerate(ArgOperands.operands())) { // Some intrinsics have a scalar argument - don't replace it with a // vector. - if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i)) - Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part); + Value *Arg; + if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index())) + Arg = State.get(I.value(), Part); + else + Arg = State.get(I.value(), {0, 0}); Args.push_back(Arg); } @@ -6891,8 +6895,8 @@ return new VPBlendRecipe(Phi, Masks); } -VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I, - VFRange &Range) { +VPWidenCallRecipe * +VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); @@ -6923,7 +6927,12 @@ return nullptr; // Success: widen this call. - return new VPWidenCallRecipe(*CI); + auto VPValues = map_range(CI->arg_operands(), [&Plan](Value *Op) { + return Plan.getOrAddVPValue(Op); + }); + SmallVector Values(VPValues.begin(), VPValues.end()); + + return new VPWidenCallRecipe(*CI, Values); } VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range, @@ -7080,7 +7089,7 @@ // First, check for specific widening recipes that deal with calls, memory // operations, inductions and Phi nodes. - if ((Recipe = tryToWidenCall(Instr, Range)) || + if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) || (Recipe = tryToWidenMemory(Instr, Range, Plan)) || (Recipe = tryToOptimizeInduction(Instr, Range)) || (Recipe = tryToBlend(Instr, Plan)) || @@ -7399,7 +7408,7 @@ } void VPWidenCallRecipe::execute(VPTransformState &State) { - State.ILV->widenCallInstruction(Ingredient); + State.ILV->widenCallInstruction(Ingredient, User, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -110,7 +110,8 @@ /// Handle call instruction. If \p I is a call that can be widened for \p /// Range.Start, return a new VPWidenCallRecipe. Range.End may be decreased to /// ensure same decision from \p Range.Start to \p Range.End. - VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range); + VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range, + VPlan &Plan); /// Check if \p I can be widened within the given VF \p Range. If \p I can be /// widened for \p Range.Start, build a new VPWidenRecipe and return it. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -793,8 +793,12 @@ /// Hold the call to be widened. CallInst &Ingredient; + /// Hold VPValues for the arguments of the call. + VPUser User; + public: - VPWidenCallRecipe(CallInst &I) : VPRecipeBase(VPWidenCallSC), Ingredient(I) {} + VPWidenCallRecipe(CallInst &I, ArrayRef CallArguments) + : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {} ~VPWidenCallRecipe() override = default;