diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -419,7 +419,7 @@ VPTransformState &State); /// Widen a single call instruction within the innermost loop. - void widenCallInstruction(CallInst &I, VPUser &ArgOperands, + void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands, VPTransformState &State); /// Widen a single select instruction within the innermost loop. @@ -4425,7 +4425,8 @@ } // end of switch. } -void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands, +void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def, + VPUser &ArgOperands, VPTransformState &State) { assert(!isa(I) && "DbgInfoIntrinsic should have been dropped during VPlan construction"); @@ -4489,7 +4490,7 @@ if (isa(V)) V->copyFastMathFlags(CI); - VectorLoopValueMap.setVectorValue(&I, Part, V); + State.set(Def, &I, V, Part); addMetadata(V, &I); } } @@ -7051,7 +7052,10 @@ if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) return nullptr; - return new VPWidenCallRecipe(*CI, Plan.mapToVPValues(CI->arg_operands())); + auto *WidenCall = + new VPWidenCallRecipe(*CI, Plan.mapToVPValues(CI->arg_operands())); + Plan.addVPValue(CI, WidenCall); + return WidenCall; } bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const { @@ -7511,7 +7515,7 @@ } void VPWidenCallRecipe::execute(VPTransformState &State) { - State.ILV->widenCallInstruction(Ingredient, User, State); + State.ILV->widenCallInstruction(*getUnderlyingCall(), this, User, State); } void VPWidenSelectRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -806,17 +806,15 @@ }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase { - /// Hold the call to be widened. - CallInst &Ingredient; - +class VPWidenCallRecipe : public VPRecipeBase, public VPValue { /// Hold VPValues for the arguments of the call. VPUser User; public: template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) - : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {} + : VPRecipeBase(VPWidenCallSC), VPValue(VPValue::VPVWidenCallSC, &I), + User(CallArguments) {} ~VPWidenCallRecipe() override = default; @@ -831,6 +829,11 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + CallInst *getUnderlyingCall() { return cast(getUnderlyingValue()); } + const CallInst *getUnderlyingCall() const { + return cast(getUnderlyingValue()); + } }; /// A recipe for widening select instructions. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -738,7 +738,7 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN-CALL " << VPlanIngredient(&Ingredient); + O << "\"WIDEN-CALL " << VPlanIngredient(getUnderlyingCall()); } void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -76,7 +76,7 @@ /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC }; + enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} VPValue(const VPValue &) = delete;