diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1050,8 +1050,8 @@ // Add new definitions to the worklist. for (VPValue *operand : CurRec->operands()) - if (VPDef *OpDef = operand->getDef()) - Worklist.push_back(cast(OpDef)); + if (VPRecipeBase *OpDef = operand->getDefiningRecipe()) + Worklist.push_back(OpDef); } }); @@ -1064,13 +1064,12 @@ for (VPRecipeBase &Recipe : *VPBB) { if (auto *WidenRec = dyn_cast(&Recipe)) { Instruction &UnderlyingInstr = WidenRec->getIngredient(); - VPDef *AddrDef = WidenRec->getAddr()->getDef(); + VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe(); if (AddrDef && WidenRec->isConsecutive() && Legal->blockNeedsPredication(UnderlyingInstr.getParent())) - collectPoisonGeneratingInstrsInBackwardSlice( - cast(AddrDef)); + collectPoisonGeneratingInstrsInBackwardSlice(AddrDef); } else if (auto *InterleaveRec = dyn_cast(&Recipe)) { - VPDef *AddrDef = InterleaveRec->getAddr()->getDef(); + VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe(); if (AddrDef) { // Check if any member of the interleave group needs predication. const InterleaveGroup *InterGroup = @@ -1085,8 +1084,7 @@ } if (NeedPredication) - collectPoisonGeneratingInstrsInBackwardSlice( - cast(AddrDef)); + collectPoisonGeneratingInstrsInBackwardSlice(AddrDef); } } } @@ -8481,11 +8479,12 @@ // value. Avoid hoisting the insert-element which packs the scalar value into // a vector value, as that happens iff all users use the vector value. for (VPValue *Op : Recipe->operands()) { - auto *PredR = dyn_cast_or_null(Op->getDef()); + auto *PredR = + dyn_cast_or_null(Op->getDefiningRecipe()); if (!PredR) continue; - auto *RepR = - cast_or_null(PredR->getOperand(0)->getDef()); + auto *RepR = cast_or_null( + PredR->getOperand(0)->getDefiningRecipe()); assert(RepR->isPredicated() && "expected Replicate recipe to be predicated"); RepR->setAlsoPack(false); @@ -8911,7 +8910,7 @@ Plan->addVPValue(Instr, VPV); // If the re-used value is a recipe, register the recipe for the // instruction, in case the recipe for Instr needs to be recorded. - if (auto *R = dyn_cast_or_null(VPV->getDef())) + if (VPRecipeBase *R = VPV->getDefiningRecipe()) RecipeBuilder.setRecipe(Instr, R); continue; } @@ -9055,12 +9054,12 @@ if (!RecurPhi) continue; - VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe(); + VPRecipeBase *PrevRecipe = &RecurPhi->getBackedgeRecipe(); // Fixed-order recurrences do not contain cycles, so this loop is guaranteed // to terminate. while (auto *PrevPhi = dyn_cast(PrevRecipe)) - PrevRecipe = PrevPhi->getBackedgeRecipe(); + PrevRecipe = &PrevPhi->getBackedgeRecipe(); VPBasicBlock *InsertBlock = PrevRecipe->getParent(); auto *Region = GetReplicateRegion(PrevRecipe); if (Region) @@ -9283,7 +9282,7 @@ VPValue *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan); VPValue *Red = PhiR->getBackedgeValue(); - assert(cast(Red->getDef())->getParent() != LatchVPBB && + assert(Red->getDefiningRecipe()->getParent() != LatchVPBB && "reduction recipe must be defined before latch"); Builder.createNaryOp(Instruction::Select, {Cond, Red, PhiR}); } @@ -9657,7 +9656,7 @@ // A store of a loop varying value to a loop invariant address only // needs only the last copy of the store. - if (isa(UI) && !getOperand(1)->getDef()) { + if (isa(UI) && !getOperand(1)->getDefiningRecipe()) { auto Lane = VPLane::getLastLaneForVF(State.VF); State.ILV->scalarizeInstruction(UI, this, VPIteration(State.UF - 1, Lane), IsPredicated, State); @@ -9875,8 +9874,8 @@ // Check if there is a scalar value for the selected lane. if (!hasScalarValue(Def, {Part, LastLane})) { // At the moment, VPWidenIntOrFpInductionRecipes can also be uniform. - assert((isa(Def->getDef()) || - isa(Def->getDef())) && + assert((isa(Def->getDefiningRecipe()) || + isa(Def->getDefiningRecipe())) && "unexpected recipe found to be invariant"); IsUniform = true; LastLane = 0; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1175,8 +1175,8 @@ /// Returns the backedge value as a recipe. The backedge value is guaranteed /// to be a recipe. - VPRecipeBase *getBackedgeRecipe() { - return cast(getBackedgeValue()->getDef()); + VPRecipeBase &getBackedgeRecipe() { + return *getBackedgeValue()->getDefiningRecipe(); } }; @@ -1944,7 +1944,7 @@ /// Returns the scalar type of the induction. const Type *getScalarType() const { - return cast(getOperand(0)->getDef()) + return cast(getOperand(0)->getDefiningRecipe()) ->getScalarType(); } }; @@ -3059,7 +3059,7 @@ // vectorization inside a vector region. if (VPV->isDefinedOutsideVectorRegions()) return true; - VPDef *Def = VPV->getDef(); + VPRecipeBase *Def = VPV->getDefiningRecipe(); assert(Def && "Must have definition for value defined inside vector region"); if (auto Rep = dyn_cast(Def)) return Rep->isUniform(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -109,6 +109,14 @@ } #endif +VPRecipeBase *VPValue::getDefiningRecipe() { + return cast_or_null(Def); +} + +const VPRecipeBase *VPValue::getDefiningRecipe() const { + return cast_or_null(Def); +} + // Get the top-most entry block of \p Start. This is the entry block of the // containing VPlan. This function is templated to support both const and non-const blocks template static T *getPlanEntry(T *Start) { @@ -202,7 +210,7 @@ } Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) { - if (!Def->getDef()) + if (!Def->getDefiningRecipe()) return Def->getLiveInIRValue(); if (hasScalarValue(Def, Instance)) { @@ -1104,6 +1112,6 @@ VPBasicBlock *Preheader = Plan.getEntry()->getEntryBasicBlock(); VPValue *Step = new VPExpandSCEVRecipe(Expr, SE); - Preheader->appendRecipe(cast(Step->getDef())); + Preheader->appendRecipe(Step->getDefiningRecipe()); return Step; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -723,7 +723,7 @@ if (CanIV->getStartValue() != getStartValue()) return false; auto *StepVPV = getStepValue(); - if (StepVPV->getDef()) + if (StepVPV->getDefiningRecipe()) return false; auto *StepC = dyn_cast_or_null(StepVPV->getLiveInIRValue()); return StepC && StepC->isOne(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -182,19 +182,21 @@ void replaceAllUsesWith(VPValue *New); - VPDef *getDef() { return Def; } - const VPDef *getDef() const { return Def; } + /// Returns the recipe defining this VPValue or nullptr if it is not defined + /// by a recipe, i.e. is a live-in. + VPRecipeBase *getDefiningRecipe(); + const VPRecipeBase *getDefiningRecipe() const; /// Returns the underlying IR value, if this VPValue is defined outside the /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef /// inside a VPlan. Value *getLiveInIRValue() { - assert(!getDef() && + assert(!getDefiningRecipe() && "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); return getUnderlyingValue(); } const Value *getLiveInIRValue() const { - assert(!getDef() && + assert(!getDefiningRecipe() && "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); return getUnderlyingValue(); } @@ -202,7 +204,7 @@ /// Returns true if the VPValue is defined outside any vector regions, i.e. it /// is a live-in value. /// TODO: Also handle recipes defined in pre-header blocks. - bool isDefinedOutsideVectorRegions() const { return !getDef(); } + bool isDefinedOutsideVectorRegions() const { return !getDefiningRecipe(); } }; typedef DenseMap Value2VPValueTy; @@ -328,7 +330,7 @@ /// Add \p V as a defined value by this VPDef. void addDefinedValue(VPValue *V) { - assert(V->getDef() == this && + assert(V->Def == this && "can only add VPValue already linked with this VPDef"); DefinedValues.push_back(V); } @@ -336,8 +338,7 @@ /// Remove \p V from the values defined by this VPDef. \p V must be a defined /// value of this VPDef. void removeDefinedValue(VPValue *V) { - assert(V->getDef() == this && - "can only remove VPValue linked with this VPDef"); + assert(V->Def == this && "can only remove VPValue linked with this VPDef"); assert(is_contained(DefinedValues, V) && "VPValue to remove must be in DefinedValues"); erase_value(DefinedValues, V); diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -813,8 +813,8 @@ EXPECT_EQ(&Recipe, BaseR); VPValue *VPV = &Recipe; - EXPECT_TRUE(isa(VPV->getDef())); - EXPECT_EQ(&Recipe, dyn_cast(VPV->getDef())); + EXPECT_TRUE(isa(VPV->getDefiningRecipe())); + EXPECT_EQ(&Recipe, dyn_cast(VPV->getDefiningRecipe())); delete Call; } @@ -841,8 +841,8 @@ EXPECT_EQ(&WidenSelectR, BaseR); VPValue *VPV = &WidenSelectR; - EXPECT_TRUE(isa(VPV->getDef())); - EXPECT_EQ(&WidenSelectR, dyn_cast(VPV->getDef())); + EXPECT_TRUE(isa(VPV->getDefiningRecipe())); + EXPECT_EQ(&WidenSelectR, dyn_cast(VPV->getDefiningRecipe())); delete SelectI; } @@ -866,8 +866,8 @@ EXPECT_EQ(&Recipe, BaseR); VPValue *VPV = &Recipe; - EXPECT_TRUE(isa(VPV->getDef())); - EXPECT_EQ(&Recipe, dyn_cast(VPV->getDef())); + EXPECT_TRUE(isa(VPV->getDefiningRecipe())); + EXPECT_EQ(&Recipe, dyn_cast(VPV->getDefiningRecipe())); delete GEP; } @@ -945,8 +945,8 @@ EXPECT_EQ(&Recipe, BaseR); VPValue *VPV = Recipe.getVPSingleValue(); - EXPECT_TRUE(isa(VPV->getDef())); - EXPECT_EQ(&Recipe, dyn_cast(VPV->getDef())); + EXPECT_TRUE(isa(VPV->getDefiningRecipe())); + EXPECT_EQ(&Recipe, dyn_cast(VPV->getDefiningRecipe())); delete Load; } @@ -1207,10 +1207,10 @@ EXPECT_EQ(&I3, DoubleValueDefV1Users[1]); // Now check that we can get the right VPDef for each defined value. - EXPECT_EQ(&DoubleValueDef, I1.getOperand(0)->getDef()); - EXPECT_EQ(&DoubleValueDef, I1.getOperand(1)->getDef()); - EXPECT_EQ(&DoubleValueDef, I2.getOperand(0)->getDef()); - EXPECT_EQ(&DoubleValueDef, I3.getOperand(0)->getDef()); + EXPECT_EQ(&DoubleValueDef, I1.getOperand(0)->getDefiningRecipe()); + EXPECT_EQ(&DoubleValueDef, I1.getOperand(1)->getDefiningRecipe()); + EXPECT_EQ(&DoubleValueDef, I2.getOperand(0)->getDefiningRecipe()); + EXPECT_EQ(&DoubleValueDef, I3.getOperand(0)->getDefiningRecipe()); } } // namespace