diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1073,12 +1073,14 @@ // either drop them directly if the recipe already models the flags or // collect them in a set. // TODO: Migrate all relevant recipes to hold their own flags. - Instruction *Instr = CurRec->getUnderlyingInstr(); - if (Instr && Instr->hasPoisonGeneratingFlags()) { - if (auto *RecWithFlags = dyn_cast(CurRec)) - RecWithFlags->dropPoisonGeneratingFlags(); - else - State.MayGeneratePoisonRecipes.insert(CurRec); + if (auto *OpWithFlags = dyn_cast(CurRec)) { + OpWithFlags->dropPoisonGeneratingFlags(); + } else { + Instruction *Instr = CurRec->getUnderlyingInstr(); + (void)Instr; + assert((!Instr || !Instr->hasPoisonGeneratingFlags()) && + "found instruction with poison generating flags not covered by " + "VPRecipeWithIRFlags"); } // Add new definitions to the worklist. @@ -2844,8 +2846,8 @@ // poison-generating flags (nuw/nsw, exact, inbounds, etc.). The scalarized // instruction could feed a poison value to the base address of the widen // load/store. - if (State.MayGeneratePoisonRecipes.contains(RepRecipe)) - Cloned->dropPoisonGeneratingFlags(); + Cloned->dropPoisonGeneratingFlags(); + RepRecipe->setFlags(Cloned); if (Instr->getDebugLoc()) State.setDebugLocFromInst(Instr); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -398,10 +398,6 @@ /// Pointer to the VPlan code is generated for. VPlan *Plan; - /// Holds recipes that may generate a poison value that is used after - /// vectorization, even when their operands are not poison. - SmallPtrSet MayGeneratePoisonRecipes; - /// The loop object for the current parent region, or nullptr. Loop *CurrentVectorLoop = nullptr; @@ -1013,7 +1009,8 @@ static inline bool classof(const VPRecipeBase *R) { return R->getVPDefID() == VPRecipeBase::VPWidenSC || - R->getVPDefID() == VPRecipeBase::VPWidenGEPSC; + R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || + R->getVPDefID() == VPRecipeBase::VPReplicateSC; } /// Drop all poison-generating flags. @@ -1713,7 +1710,7 @@ /// copies of the original scalar type, one per lane, instead of producing a /// single copy of widened type for all lanes. If the instruction is known to be /// uniform only one copy, per lane zero, will be generated. -class VPReplicateRecipe : public VPRecipeBase, public VPValue { +class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPValue { /// Indicator if only a single replica per lane is needed. bool IsUniform; @@ -1724,8 +1721,8 @@ template VPReplicateRecipe(Instruction *I, iterator_range Operands, bool IsUniform, VPValue *Mask = nullptr) - : VPRecipeBase(VPDef::VPReplicateSC, Operands), VPValue(this, I), - IsUniform(IsUniform), IsPredicated(Mask) { + : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I), + VPValue(this, I), IsUniform(IsUniform), IsPredicated(Mask) { if (Mask) addOperand(Mask); }