diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1083,15 +1083,15 @@ // This recipe contributes to the address computation of a widen // load/store. If the underlying instruction has poison-generating flags, - // either drop them directly if the recipe already models the flags or - // collect them in a set. - // TODO: Migrate all relevant recipes to hold their own flags. - Instruction *Instr = CurRec->getUnderlyingInstr(); - if (Instr && Instr->hasPoisonGeneratingFlags()) { - if (auto *RecWithFlags = dyn_cast(CurRec)) - RecWithFlags->dropPoisonGeneratingFlags(); - else - State.MayGeneratePoisonRecipes.insert(CurRec); + // drop them directly. + if (auto *RecWithFlags = dyn_cast(CurRec)) { + RecWithFlags->dropPoisonGeneratingFlags(); + } else { + Instruction *Instr = CurRec->getUnderlyingInstr(); + (void)Instr; + assert((!Instr || !Instr->hasPoisonGeneratingFlags()) && + "found instruction with poison generating flags not covered by " + "VPRecipeWithIRFlags"); } // Add new definitions to the worklist. @@ -2799,14 +2799,7 @@ if (!IsVoidRetTy) Cloned->setName(Instr->getName() + ".cloned"); - // If the scalarized instruction contributes to the address computation of a - // widen masked load/store which was in a basic block that needed predication - // and is not predicated after vectorization, we can't propagate - // poison-generating flags (nuw/nsw, exact, inbounds, etc.). The scalarized - // instruction could feed a poison value to the base address of the widen - // load/store. - if (State.MayGeneratePoisonRecipes.contains(RepRecipe)) - Cloned->dropPoisonGeneratingFlags(); + RepRecipe->setFlags(Cloned); if (Instr->getDebugLoc()) State.setDebugLocFromInst(Instr); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -398,10 +398,6 @@ /// Pointer to the VPlan code is generated for. VPlan *Plan; - /// Holds recipes that may generate a poison value that is used after - /// vectorization, even when their operands are not poison. - SmallPtrSet MayGeneratePoisonRecipes; - /// The loop object for the current parent region, or nullptr. Loop *CurrentVectorLoop = nullptr; @@ -1017,7 +1013,8 @@ static inline bool classof(const VPRecipeBase *R) { return R->getVPDefID() == VPRecipeBase::VPWidenSC || - R->getVPDefID() == VPRecipeBase::VPWidenGEPSC; + R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || + R->getVPDefID() == VPRecipeBase::VPReplicateSC; } /// Drop all poison-generating flags. @@ -1723,7 +1720,7 @@ /// copies of the original scalar type, one per lane, instead of producing a /// single copy of widened type for all lanes. If the instruction is known to be /// uniform only one copy, per lane zero, will be generated. -class VPReplicateRecipe : public VPRecipeBase, public VPValue { +class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPValue { /// Indicator if only a single replica per lane is needed. bool IsUniform; @@ -1734,8 +1731,8 @@ template VPReplicateRecipe(Instruction *I, iterator_range Operands, bool IsUniform, VPValue *Mask = nullptr) - : VPRecipeBase(VPDef::VPReplicateSC, Operands), VPValue(this, I), - IsUniform(IsUniform), IsPredicated(Mask) { + : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I), + VPValue(this, I), IsUniform(IsUniform), IsPredicated(Mask) { if (Mask) addOperand(Mask); } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1006,10 +1006,14 @@ Args.push_back(&Op1); Args.push_back(&Op2); - VPReplicateRecipe Recipe(nullptr, make_range(Args.begin(), Args.end()), true); + IntegerType *Int32 = IntegerType::get(C, 32); + FunctionType *FTy = FunctionType::get(Int32, false); + auto *Call = CallInst::Create(FTy, UndefValue::get(FTy)); + VPReplicateRecipe Recipe(Call, make_range(Args.begin(), Args.end()), true); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); + delete Call; } TEST(VPRecipeTest, CastVPBranchOnMaskRecipeToVPUser) {