diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6987,7 +6987,6 @@ case Instruction::FSub: case Instruction::ICmp: case Instruction::IntToPtr: - case Instruction::Load: case Instruction::LShr: case Instruction::Mul: case Instruction::Or: @@ -6999,7 +6998,6 @@ case Instruction::Shl: case Instruction::SIToFP: case Instruction::SRem: - case Instruction::Store: case Instruction::Sub: case Instruction::Trunc: case Instruction::UDiv: @@ -7019,12 +7017,6 @@ if (!isa(I) && (CM.isScalarAfterVectorization(I, VF) || CM.isProfitableToScalarize(I, VF))) return false; - if (isa(I) || isa(I)) { - assert(CM.getWideningDecision(I, VF) == - LoopVectorizationCostModel::CM_Scalarize && - "Memory widening decisions should have been taken care by now"); - return false; - } return true; }; @@ -7049,6 +7041,23 @@ bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); + // Make sure only memory instructions explicitly marked or profitable with + // scalarization are handled with replication. Otherwise they should have been + // widened earlier. + assert( + IsPredicated || + LoopVectorizationPlanner::getDecisionAndClampRange( + [this, I](unsigned VF) { + if (VF == 1 || (!isa(I) && !isa(I))) + return true; + return CM.isProfitableToScalarize(I, VF) || + CM.isScalarAfterVectorization(I, VF) || + CM.getWideningDecision(I, VF) == + LoopVectorizationCostModel::CM_Scalarize; + }, + Range) && + "Memory widening decisions should have been taken care of by now."); + auto *Recipe = new VPReplicateRecipe(I, IsUniform, IsPredicated); setRecipe(I, Recipe);