diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -481,12 +481,13 @@ /// Construct the vector value of a scalarized value \p V one lane at a time. void packScalarIntoVectorValue(Value *V, const VPIteration &Instance); - /// Try to vectorize the interleaved access group that \p Instr belongs to - /// with the base address given in \p Addr, optionally masking the vector - /// operations if \p BlockInMask is non-null. Use \p State to translate given - /// VPValues to IR values in the vectorized loop. - void vectorizeInterleaveGroup(Instruction *Instr, VPTransformState &State, - VPValue *Addr, VPValue *BlockInMask = nullptr); + /// Try to vectorize interleaved access group \p Group with the base address + /// given in \p Addr, optionally masking the vector operations if \p + /// BlockInMask is non-null. Use \p State to translate given VPValues to IR + /// values in the vectorized loop. + void vectorizeInterleaveGroup(const InterleaveGroup *Group, + VPTransformState &State, VPValue *Addr, + VPValue *BlockInMask = nullptr); /// Vectorize Load and Store instructions with the base address given in \p /// Addr, optionally masking the vector operations if \p BlockInMask is @@ -2176,18 +2177,10 @@ // %interleaved.vec = shuffle %R_G.vec, %B_U.vec, // <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements // store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B -void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr, - VPTransformState &State, - VPValue *Addr, - VPValue *BlockInMask) { - const InterleaveGroup *Group = - Cost->getInterleavedAccessGroup(Instr); - assert(Group && "Fail to get an interleaved access group."); - - // Skip if current instruction is not the insert position. - if (Instr != Group->getInsertPos()) - return; - +void InnerLoopVectorizer::vectorizeInterleaveGroup( + const InterleaveGroup *Group, VPTransformState &State, + VPValue *Addr, VPValue *BlockInMask) { + Instruction *Instr = Group->getInsertPos(); const DataLayout &DL = Instr->getModule()->getDataLayout(); // Prepare for the vector type of the interleaved load/store. @@ -2376,10 +2369,10 @@ LoopVectorizationCostModel::InstWidening Decision = Cost->getWideningDecision(Instr, VF); - assert(Decision != LoopVectorizationCostModel::CM_Unknown && - "CM decision should be taken at this point"); - if (Decision == LoopVectorizationCostModel::CM_Interleave) - return vectorizeInterleaveGroup(Instr, State, Addr, BlockInMask); + assert((Decision == LoopVectorizationCostModel::CM_Widen || + Decision == LoopVectorizationCostModel::CM_Widen_Reverse || + Decision == LoopVectorizationCostModel::CM_GatherScatter) && + "CM decision is not to widen the memory instruction"); Type *ScalarDataTy = getMemInstValueType(Instr); Type *DataTy = VectorType::get(ScalarDataTy, VF); @@ -7491,8 +7484,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Interleave group being replicated."); - State.ILV->vectorizeInterleaveGroup(IG->getInsertPos(), State, getAddr(), - getMask()); + State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getMask()); } void VPReplicateRecipe::execute(VPTransformState &State) {