diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2768,6 +2768,14 @@ return FoundLane; } + /// Build a shuffle mask for graph entry which represents a merge of main + /// and alternate operations. + void + buildAltOpShuffleMask(const function_ref IsAltOp, + SmallVectorImpl &Mask, + SmallVectorImpl *OpScalars = nullptr, + SmallVectorImpl *AltScalars = nullptr) const; + #ifndef NDEBUG /// Debug printer. LLVM_DUMP_METHOD void dump() const { @@ -6364,16 +6372,11 @@ return {IntrinsicCost, LibCost}; } -/// Build shuffle mask for shuffle graph entries and lists of main and alternate -/// operations operands. -static void -buildShuffleEntryMask(ArrayRef VL, ArrayRef ReorderIndices, - ArrayRef ReusesIndices, - const function_ref IsAltOp, - SmallVectorImpl &Mask, - SmallVectorImpl *OpScalars = nullptr, - SmallVectorImpl *AltScalars = nullptr) { - unsigned Sz = VL.size(); +void BoUpSLP::TreeEntry::buildAltOpShuffleMask( + const function_ref IsAltOp, SmallVectorImpl &Mask, + SmallVectorImpl *OpScalars, + SmallVectorImpl *AltScalars) const { + unsigned Sz = Scalars.size(); Mask.assign(Sz, PoisonMaskElem); SmallVector OrderMask; if (!ReorderIndices.empty()) @@ -6382,7 +6385,7 @@ unsigned Idx = I; if (!ReorderIndices.empty()) Idx = OrderMask[I]; - auto *OpInst = cast(VL[Idx]); + auto *OpInst = cast(Scalars[Idx]); if (IsAltOp(OpInst)) { Mask[I] = Sz + Idx; if (AltScalars) @@ -6393,9 +6396,9 @@ OpScalars->push_back(OpInst); } } - if (!ReusesIndices.empty()) { - SmallVector NewMask(ReusesIndices.size(), PoisonMaskElem); - transform(ReusesIndices, NewMask.begin(), [&Mask](int Idx) { + if (!ReuseShuffleIndices.empty()) { + SmallVector NewMask(ReuseShuffleIndices.size(), PoisonMaskElem); + transform(ReuseShuffleIndices, NewMask.begin(), [&Mask](int Idx) { return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem; }); Mask.swap(NewMask); @@ -8040,8 +8043,7 @@ TTI::CastContextHint::None, CostKind); } SmallVector Mask; - buildShuffleEntryMask( - E->Scalars, E->ReorderIndices, E->ReuseShuffleIndices, + E->buildAltOpShuffleMask( [E](Instruction *I) { assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); return I->getOpcode() == E->getAltOpcode(); @@ -10791,8 +10793,7 @@ // each vector operation. ValueList OpScalars, AltScalars; SmallVector Mask; - buildShuffleEntryMask( - E->Scalars, E->ReorderIndices, E->ReuseShuffleIndices, + E->buildAltOpShuffleMask( [E, this](Instruction *I) { assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),