diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9145,6 +9145,39 @@ ShuffleInstructionBuilder(IRBuilderBase &Builder, BoUpSLP &R) : Builder(Builder), R(R) {} + /// Adjusts extractelements after reusing them. + Value *adjustExtracts(const TreeEntry *E, ArrayRef Mask) { + Value *VecBase = nullptr; + for (int I = 0, Sz = Mask.size(); I < Sz; ++I) { + int Idx = Mask[I]; + if (Idx == UndefMaskElem) + continue; + auto *EI = cast(E->Scalars[I]); + VecBase = EI->getVectorOperand(); + // If the only one use is vectorized - can delete the extractelement + // itself. + if (!EI->hasOneUse() || any_of(EI->users(), [&](User *U) { + return !R.ScalarToTreeEntry.count(U); + })) + continue; + R.eraseInstruction(EI); + } + return VecBase; + }; + /// Checks if the specified entry \p E needs to be delayed because of its + /// dependency nodes. + Value *needToDelay(const TreeEntry *E, ArrayRef Deps) { + // No need to delay emission if all deps are ready. + if (all_of(Deps, [](const TreeEntry *TE) { return TE->VectorizedValue; })) + return nullptr; + // Postpone gather emission, will be emitted after the end of the + // process to keep correct order. + auto *VecTy = FixedVectorType::get(E->Scalars.front()->getType(), + E->getVectorFactor()); + Value *Vec = Builder.CreateAlignedLoad( + VecTy, PoisonValue::get(VecTy->getPointerTo()), MaybeAlign()); + return Vec; + }; /// Adds 2 input vectors and the mask for their shuffling. void add(Value *V1, Value *V2, ArrayRef Mask) { assert(V1 && V2 && !Mask.empty() && "Expected non-empty input vectors."); @@ -9408,38 +9441,6 @@ assert(E->State == TreeEntry::NeedToGather && "Expected gather node."); unsigned VF = E->getVectorFactor(); - auto AdjustExtracts = [&](const TreeEntry *E, ArrayRef Mask) { - Value *VecBase = nullptr; - for (int I = 0, Sz = Mask.size(); I < Sz; ++I) { - int Idx = Mask[I]; - if (Idx == UndefMaskElem) - continue; - auto *EI = cast(E->Scalars[I]); - VecBase = EI->getVectorOperand(); - // If the only one use is vectorized - can delete the extractelement - // itself. - if (!EI->hasOneUse() || any_of(EI->users(), [&](User *U) { - return !ScalarToTreeEntry.count(U); - })) - continue; - eraseInstruction(EI); - } - return VecBase; - }; - auto NeedToDelay = [=](const TreeEntry *E, - ArrayRef Deps) -> Value * { - // No need to delay emission if all deps are ready. - if (all_of(Deps, [](const TreeEntry *TE) { return TE->VectorizedValue; })) - return nullptr; - // Postpone gather emission, will be emitted after the end of the - // process to keep correct order. - auto *VecTy = FixedVectorType::get(E->Scalars.front()->getType(), - E->getVectorFactor()); - Value *Vec = Builder.CreateAlignedLoad( - VecTy, PoisonValue::get(VecTy->getPointerTo()), MaybeAlign()); - return Vec; - }; - bool NeedFreeze = false; SmallVector ReuseShuffleIndicies(E->ReuseShuffleIndices.begin(), E->ReuseShuffleIndices.end()); @@ -9492,7 +9493,7 @@ if (UserIgnoreList) IgnoredVals.assign(UserIgnoreList->begin(), UserIgnoreList->end()); bool Resized = false; - if (Value *VecBase = AdjustExtracts(E, ExtractMask)) + if (Value *VecBase = ShuffleBuilder.adjustExtracts(E, ExtractMask)) if (auto *VecBaseTy = dyn_cast(VecBase->getType())) if (VF == VecBaseTy->getNumElements() && GatheredScalars.size() != VF) { Resized = true; @@ -9508,7 +9509,7 @@ GatherShuffle = isGatherShuffledEntry(E, GatheredScalars, Mask, Entries); } if (GatherShuffle) { - if (Value *Delayed = NeedToDelay(E, Entries)) { + if (Value *Delayed = ShuffleBuilder.needToDelay(E, Entries)) { // Delay emission of gathers which are not ready yet. PostponedGathers.insert(E); // Postpone gather emission, will be emitted after the end of the