diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -611,6 +611,8 @@ } /// Check if ArrayType or StructType is isomorphic to some VectorType. + /// Accepts homogeneous aggregate of vectors like + /// { <2 x float>, <2 x float> } /// /// \returns number of elements in vector if isomorphism exists, 0 otherwise. unsigned canMapToVector(Type *T, const DataLayout &DL) const; @@ -2872,6 +2874,14 @@ N = cast(T)->getNumElements(); EltTy = cast(T)->getElementType(); } + + if (auto *VT = dyn_cast(EltTy)) { + if (VT->isScalable()) + return 0; + EltTy = VT->getElementType(); + N *= VT->getNumElements(); + } + if (!isValidElementType(EltTy)) return 0; uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N)); @@ -2880,7 +2890,7 @@ if (ST) { // Check that struct is homogeneous. for (const auto *Ty : ST->elements()) - if (Ty != EltTy) + if (Ty != *ST->element_begin()) return 0; } return N; @@ -6745,12 +6755,27 @@ } /// Like findBuildVector, but looks for construction of aggregate. +/// Accepts homegeneous aggregate of vectors like { <2 x float>, <2 x float> }. /// /// \return true if it matches. static bool findBuildAggregate(InsertValueInst *IV, - SmallVectorImpl &BuildVectorOpds) { + TargetTransformInfo *TTI, + SmallVectorImpl &BuildVectorOpds, + int &UserCost) { + UserCost = 0; do { - BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + if (auto *IE = dyn_cast(IV->getInsertedValueOperand())) { + int TmpUserCost; + SmallVector TmpBuildVectorOpds; + if (!findBuildVector(IE, TTI, TmpBuildVectorOpds, TmpUserCost)) + return false; + for (auto OpdIt = TmpBuildVectorOpds.rbegin(); + OpdIt != TmpBuildVectorOpds.rend(); OpdIt++) + BuildVectorOpds.push_back(*OpdIt); + UserCost += TmpUserCost; + } else { + BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + } Value *V = IV->getAggregateOperand(); if (isa(V)) break; @@ -6923,18 +6948,19 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, BoUpSLP &R) { + int UserCost = 0; const DataLayout &DL = BB->getModule()->getDataLayout(); if (!R.canMapToVector(IVI->getType(), DL)) return false; SmallVector BuildVectorOpds; - if (!findBuildAggregate(IVI, BuildVectorOpds)) + if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, UserCost)) return false; LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); // Aggregate value is unlikely to be processed in vector register, we need to // extract scalars into scalar registers, so NeedExtraction is set true. - return tryToVectorizeList(BuildVectorOpds, R); + return tryToVectorizeList(BuildVectorOpds, R, UserCost); } bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,