diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -611,6 +611,8 @@ } /// Check if ArrayType or StructType is isomorphic to some VectorType. + /// Accepts homogeneous aggregate of vectors like + /// { <2 x float>, <2 x float> } /// /// \returns number of elements in vector if isomorphism exists, 0 otherwise. unsigned canMapToVector(Type *T, const DataLayout &DL) const; @@ -2872,6 +2874,12 @@ N = cast(T)->getNumElements(); EltTy = cast(T)->getElementType(); } + + if (auto *VT = dyn_cast(EltTy)) { + EltTy = VT->getElementType(); + N *= VT->getNumElements(); + } + if (!isValidElementType(EltTy)) return 0; uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N)); @@ -2880,7 +2888,7 @@ if (ST) { // Check that struct is homogeneous. for (const auto *Ty : ST->elements()) - if (Ty != EltTy) + if (Ty != *ST->element_begin()) return 0; } return N; @@ -6745,12 +6753,24 @@ } /// Like findBuildVector, but looks for construction of aggregate. +/// Accepts homegeneous aggregate of vectors like { <2 x float>, <2 x float> }. /// /// \return true if it matches. -static bool findBuildAggregate(InsertValueInst *IV, - SmallVectorImpl &BuildVectorOpds) { +static bool findBuildAggregate(InsertValueInst *IV, TargetTransformInfo *TTI, + SmallVectorImpl &BuildVectorOpds, + int &UserCost) { + UserCost = 0; do { - BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + if (auto *IE = dyn_cast(IV->getInsertedValueOperand())) { + int TmpUserCost; + SmallVector TmpBuildVectorOpds; + if (!findBuildVector(IE, TTI, TmpBuildVectorOpds, TmpUserCost)) + return false; + BuildVectorOpds.append(TmpBuildVectorOpds.rbegin(), TmpBuildVectorOpds.rend()); + UserCost += TmpUserCost; + } else { + BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + } Value *V = IV->getAggregateOperand(); if (isa(V)) break; @@ -6923,18 +6943,19 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, BoUpSLP &R) { + int UserCost = 0; const DataLayout &DL = BB->getModule()->getDataLayout(); if (!R.canMapToVector(IVI->getType(), DL)) return false; SmallVector BuildVectorOpds; - if (!findBuildAggregate(IVI, BuildVectorOpds)) + if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, UserCost)) return false; LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); // Aggregate value is unlikely to be processed in vector register, we need to // extract scalars into scalar registers, so NeedExtraction is set true. - return tryToVectorizeList(BuildVectorOpds, R); + return tryToVectorizeList(BuildVectorOpds, R, UserCost); } bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll @@ -0,0 +1,43 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +; Checks that vector insertvalues into the struct become SLP seeds. +define { <2 x float>, <2 x float> } @StructOfVectors(float *%Ptr) { +; CHECK-LABEL: @StructOfVectors( +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>* +; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0 +; CHECK-NEXT: [[VECIN0:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1 +; CHECK-NEXT: [[VECIN1:%.*]] = insertelement <2 x float> [[VECIN0]], float [[TMP4]], i64 1 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2 +; CHECK-NEXT: [[VECIN2:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3 +; CHECK-NEXT: [[VECIN3:%.*]] = insertelement <2 x float> [[VECIN2]], float [[TMP6]], i64 1 +; CHECK-NEXT: [[RET0:%.*]] = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> [[VECIN1]], 0 +; CHECK-NEXT: [[RET1:%.*]] = insertvalue { <2 x float>, <2 x float> } [[RET0]], <2 x float> [[VECIN3]], 1 +; CHECK-NEXT: ret { <2 x float>, <2 x float> } [[RET1]] +; + %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0 + %L0 = load float, float * %GEP0 + %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1 + %L1 = load float, float * %GEP1 + %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2 + %L2 = load float, float * %GEP2 + %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3 + %L3 = load float, float * %GEP3 + + %VecIn0 = insertelement <2 x float> undef, float %L0, i64 0 + %VecIn1 = insertelement <2 x float> %VecIn0, float %L1, i64 1 + + %VecIn2 = insertelement <2 x float> undef, float %L2, i64 0 + %VecIn3 = insertelement <2 x float> %VecIn2, float %L3, i64 1 + + %Ret0 = insertvalue {<2 x float>, <2 x float>} undef, <2 x float> %VecIn1, 0 + %Ret1 = insertvalue {<2 x float>, <2 x float>} %Ret0, <2 x float> %VecIn3, 1 + ret {<2 x float>, <2 x float>} %Ret1 +}