diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -611,6 +611,8 @@ } /// Check if ArrayType or StructType is isomorphic to some VectorType. + /// Accepts homogeneous aggregate of vectors like + /// { <2 x float>, <2 x float> } /// /// \returns number of elements in vector if isomorphism exists, 0 otherwise. unsigned canMapToVector(Type *T, const DataLayout &DL) const; @@ -2872,6 +2874,14 @@ N = cast(T)->getNumElements(); EltTy = cast(T)->getElementType(); } + + if (auto *VT = dyn_cast(EltTy)) { + if (VT->isScalable()) + return 0; + EltTy = VT->getElementType(); + N *= VT->getNumElements(); + } + if (!isValidElementType(EltTy)) return 0; uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N)); @@ -2880,7 +2890,7 @@ if (ST) { // Check that struct is homogeneous. for (const auto *Ty : ST->elements()) - if (Ty != EltTy) + if (Ty != *ST->element_begin()) return 0; } return N; @@ -6745,12 +6755,27 @@ } /// Like findBuildVector, but looks for construction of aggregate. +/// Accepts homegeneous aggregate of vectors like { <2 x float>, <2 x float> }. /// /// \return true if it matches. static bool findBuildAggregate(InsertValueInst *IV, - SmallVectorImpl &BuildVectorOpds) { + TargetTransformInfo *TTI, + SmallVectorImpl &BuildVectorOpds, + int &UserCost) { + UserCost = 0; do { - BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + if (auto *IE = dyn_cast(IV->getInsertedValueOperand())) { + int TmpUserCost; + SmallVector TmpBuildVectorOpds; + if (!findBuildVector(IE, TTI, TmpBuildVectorOpds, TmpUserCost)) + return false; + for (auto OpdIt = TmpBuildVectorOpds.rbegin(); + OpdIt != TmpBuildVectorOpds.rend(); OpdIt++) + BuildVectorOpds.push_back(*OpdIt); + UserCost += TmpUserCost; + } else { + BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + } Value *V = IV->getAggregateOperand(); if (isa(V)) break; @@ -6923,18 +6948,19 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, BoUpSLP &R) { + int UserCost = 0; const DataLayout &DL = BB->getModule()->getDataLayout(); if (!R.canMapToVector(IVI->getType(), DL)) return false; SmallVector BuildVectorOpds; - if (!findBuildAggregate(IVI, BuildVectorOpds)) + if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, UserCost)) return false; LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); // Aggregate value is unlikely to be processed in vector register, we need to // extract scalars into scalar registers, so NeedExtraction is set true. - return tryToVectorizeList(BuildVectorOpds, R); + return tryToVectorizeList(BuildVectorOpds, R, UserCost); } bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll @@ -0,0 +1,35 @@ +; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s +; CHECK: fadd <4 x float> + +%struct.Vector4 = type { float, float, float, float } + +; Function Attrs: norecurse nounwind readonly uwtable +define dso_local { <2 x float>, <2 x float> } @_Z6AddBadRK7Vector4S1_(%struct.Vector4* nocapture readonly dereferenceable(16) %0, %struct.Vector4* nocapture readonly dereferenceable(16) %1) local_unnamed_addr #0 { + %3 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %0, i64 0, i32 0 + %4 = load float, float* %3, align 4 + %5 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %1, i64 0, i32 0 + %6 = load float, float* %5, align 4 + %7 = fadd float %4, %6 + %8 = insertelement <2 x float> undef, float %7, i32 0 + %9 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %0, i64 0, i32 1 + %10 = load float, float* %9, align 4 + %11 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %1, i64 0, i32 1 + %12 = load float, float* %11, align 4 + %13 = fadd float %10, %12 + %14 = insertelement <2 x float> %8, float %13, i32 1 + %15 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %0, i64 0, i32 2 + %16 = load float, float* %15, align 4 + %17 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %1, i64 0, i32 2 + %18 = load float, float* %17, align 4 + %19 = fadd float %16, %18 + %20 = insertelement <2 x float> undef, float %19, i32 0 + %21 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %0, i64 0, i32 3 + %22 = load float, float* %21, align 4 + %23 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %1, i64 0, i32 3 + %24 = load float, float* %23, align 4 + %25 = fadd float %22, %24 + %26 = insertelement <2 x float> %20, float %25, i32 1 + %27 = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> %14, 0 + %28 = insertvalue { <2 x float>, <2 x float> } %27, <2 x float> %26, 1 + ret { <2 x float>, <2 x float> } %28 +}