diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -611,6 +611,8 @@ } /// Check if ArrayType or StructType is isomorphic to some VectorType. + /// Accepts homogeneous aggregate of vectors like + /// { <2 x float>, <2 x float> } /// /// \returns number of elements in vector if isomorphism exists, 0 otherwise. unsigned canMapToVector(Type *T, const DataLayout &DL) const; @@ -2872,6 +2874,12 @@ N = cast(T)->getNumElements(); EltTy = cast(T)->getElementType(); } + + if (auto *VT = dyn_cast(EltTy)) { + EltTy = VT->getElementType(); + N *= VT->getNumElements(); + } + if (!isValidElementType(EltTy)) return 0; uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N)); @@ -2880,7 +2888,7 @@ if (ST) { // Check that struct is homogeneous. for (const auto *Ty : ST->elements()) - if (Ty != EltTy) + if (Ty != *ST->element_begin()) return 0; } return N; @@ -6745,12 +6753,24 @@ } /// Like findBuildVector, but looks for construction of aggregate. +/// Accepts homegeneous aggregate of vectors like { <2 x float>, <2 x float> }. /// /// \return true if it matches. -static bool findBuildAggregate(InsertValueInst *IV, - SmallVectorImpl &BuildVectorOpds) { +static bool findBuildAggregate(InsertValueInst *IV, TargetTransformInfo *TTI, + SmallVectorImpl &BuildVectorOpds, + int &UserCost) { + UserCost = 0; do { - BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + if (auto *IE = dyn_cast(IV->getInsertedValueOperand())) { + int TmpUserCost; + SmallVector TmpBuildVectorOpds; + if (!findBuildVector(IE, TTI, TmpBuildVectorOpds, TmpUserCost)) + return false; + BuildVectorOpds.append(TmpBuildVectorOpds.rbegin(), TmpBuildVectorOpds.rend()); + UserCost += TmpUserCost; + } else { + BuildVectorOpds.push_back(IV->getInsertedValueOperand()); + } Value *V = IV->getAggregateOperand(); if (isa(V)) break; @@ -6923,18 +6943,19 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, BoUpSLP &R) { + int UserCost = 0; const DataLayout &DL = BB->getModule()->getDataLayout(); if (!R.canMapToVector(IVI->getType(), DL)) return false; SmallVector BuildVectorOpds; - if (!findBuildAggregate(IVI, BuildVectorOpds)) + if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, UserCost)) return false; LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); // Aggregate value is unlikely to be processed in vector register, we need to // extract scalars into scalar registers, so NeedExtraction is set true. - return tryToVectorizeList(BuildVectorOpds, R); + return tryToVectorizeList(BuildVectorOpds, R, UserCost); } bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll @@ -0,0 +1,185 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +; See https://reviews.llvm.org/D70068 for context + +; Checks that vector insertvalues into the struct become SLP seeds. +define { <2 x float>, <2 x float> } @StructOfVectors(float *%Ptr) { +; CHECK-LABEL: @StructOfVectors( +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>* +; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0 +; CHECK-NEXT: [[VECIN0:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1 +; CHECK-NEXT: [[VECIN1:%.*]] = insertelement <2 x float> [[VECIN0]], float [[TMP4]], i64 1 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2 +; CHECK-NEXT: [[VECIN2:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3 +; CHECK-NEXT: [[VECIN3:%.*]] = insertelement <2 x float> [[VECIN2]], float [[TMP6]], i64 1 +; CHECK-NEXT: [[RET0:%.*]] = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> [[VECIN1]], 0 +; CHECK-NEXT: [[RET1:%.*]] = insertvalue { <2 x float>, <2 x float> } [[RET0]], <2 x float> [[VECIN3]], 1 +; CHECK-NEXT: ret { <2 x float>, <2 x float> } [[RET1]] +; + %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0 + %L0 = load float, float * %GEP0 + %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1 + %L1 = load float, float * %GEP1 + %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2 + %L2 = load float, float * %GEP2 + %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3 + %L3 = load float, float * %GEP3 + + %VecIn0 = insertelement <2 x float> undef, float %L0, i64 0 + %VecIn1 = insertelement <2 x float> %VecIn0, float %L1, i64 1 + + %VecIn2 = insertelement <2 x float> undef, float %L2, i64 0 + %VecIn3 = insertelement <2 x float> %VecIn2, float %L3, i64 1 + + %Ret0 = insertvalue {<2 x float>, <2 x float>} undef, <2 x float> %VecIn1, 0 + %Ret1 = insertvalue {<2 x float>, <2 x float>} %Ret0, <2 x float> %VecIn3, 1 + ret {<2 x float>, <2 x float>} %Ret1 +} + +%StructTy = type { float, float} + +define [2 x %StructTy] @ArrayOfStruct(float *%Ptr) { +; CHECK-LABEL: @ArrayOfStruct( +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0 +; CHECK-NEXT: [[L0:%.*]] = load float, float* [[GEP0]] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1 +; CHECK-NEXT: [[L1:%.*]] = load float, float* [[GEP1]] +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2 +; CHECK-NEXT: [[L2:%.*]] = load float, float* [[GEP2]] +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3 +; CHECK-NEXT: [[L3:%.*]] = load float, float* [[GEP3]] +; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01 +; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01 +; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01 +; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01 +; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0 +; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[FADD1]], 1 +; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[FADD2]], 0 +; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] %StructIn2, float [[FADD3]], 1 +; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x %StructTy] undef, [[STRUCTTY]] %StructIn1, 0 +; CHECK-NEXT: [[RET1:%.*]] = insertvalue [2 x %StructTy] [[RET0]], [[STRUCTTY]] %StructIn3, 1 +; CHECK-NEXT: ret [2 x %StructTy] [[RET1]] +; + %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0 + %L0 = load float, float * %GEP0 + %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1 + %L1 = load float, float * %GEP1 + %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2 + %L2 = load float, float * %GEP2 + %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3 + %L3 = load float, float * %GEP3 + + %Fadd0 = fadd fast float %L0, 1.1e+01 + %Fadd1 = fadd fast float %L1, 1.2e+01 + %Fadd2 = fadd fast float %L2, 1.3e+01 + %Fadd3 = fadd fast float %L3, 1.4e+01 + + %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0 + %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1 + + %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0 + %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1 + + %Ret0 = insertvalue [2 x %StructTy] undef, %StructTy %StructIn1, 0 + %Ret1 = insertvalue [2 x %StructTy] %Ret0, %StructTy %StructIn3, 1 + ret [2 x %StructTy] %Ret1 +} + +define {%StructTy, %StructTy} @StructOfStruct(float *%Ptr) { +; CHECK-LABEL: @StructOfStruct( +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0 +; CHECK-NEXT: [[L0:%.*]] = load float, float* [[GEP0]] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1 +; CHECK-NEXT: [[L1:%.*]] = load float, float* [[GEP1]] +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2 +; CHECK-NEXT: [[L2:%.*]] = load float, float* [[GEP2]] +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3 +; CHECK-NEXT: [[L3:%.*]] = load float, float* [[GEP3]] +; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01 +; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01 +; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01 +; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01 +; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0 +; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[FADD1]], 1 +; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[FADD2]], 0 +; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] %StructIn2, float [[FADD3]], 1 +; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } undef, [[STRUCTTY]] %StructIn1, 0 +; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } [[RET0]], [[STRUCTTY]] %StructIn3, 1 +; CHECK-NEXT: ret { [[STRUCTTY]], [[STRUCTTY]] } [[RET1]] +; + %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0 + %L0 = load float, float * %GEP0 + %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1 + %L1 = load float, float * %GEP1 + %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2 + %L2 = load float, float * %GEP2 + %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3 + %L3 = load float, float * %GEP3 + + %Fadd0 = fadd fast float %L0, 1.1e+01 + %Fadd1 = fadd fast float %L1, 1.2e+01 + %Fadd2 = fadd fast float %L2, 1.3e+01 + %Fadd3 = fadd fast float %L3, 1.4e+01 + + %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0 + %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1 + + %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0 + %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1 + + %Ret0 = insertvalue {%StructTy, %StructTy} undef, %StructTy %StructIn1, 0 + %Ret1 = insertvalue {%StructTy, %StructTy} %Ret0, %StructTy %StructIn3, 1 + ret {%StructTy, %StructTy} %Ret1 +} + +define {%StructTy, float, float} @NonHomogeneousStruct(float *%Ptr) { +; CHECK-LABEL: @NonHomogeneousStruct( +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0 +; CHECK-NEXT: [[L0:%.*]] = load float, float* [[GEP0]] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1 +; CHECK-NEXT: [[L1:%.*]] = load float, float* [[GEP1]] +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2 +; CHECK-NEXT: [[L2:%.*]] = load float, float* [[GEP2]] +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3 +; CHECK-NEXT: [[L3:%.*]] = load float, float* [[GEP3]] +; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01 +; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01 +; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01 +; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01 +; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0 +; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] %StructIn0, float [[FADD1]], 1 +; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], float, float } undef, [[STRUCTTY]] %StructIn1, 0 +; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET0]], float [[FADD2]], 1 +; CHECK-NEXT: [[RET2:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET1]], float [[FADD3]], 2 +; CHECK-NEXT: ret { [[STRUCTTY]], float, float } [[RET2]] +; + %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0 + %L0 = load float, float * %GEP0 + %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1 + %L1 = load float, float * %GEP1 + %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2 + %L2 = load float, float * %GEP2 + %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3 + %L3 = load float, float * %GEP3 + + %Fadd0 = fadd fast float %L0, 1.1e+01 + %Fadd1 = fadd fast float %L1, 1.2e+01 + %Fadd2 = fadd fast float %L2, 1.3e+01 + %Fadd3 = fadd fast float %L3, 1.4e+01 + + %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0 + %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1 + + %Ret0 = insertvalue {%StructTy, float, float} undef, %StructTy %StructIn1, 0 + %Ret1 = insertvalue {%StructTy, float, float} %Ret0, float %Fadd2, 1 + %Ret2 = insertvalue {%StructTy, float, float} %Ret1, float %Fadd3, 2 + ret {%StructTy, float, float} %Ret2 +}