Index: lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/SLPVectorizer.cpp +++ lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2192,10 +2192,24 @@ // If all users are going to be vectorized, instruction can be // considered as dead. // The same, if have only one user, it will be vectorized for sure. - if (areAllUsersVectorized(E)) + if (areAllUsersVectorized(E)) { // Take credit for instruction that will become dead. + if (E->hasOneUse()) { + Instruction *Ext = E->user_back(); + if (isa(Ext) || isa(Ext)) { + // Use getExtractWithExtendCost() to calculate the cost of + // extractelement/ext pair. + DeadCost -= TTI->getExtractWithExtendCost( + Ext->getOpcode(), Ext->getType(), VecTy, i); + // Add back the cost of s|zext which is subtracted seperately. + DeadCost += TTI->getCastInstrCost( + Ext->getOpcode(), Ext->getType(), E->getType(), Ext); + continue; + } + } DeadCost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i); + } } return DeadCost; } Index: test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll =================================================================== --- test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll +++ test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -instcombine < %s | FileCheck %s +; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu < %s | FileCheck %s target datalayout = "e-m:e-i32:64-i128:128-n32:64-S128" @@ -11,23 +11,23 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: [[Z0:%.*]] = zext <4 x i16> [[A:%.*]] to <4 x i32> ; CHECK-NEXT: [[Z1:%.*]] = zext <4 x i16> [[B:%.*]] to <4 x i32> -; CHECK-NEXT: [[SUB0:%.*]] = sub nsw <4 x i32> [[Z0]], [[Z1]] -; CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x i32> [[SUB0]], i32 0 -; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[TMP1]] -; CHECK-NEXT: [[LOAD0:%.*]] = load i64, i64* [[GEP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[SUB0]], i32 1 -; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64 -; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP3]] -; CHECK-NEXT: [[LOAD1:%.*]] = load i64, i64* [[GEP1]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[SUB0]], i32 2 -; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[TMP4]] to i64 -; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP5]] -; CHECK-NEXT: [[LOAD2:%.*]] = load i64, i64* [[GEP2]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 -; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[TMP6]] to i64 -; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[LOAD3:%.*]] = load i64, i64* [[GEP3]], align 4 +; CHECK-NEXT: [[SUB0:%.*]] = sub <4 x i32> [[Z0]], [[Z1]] +; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x i32> [[SUB0]], i32 0 +; CHECK-NEXT: [[S0:%.*]] = sext i32 [[E0]] to i64 +; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[S0]] +; CHECK-NEXT: [[LOAD0:%.*]] = load i64, i64* [[GEP0]] +; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x i32> [[SUB0]], i32 1 +; CHECK-NEXT: [[S1:%.*]] = sext i32 [[E1]] to i64 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S1]] +; CHECK-NEXT: [[LOAD1:%.*]] = load i64, i64* [[GEP1]] +; CHECK-NEXT: [[E2:%.*]] = extractelement <4 x i32> [[SUB0]], i32 2 +; CHECK-NEXT: [[S2:%.*]] = sext i32 [[E2]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S2]] +; CHECK-NEXT: [[LOAD2:%.*]] = load i64, i64* [[GEP2]] +; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 +; CHECK-NEXT: [[S3:%.*]] = sext i32 [[E3]] to i64 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S3]] +; CHECK-NEXT: [[LOAD3:%.*]] = load i64, i64* [[GEP3]] ; CHECK-NEXT: call void @foo(i64 [[LOAD0]], i64 [[LOAD1]], i64 [[LOAD2]], i64 [[LOAD3]]) ; CHECK-NEXT: ret void ;