diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6300,16 +6300,33 @@ return TTI.getShuffleCost(ShuffleKind, VecTy, Mask); bool AllConsecutive = true; + auto IsExtract = [](Value *V) { + return isa(V) || isa(V); + }; + bool ExtractFromCommonOperand = true; unsigned EltsPerVector = VecTy->getNumElements() / NumOfParts; unsigned Idx = -1; InstructionCost Cost = 0; - // Process extracts in blocks of EltsPerVector to check if the source vector - // operand can be re-used directly. If not, add the cost of creating a shuffle - // to extract the values into a vector register. + auto GetExtractOperand = [](Value *V) -> Value * { + if (auto *EEI = dyn_cast(V)) + return EEI->getVectorOperand(); + if (auto *EVI = dyn_cast(V)) + return EVI->getAggregateOperand(); + llvm_unreachable("Expected ExtractElement or ExtractValue"); + }; + Value *ExtractOperand0 = IsExtract(VL[0]) ? GetExtractOperand(VL[0]) : VL[0]; + + // Process extracts in blocks of EltsPerVector to check if the source + // vector operand can be re-used directly. If not, add the cost of + // creating a shuffle to extract the values into a vector register. SmallVector RegMask(EltsPerVector, UndefMaskElem); for (auto *V : VL) { ++Idx; + // Check if we extract from a single common operand. + Value *Operand = IsExtract(V) ? GetExtractOperand(V) : V; + if (Operand != ExtractOperand0) + ExtractFromCommonOperand = false; // Reached the start of a new vector registers. if (Idx % EltsPerVector == 0) { @@ -6332,7 +6349,9 @@ RegMask[Idx % EltsPerVector] = CurrentIdx % EltsPerVector; } - if (AllConsecutive) + // If all are consecutive and are all extracts from the same operand then we + // can reuse that operand. + if (AllConsecutive && ExtractFromCommonOperand) continue; // Skip all indices, except for the last index per vector block. diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-float-and-extract-lane1.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-float-and-extract-lane1.ll --- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-float-and-extract-lane1.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-float-and-extract-lane1.ll @@ -13,7 +13,7 @@ ; YAML: Function: test ; YAML: Args: ; YAML: - String: 'SLP vectorized with cost ' -; YAML: - Cost: '-2' +; YAML: - Cost: '-1' ; YAML: - String: ' and with tree size ' ; YAML: - TreeSize: '3' ; YAML: ...