diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -738,11 +738,30 @@ return OpCost; } + TTI::ShuffleKind deduceShuffleKindFromMask(TTI::ShuffleKind Kind, + ArrayRef Mask) { + if (Mask.empty()) + return Kind; + if (Kind == TTI::SK_PermuteSingleSrc) { + if (ShuffleVectorInst::isReverseMask(Mask)) + return TargetTransformInfo::SK_Reverse; + if (ShuffleVectorInst::isZeroEltSplatMask(Mask)) + return TargetTransformInfo::SK_Broadcast; + } else if (Kind == TTI::SK_PermuteTwoSrc) { + if (ShuffleVectorInst::isSelectMask(Mask)) + return TargetTransformInfo::SK_Select; + if (ShuffleVectorInst::isTransposeMask(Mask)) + return TargetTransformInfo::SK_Transpose; + return Kind; + } + return Kind; + } + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, int Index, VectorType *SubTp) { - switch (Kind) { + switch (deduceShuffleKindFromMask(Kind, Mask)) { case TTI::SK_Broadcast: return getBroadcastShuffleOverhead(cast(Tp)); case TTI::SK_Select: diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1437,6 +1437,7 @@ VectorType *Tp, ArrayRef Mask, int Index, VectorType *SubTp) { + Kind = deduceShuffleKindFromMask(Kind, Mask); if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose || Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc || Kind == TTI::SK_Reverse) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -1138,6 +1138,7 @@ InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *VT, ArrayRef Mask, int Index, VectorType *SubTp) { + Kind = deduceShuffleKindFromMask(Kind, Mask); if (ST->hasVOP3PInsts()) { if (cast(VT)->getNumElements() == 2 && DL.getTypeSizeInBits(VT->getElementType()) == 16) { diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1138,6 +1138,7 @@ InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, int Index, VectorType *SubTp) { + Kind = deduceShuffleKindFromMask(Kind, Mask); if (ST->hasNEON()) { if (Kind == TTI::SK_Broadcast) { static const CostTblEntry NEONDupTbl[] = { diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -558,6 +558,7 @@ VectorType *Tp, ArrayRef Mask, int Index, VectorType *SubTp) { + Kind = deduceShuffleKindFromMask(Kind, Mask); if (ST->hasVector()) { unsigned NumVectors = getNumVectorRegs(Tp); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -968,6 +968,7 @@ // 64-bit packed integer vectors (v2i32) are widened to type v4i32. std::pair LT = TLI->getTypeLegalizationCost(DL, BaseTp); + Kind = deduceShuffleKindFromMask(Kind, Mask); // Treat Transpose as 2-op shuffles - there's no difference in lowering. if (Kind == TTI::SK_Transpose) Kind = TTI::SK_PermuteTwoSrc; diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -4310,18 +4310,10 @@ } return None; } - if (NumShuffles == 1) { - if (ShuffleVectorInst::isReverseMask(Mask)) - return TargetTransformInfo::SK_Reverse; + if (NumShuffles == 1) return TargetTransformInfo::SK_PermuteSingleSrc; - } - if (NumShuffles == 2) { - if (ShuffleVectorInst::isSelectMask(Mask)) - return TargetTransformInfo::SK_Select; - if (ShuffleVectorInst::isTransposeMask(Mask)) - return TargetTransformInfo::SK_Transpose; + if (NumShuffles == 2) return TargetTransformInfo::SK_PermuteTwoSrc; - } return None; }