diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -935,31 +935,28 @@ ArrayRef Mask, VectorType *Ty, int &Index, VectorType *&SubTy) const { - int Limit = Mask.size() * 2; - if (Mask.empty() || - // Extra check required by isSingleSourceMaskImpl function (called by - // ShuffleVectorInst::isSingleSourceMask). - any_of(Mask, [Limit](int I) { return I >= Limit; })) + if (Mask.empty()) return Kind; + int NumSrcElts = Ty->getElementCount().getKnownMinValue(); switch (Kind) { case TTI::SK_PermuteSingleSrc: - if (ShuffleVectorInst::isReverseMask(Mask)) + if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts)) return TTI::SK_Reverse; - if (ShuffleVectorInst::isZeroEltSplatMask(Mask)) + if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts)) return TTI::SK_Broadcast; break; case TTI::SK_PermuteTwoSrc: { int NumSubElts; if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask( - Mask, Mask.size(), NumSubElts, Index)) { + Mask, NumSrcElts, NumSubElts, Index)) { SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts); return TTI::SK_InsertSubvector; } - if (ShuffleVectorInst::isSelectMask(Mask)) + if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts)) return TTI::SK_Select; - if (ShuffleVectorInst::isTransposeMask(Mask)) + if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts)) return TTI::SK_Transpose; - if (ShuffleVectorInst::isSpliceMask(Mask, Index)) + if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index)) return TTI::SK_Splice; break; } diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -2068,13 +2068,14 @@ /// Return true if this shuffle mask chooses elements from exactly one source /// vector. /// Example: <7,5,undef,7> - /// This assumes that vector operands are the same length as the mask. - static bool isSingleSourceMask(ArrayRef Mask); - static bool isSingleSourceMask(const Constant *Mask) { + /// This assumes that vector operands (of length \p NumSrcElts) are the same + /// length as the mask. + static bool isSingleSourceMask(ArrayRef Mask, int NumSrcElts); + static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSingleSourceMask(MaskAsInts); + return isSingleSourceMask(MaskAsInts, NumSrcElts); } /// Return true if this shuffle chooses elements from exactly one source @@ -2082,7 +2083,8 @@ /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3> /// TODO: Optionally allow length-changing shuffles. bool isSingleSource() const { - return !changesLength() && isSingleSourceMask(ShuffleMask); + return !changesLength() && + isSingleSourceMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle mask chooses elements from exactly one source @@ -2090,8 +2092,8 @@ /// necessarily a no-op because it may change the number of elements from its /// input vectors or it may provide demanded bits knowledge via undef lanes. /// Example: - static bool isIdentityMask(ArrayRef Mask); - static bool isIdentityMask(const Constant *Mask) { + static bool isIdentityMask(ArrayRef Mask, int NumSrcElts); + static bool isIdentityMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); // Not possible to express a shuffle mask for a scalable vector for this @@ -2101,7 +2103,7 @@ SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isIdentityMask(MaskAsInts); + return isIdentityMask(MaskAsInts, NumSrcElts); } /// Return true if this shuffle chooses elements from exactly one source @@ -2114,7 +2116,7 @@ if (isa(getType())) return false; - return !changesLength() && isIdentityMask(ShuffleMask); + return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle lengthens exactly one source vector with @@ -2138,12 +2140,12 @@ /// In that case, the shuffle is better classified as an identity shuffle. /// This assumes that vector operands are the same length as the mask /// (a length-changing shuffle can never be equivalent to a vector select). - static bool isSelectMask(ArrayRef Mask); - static bool isSelectMask(const Constant *Mask) { + static bool isSelectMask(ArrayRef Mask, int NumSrcElts); + static bool isSelectMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSelectMask(MaskAsInts); + return isSelectMask(MaskAsInts, NumSrcElts); } /// Return true if this shuffle chooses elements from its source vectors @@ -2155,19 +2157,20 @@ /// In that case, the shuffle is better classified as an identity shuffle. /// TODO: Optionally allow length-changing shuffles. bool isSelect() const { - return !changesLength() && isSelectMask(ShuffleMask); + return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle mask swaps the order of elements from exactly /// one source vector. /// Example: <7,6,undef,4> - /// This assumes that vector operands are the same length as the mask. - static bool isReverseMask(ArrayRef Mask); - static bool isReverseMask(const Constant *Mask) { + /// This assumes that vector operands (of length \p NumSrcElts) are the same + /// length as the mask. + static bool isReverseMask(ArrayRef Mask, int NumSrcElts); + static bool isReverseMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isReverseMask(MaskAsInts); + return isReverseMask(MaskAsInts, NumSrcElts); } /// Return true if this shuffle swaps the order of elements from exactly @@ -2175,19 +2178,20 @@ /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef> /// TODO: Optionally allow length-changing shuffles. bool isReverse() const { - return !changesLength() && isReverseMask(ShuffleMask); + return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle mask chooses all elements with the same value /// as the first element of exactly one source vector. /// Example: <4,undef,undef,4> - /// This assumes that vector operands are the same length as the mask. - static bool isZeroEltSplatMask(ArrayRef Mask); - static bool isZeroEltSplatMask(const Constant *Mask) { + /// This assumes that vector operands (of length \p NumSrcElts) are the same + /// length as the mask. + static bool isZeroEltSplatMask(ArrayRef Mask, int NumSrcElts); + static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isZeroEltSplatMask(MaskAsInts); + return isZeroEltSplatMask(MaskAsInts, NumSrcElts); } /// Return true if all elements of this shuffle are the same value as the @@ -2197,7 +2201,8 @@ /// TODO: Optionally allow length-changing shuffles. /// TODO: Optionally allow splats from other elements. bool isZeroEltSplat() const { - return !changesLength() && isZeroEltSplatMask(ShuffleMask); + return !changesLength() && + isZeroEltSplatMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle mask is a transpose mask. @@ -2232,12 +2237,12 @@ /// ; Transposed matrix /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 > /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 > - static bool isTransposeMask(ArrayRef Mask); - static bool isTransposeMask(const Constant *Mask) { + static bool isTransposeMask(ArrayRef Mask, int NumSrcElts); + static bool isTransposeMask(const Constant *Mask, int NumSrcElts) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isTransposeMask(MaskAsInts); + return isTransposeMask(MaskAsInts, NumSrcElts); } /// Return true if this shuffle transposes the elements of its inputs without @@ -2246,19 +2251,21 @@ /// exact specification. /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6> bool isTranspose() const { - return !changesLength() && isTransposeMask(ShuffleMask); + return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size()); } /// Return true if this shuffle mask is a splice mask, concatenating the two /// inputs together and then extracts an original width vector starting from /// the splice index. /// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> - static bool isSpliceMask(ArrayRef Mask, int &Index); - static bool isSpliceMask(const Constant *Mask, int &Index) { + /// This assumes that vector operands (of length \p NumSrcElts) are the same + /// length as the mask. + static bool isSpliceMask(ArrayRef Mask, int NumSrcElts, int &Index); + static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSpliceMask(MaskAsInts, Index); + return isSpliceMask(MaskAsInts, NumSrcElts, Index); } /// Return true if this shuffle splices two inputs without changing the length @@ -2266,7 +2273,8 @@ /// then extracts an original width vector starting from the splice index. /// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> bool isSplice(int &Index) const { - return !changesLength() && isSpliceMask(ShuffleMask, Index); + return !changesLength() && + isSpliceMask(ShuffleMask, ShuffleMask.size(), Index); } /// Return true if this shuffle mask is an extract subvector mask. diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -24119,7 +24119,7 @@ // Profitability check: only deal with extractions from the first subvector // unless the mask becomes an identity mask. - if (!ShuffleVectorInst::isIdentityMask(NewMask) || + if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) || any_of(NewMask, [](int M) { return M < 0; })) for (auto &DemandedSubvector : DemandedSubvectors) if (DemandedSubvector.second != 0) diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -2136,10 +2136,10 @@ return UsesLHS || UsesRHS; } -bool ShuffleVectorInst::isSingleSourceMask(ArrayRef Mask) { +bool ShuffleVectorInst::isSingleSourceMask(ArrayRef Mask, int NumSrcElts) { // We don't have vector operand size information, so assume operands are the // same size as the mask. - return isSingleSourceMaskImpl(Mask, Mask.size()); + return isSingleSourceMaskImpl(Mask, NumSrcElts); } static bool isIdentityMaskImpl(ArrayRef Mask, int NumOpElts) { @@ -2154,65 +2154,75 @@ return true; } -bool ShuffleVectorInst::isIdentityMask(ArrayRef Mask) { +bool ShuffleVectorInst::isIdentityMask(ArrayRef Mask, int NumSrcElts) { + if (Mask.size() != static_cast(NumSrcElts)) + return false; // We don't have vector operand size information, so assume operands are the // same size as the mask. - return isIdentityMaskImpl(Mask, Mask.size()); + return isIdentityMaskImpl(Mask, NumSrcElts); } -bool ShuffleVectorInst::isReverseMask(ArrayRef Mask) { - if (!isSingleSourceMask(Mask)) +bool ShuffleVectorInst::isReverseMask(ArrayRef Mask, int NumSrcElts) { + if (Mask.size() != static_cast(NumSrcElts)) + return false; + if (!isSingleSourceMask(Mask, NumSrcElts)) return false; // The number of elements in the mask must be at least 2. - int NumElts = Mask.size(); - if (NumElts < 2) + if (NumSrcElts < 2) return false; - for (int i = 0; i < NumElts; ++i) { - if (Mask[i] == -1) + for (int I = 0, E = Mask.size(); I < E; ++I) { + if (Mask[I] == -1) continue; - if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i)) + if (Mask[I] != (NumSrcElts - 1 - I) && + Mask[I] != (NumSrcElts + NumSrcElts - 1 - I)) return false; } return true; } -bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef Mask) { - if (!isSingleSourceMask(Mask)) +bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef Mask, int NumSrcElts) { + if (Mask.size() != static_cast(NumSrcElts)) return false; - for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) { - if (Mask[i] == -1) + if (!isSingleSourceMask(Mask, NumSrcElts)) + return false; + for (int I = 0, E = Mask.size(); I < E; ++I) { + if (Mask[I] == -1) continue; - if (Mask[i] != 0 && Mask[i] != NumElts) + if (Mask[I] != 0 && Mask[I] != NumSrcElts) return false; } return true; } -bool ShuffleVectorInst::isSelectMask(ArrayRef Mask) { +bool ShuffleVectorInst::isSelectMask(ArrayRef Mask, int NumSrcElts) { + if (Mask.size() != static_cast(NumSrcElts)) + return false; // Select is differentiated from identity. It requires using both sources. - if (isSingleSourceMask(Mask)) + if (isSingleSourceMask(Mask, NumSrcElts)) return false; - for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) { - if (Mask[i] == -1) + for (int I = 0, E = Mask.size(); I < E; ++I) { + if (Mask[I] == -1) continue; - if (Mask[i] != i && Mask[i] != (NumElts + i)) + if (Mask[I] != I && Mask[I] != (NumSrcElts + I)) return false; } return true; } -bool ShuffleVectorInst::isTransposeMask(ArrayRef Mask) { +bool ShuffleVectorInst::isTransposeMask(ArrayRef Mask, int NumSrcElts) { // Example masks that will return true: // v1 = // v2 = // trn1 = shufflevector v1, v2 <0, 4, 2, 6> = // trn2 = shufflevector v1, v2 <1, 5, 3, 7> = + if (Mask.size() != static_cast(NumSrcElts)) + return false; // 1. The number of elements in the mask must be a power-of-2 and at least 2. - int NumElts = Mask.size(); - if (NumElts < 2 || !isPowerOf2_32(NumElts)) + int Sz = Mask.size(); + if (Sz < 2 || !isPowerOf2_32(Sz)) return false; // 2. The first element of the mask must be either a 0 or a 1. @@ -2221,23 +2231,26 @@ // 3. The difference between the first 2 elements must be equal to the // number of elements in the mask. - if ((Mask[1] - Mask[0]) != NumElts) + if ((Mask[1] - Mask[0]) != NumSrcElts) return false; // 4. The difference between consecutive even-numbered and odd-numbered // elements must be equal to 2. - for (int i = 2; i < NumElts; ++i) { - int MaskEltVal = Mask[i]; + for (int I = 2; I < Sz; ++I) { + int MaskEltVal = Mask[I]; if (MaskEltVal == -1) return false; - int MaskEltPrevVal = Mask[i - 2]; + int MaskEltPrevVal = Mask[I - 2]; if (MaskEltVal - MaskEltPrevVal != 2) return false; } return true; } -bool ShuffleVectorInst::isSpliceMask(ArrayRef Mask, int &Index) { +bool ShuffleVectorInst::isSpliceMask(ArrayRef Mask, int NumSrcElts, + int &Index) { + if (Mask.size() != static_cast(NumSrcElts)) + return false; // Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> int StartIndex = -1; for (int I = 0, E = Mask.size(); I != E; ++I) { @@ -2248,7 +2261,7 @@ if (StartIndex == -1) { // Don't support a StartIndex that begins in the second input, or if the // first non-undef index would access below the StartIndex. - if (MaskEltVal < I || E <= (MaskEltVal - I)) + if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I)) return false; StartIndex = MaskEltVal - I; @@ -2543,7 +2556,7 @@ // case. if (isa(getType())) return false; - if (!isSingleSourceMask(ShuffleMask)) + if (!isSingleSourceMask(ShuffleMask, VF)) return false; return isOneUseSingleSourceMask(ShuffleMask, VF); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -11802,7 +11802,7 @@ if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) || (VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) && - ShuffleVectorInst::isReverseMask(ShuffleMask)) { + ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) { SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1); return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev, DAG.getConstant(8, dl, MVT::i32)); @@ -25946,7 +25946,8 @@ unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits(); unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits(); if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) { - if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) { + if (ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size()) && + Op2.isUndef()) { Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1); return convertFromScalableVector(DAG, VT, Op); } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -8375,7 +8375,7 @@ unsigned EltSize = VT.getScalarSizeInBits(); if (EltSize >= 32 || ShuffleVectorSDNode::isSplatMask(&M[0], VT) || - ShuffleVectorInst::isIdentityMask(M) || + ShuffleVectorInst::isIdentityMask(M, M.size()) || isVREVMask(M, VT, 64) || isVREVMask(M, VT, 32) || isVREVMask(M, VT, 16)) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -4333,7 +4333,8 @@ assert(VT.getVectorElementType() == MVT::i1); - if (!ShuffleVectorInst::isReverseMask(SVN->getMask()) || + if (!ShuffleVectorInst::isReverseMask(SVN->getMask(), + SVN->getMask().size()) || !SVN->getOperand(1).isUndef()) return SDValue(); diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -322,7 +322,10 @@ LT.second.getVectorElementType().getSizeInBits() == Tp->getElementType()->getPrimitiveSizeInBits() && LT.second.getVectorNumElements() < - cast(Tp)->getNumElements()) { + cast(Tp)->getNumElements() && + divideCeil(Mask.size(), + cast(Tp)->getNumElements()) == + static_cast(*LT.first.getValue())) { unsigned NumRegs = *LT.first.getValue(); unsigned VF = cast(Tp)->getNumElements(); unsigned SubVF = PowerOf2Ceil(VF / NumRegs); @@ -498,7 +501,7 @@ InstructionCost Cost = MemCost; for (unsigned Index : Indices) { FixedVectorType *SubVecTy = - FixedVectorType::get(FVTy->getElementType(), VF); + FixedVectorType::get(FVTy->getElementType(), VF * Factor); auto Mask = createStrideMask(Index, Factor, VF); InstructionCost ShuffleCost = getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1635,7 +1635,7 @@ NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {}, [this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask, &Cost](ArrayRef RegMask, unsigned SrcReg, unsigned DestReg) { - if (!ShuffleVectorInst::isIdentityMask(RegMask)) { + if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) { // Check if the previous register can be just copied to the next // one. if (PrevRegMask.empty() || PrevSrcReg != SrcReg || diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -2122,8 +2122,8 @@ NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i]; // A select mask with undef elements might look like an identity mask. - assert((ShuffleVectorInst::isSelectMask(NewMask) || - ShuffleVectorInst::isIdentityMask(NewMask)) && + assert((ShuffleVectorInst::isSelectMask(NewMask, NumElts) || + ShuffleVectorInst::isIdentityMask(NewMask, NumElts)) && "Unexpected shuffle mask"); return new ShuffleVectorInst(X, Y, NewMask); } diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3812,7 +3812,7 @@ inversePermutation(Order, MaskOrder); } reorderReuses(MaskOrder, Mask); - if (ShuffleVectorInst::isIdentityMask(MaskOrder)) { + if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) { Order.clear(); return; } @@ -4333,7 +4333,7 @@ static bool isRepeatedNonIdentityClusteredMask(ArrayRef Mask, unsigned Sz) { ArrayRef FirstCluster = Mask.slice(0, Sz); - if (ShuffleVectorInst::isIdentityMask(FirstCluster)) + if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz)) return false; for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) { ArrayRef Cluster = Mask.slice(I, Sz); @@ -6646,9 +6646,24 @@ bool IsStrict) { int Limit = Mask.size(); int VF = VecTy->getNumElements(); - return (VF == Limit || !IsStrict) && - all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) && - ShuffleVectorInst::isIdentityMask(Mask); + int Index = -1; + if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit)) + return true; + if (!IsStrict) { + // Consider extract subvector starting from index 0. + if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) && + Index == 0) + return true; + // All VF-size submasks are identity (e.g. + // etc. for VF 4). + if (Limit % VF == 0 && all_of(seq(0, Limit / VF), [=](int Idx) { + ArrayRef Slice = Mask.slice(Idx * VF, VF); + return all_of(Slice, [](int I) { return I == PoisonMaskElem; }) || + ShuffleVectorInst::isIdentityMask(Slice, VF); + })) + return true; + } + return false; } /// Tries to combine 2 different masks into single one. @@ -6718,7 +6733,8 @@ if (isIdentityMask(Mask, SVTy, /*IsStrict=*/false)) { if (!IdentityOp || !SinglePermute || (isIdentityMask(Mask, SVTy, /*IsStrict=*/true) && - !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask))) { + !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask, + IdentityMask.size()))) { IdentityOp = SV; // Store current mask in the IdentityMask so later we did not lost // this info if IdentityOp is selected as the best candidate for the @@ -6788,7 +6804,7 @@ } if (auto *OpTy = dyn_cast(Op->getType()); !OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) || - ShuffleVectorInst::isZeroEltSplatMask(Mask)) { + ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) { if (IdentityOp) { V = IdentityOp; assert(Mask.size() == IdentityMask.size() && @@ -6804,7 +6820,7 @@ /*IsStrict=*/true) || (Shuffle && Mask.size() == Shuffle->getShuffleMask().size() && Shuffle->isZeroEltSplat() && - ShuffleVectorInst::isZeroEltSplatMask(Mask))); + ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))); } V = Op; return false; @@ -6909,11 +6925,9 @@ CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF); } } - const int Limit = CombinedMask1.size() * 2; - if (Op1 == Op2 && Limit == 2 * VF && - all_of(CombinedMask1, [=](int Idx) { return Idx < Limit; }) && - (ShuffleVectorInst::isIdentityMask(CombinedMask1) || - (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1) && + if (Op1 == Op2 && + (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) || + (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) && isa(Op1) && cast(Op1)->getShuffleMask() == ArrayRef(CombinedMask1)))) @@ -7183,11 +7197,9 @@ const TargetTransformInfo &TTI; static bool isEmptyOrIdentity(ArrayRef Mask, unsigned VF) { - int Limit = 2 * VF; return Mask.empty() || (VF == Mask.size() && - all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) && - ShuffleVectorInst::isIdentityMask(Mask)); + ShuffleVectorInst::isIdentityMask(Mask, VF)); } public: @@ -7445,9 +7457,7 @@ ::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true); if (CommonMask.empty()) return Cost; - int Limit = CommonMask.size() * 2; - if (all_of(CommonMask, [=](int Idx) { return Idx < Limit; }) && - ShuffleVectorInst::isIdentityMask(CommonMask)) + if (ShuffleVectorInst::isIdentityMask(CommonMask, CommonMask.size())) return Cost; return Cost + createShuffle(InVectors.front(), @@ -7624,7 +7634,7 @@ } if (NeedToShuffleReuses) ::addMask(Mask, E->ReuseShuffleIndices); - if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask)) + if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) CommonCost = TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask); assert((E->State == TreeEntry::Vectorize || @@ -7975,7 +7985,7 @@ } else { SmallVector Mask; inversePermutation(OpTE->ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask)) + if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) CCH = TTI::CastContextHint::Reversed; } } @@ -8834,9 +8844,7 @@ unsigned VecVF = TE->getVectorFactor(); if (VF != VecVF && (any_of(Mask, [VF](int Idx) { return Idx >= static_cast(VF); }) || - (all_of(Mask, - [VF](int Idx) { return Idx < 2 * static_cast(VF); }) && - !ShuffleVectorInst::isIdentityMask(Mask)))) { + !ShuffleVectorInst::isIdentityMask(Mask, VF))) { SmallVector OrigMask(VecVF, PoisonMaskElem); std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)), OrigMask.begin()); @@ -8865,9 +8873,7 @@ assert((TEs.size() == 1 || TEs.size() == 2) && "Expected exactly 1 or 2 tree entries."); if (TEs.size() == 1) { - int Limit = 2 * Mask.size(); - if (!all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) || - !ShuffleVectorInst::isIdentityMask(Mask)) { + if (!ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) { InstructionCost C = TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FTy, Mask); LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C @@ -9643,7 +9649,7 @@ return V1; unsigned VF = Mask.size(); unsigned LocalVF = cast(V1->getType())->getNumElements(); - if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask)) + if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF)) return V1; Value *Vec = Builder.CreateShuffleVector(V1, Mask); if (auto *I = dyn_cast(Vec)) { @@ -10056,9 +10062,7 @@ return false; unsigned I = *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; }); - int Sz = Mask.size(); - if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) && - ShuffleVectorInst::isIdentityMask(Mask)) + if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) std::iota(Mask.begin(), Mask.end(), 0); else std::fill(Mask.begin(), Mask.end(), I); @@ -10302,11 +10306,11 @@ (ExtractShuffle.value_or(TTI::SK_PermuteTwoSrc) == TTI::SK_PermuteSingleSrc && none_of(ExtractMask, [&](int I) { return I >= EMSz; }) && - ShuffleVectorInst::isIdentityMask(ExtractMask)) || + ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) || (GatherShuffle.value_or(TTI::SK_PermuteTwoSrc) == TTI::SK_PermuteSingleSrc && none_of(Mask, [&](int I) { return I >= MSz; }) && - ShuffleVectorInst::isIdentityMask(Mask)); + ShuffleVectorInst::isIdentityMask(Mask, MSz)); bool EnoughConstsForShuffle = IsSingleShuffle && (none_of(GatheredScalars, @@ -10599,7 +10603,7 @@ if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) && NumElts != NumScalars) { if (IsFirstUndef.all()) { - if (!ShuffleVectorInst::isIdentityMask(InsertMask)) { + if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) { SmallBitVector IsFirstPoison = isUndefVector(FirstInsert->getOperand(0), UseMask); if (!IsFirstPoison.all()) { @@ -11422,7 +11426,7 @@ // non-resizing mask. if (Mask.size() != cast(Vals.front()->getType()) ->getNumElements() || - !ShuffleVectorInst::isIdentityMask(Mask)) + !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) return CreateShuffle(Vals.front(), nullptr, Mask); return Vals.front(); } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll @@ -461,44 +461,21 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) { ; CHECK-LABEL: @combine_load_factor2_i64( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: vector.ph: -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0 -; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4 -; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> -; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]] -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 -; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] -; CHECK: middle.block: -; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] -; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1 -; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]] +; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]] ; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4 ; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1 ; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]] ; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4 ; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]] -; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]] +; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]] ; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 -; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: ; CHECK-NEXT: ret void ; diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp --- a/llvm/unittests/IR/InstructionsTest.cpp +++ b/llvm/unittests/IR/InstructionsTest.cpp @@ -1024,71 +1024,141 @@ Constant *C7 = ConstantInt::get(Int32Ty, 7); Constant *Identity = ConstantVector::get({C0, CU, C2, C3, C4}); - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(Identity)); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Identity)); // identity is distinguished from select - EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Identity)); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Identity)); // identity is always single source - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Identity)); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Identity)); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( + Identity, cast(Identity->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask( + Identity, + cast(Identity->getType()) + ->getNumElements())); // identity is distinguished from select + EXPECT_FALSE(ShuffleVectorInst::isReverseMask( + Identity, cast(Identity->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + Identity, cast(Identity->getType()) + ->getNumElements())); // identity is always single source + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( + Identity, cast(Identity->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( + Identity, cast(Identity->getType())->getNumElements())); Constant *Select = ConstantVector::get({CU, C1, C5}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Select)); - EXPECT_TRUE(ShuffleVectorInst::isSelectMask(Select)); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Select)); - EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Select)); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Select)); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Select)); - + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( + Select, cast(Select->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isSelectMask( + Select, cast(Select->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask( + Select, cast(Select->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask( + Select, cast(Select->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( + Select, cast(Select->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( + Select, cast(Select->getType())->getNumElements())); + Constant *Reverse = ConstantVector::get({C3, C2, C1, CU}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Reverse)); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Reverse)); - EXPECT_TRUE(ShuffleVectorInst::isReverseMask(Reverse)); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Reverse)); // reverse is always single source - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Reverse)); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Reverse)); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( + Reverse, cast(Reverse->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask( + Reverse, cast(Reverse->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isReverseMask( + Reverse, cast(Reverse->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + Reverse, cast(Reverse->getType()) + ->getNumElements())); // reverse is always single source + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( + Reverse, cast(Reverse->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( + Reverse, cast(Reverse->getType())->getNumElements())); Constant *SingleSource = ConstantVector::get({C2, C2, C0, CU}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(SingleSource)); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask(SingleSource)); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask(SingleSource)); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(SingleSource)); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(SingleSource)); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(SingleSource)); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( + SingleSource, + cast(SingleSource->getType())->getNumElements())); Constant *ZeroEltSplat = ConstantVector::get({C0, C0, CU, C0}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(ZeroEltSplat)); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask(ZeroEltSplat)); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask(ZeroEltSplat)); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ZeroEltSplat)); // 0-splat is always single source - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ZeroEltSplat)); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(ZeroEltSplat)); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( + ZeroEltSplat, + cast(ZeroEltSplat->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask( + ZeroEltSplat, + cast(ZeroEltSplat->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask( + ZeroEltSplat, + cast(ZeroEltSplat->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + ZeroEltSplat, cast(ZeroEltSplat->getType()) + ->getNumElements())); // 0-splat is always single source + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( + ZeroEltSplat, + cast(ZeroEltSplat->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( + ZeroEltSplat, + cast(ZeroEltSplat->getType())->getNumElements())); Constant *Transpose = ConstantVector::get({C0, C4, C2, C6}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Transpose)); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Transpose)); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Transpose)); - EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Transpose)); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Transpose)); - EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(Transpose)); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( + Transpose, + cast(Transpose->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask( + Transpose, + cast(Transpose->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask( + Transpose, + cast(Transpose->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask( + Transpose, + cast(Transpose->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( + Transpose, + cast(Transpose->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isTransposeMask( + Transpose, + cast(Transpose->getType())->getNumElements())); // More tests to make sure the logic is/stays correct... - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({CU, C1, CU, C3}))); - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({C4, CU, C6, CU}))); - - EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({C4, C1, C6, CU}))); - EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({CU, C1, C6, C3}))); - - EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C7, C6, CU, C4}))); - EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C3, CU, C1, CU}))); - - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C7, C5, CU, C7}))); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C3, C0, CU, C3}))); - - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({C4, CU, CU, C4}))); - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({CU, C0, CU, C0}))); - - EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C5, C3, C7}))); - EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}))); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( + ConstantVector::get({CU, C1, CU, C3}), 4)); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( + ConstantVector::get({C4, CU, C6, CU}), 4)); + + EXPECT_TRUE(ShuffleVectorInst::isSelectMask( + ConstantVector::get({C4, C1, C6, CU}), 4)); + EXPECT_TRUE(ShuffleVectorInst::isSelectMask( + ConstantVector::get({CU, C1, C6, C3}), 4)); + + EXPECT_TRUE(ShuffleVectorInst::isReverseMask( + ConstantVector::get({C7, C6, CU, C4}), 4)); + EXPECT_TRUE(ShuffleVectorInst::isReverseMask( + ConstantVector::get({C3, CU, C1, CU}), 4)); + + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + ConstantVector::get({C7, C5, CU, C7}), 4)); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( + ConstantVector::get({C3, C0, CU, C3}), 4)); + + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( + ConstantVector::get({C4, CU, CU, C4}), 4)); + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( + ConstantVector::get({CU, C0, CU, C0}), 4)); + + EXPECT_TRUE(ShuffleVectorInst::isTransposeMask( + ConstantVector::get({C1, C5, C3, C7}), 4)); + EXPECT_TRUE( + ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}), 2)); // Nothing special about the values here - just re-using inputs to reduce code. Constant *V0 = ConstantVector::get({C0, C1, C2, C3}); diff --git a/llvm/unittests/IR/ShuffleVectorInstTest.cpp b/llvm/unittests/IR/ShuffleVectorInstTest.cpp --- a/llvm/unittests/IR/ShuffleVectorInstTest.cpp +++ b/llvm/unittests/IR/ShuffleVectorInstTest.cpp @@ -14,51 +14,84 @@ namespace { TEST(ShuffleVectorInst, isIdentityMask) { - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3})); - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1})); - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3})); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 5)); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 4)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4})); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4})); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 3)); + + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 5)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 6)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 5)); + + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}, 4)); } TEST(ShuffleVectorInst, isSelectMask) { - ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3})); + ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 4)); + + ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3})); + ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}, 4)); } TEST(ShuffleVectorInst, isReverseMask) { - ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0})); - ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0})); + ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 4)); - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1})); + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 5)); + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 5)); + + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}, 4)); } TEST(ShuffleVectorInst, isZeroEltSplatMask) { - ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0})); - ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1})); + ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 4)); + + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 5)); + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1})); + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}, 4)); } TEST(ShuffleVectorInst, isTransposeMask) { - ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6})); - ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7})); + ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 4)); - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8})); + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 3)); + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 5)); + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 5)); + + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}, 4)); } TEST(ShuffleVectorInst, isSpliceMask) { int Index; - ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, Index)); + ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 4, Index)); ASSERT_EQ(0, Index); - ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, Index)); + ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 7, Index)); ASSERT_EQ(1, Index); - ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, Index)); + ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 3, Index)); + ASSERT_FALSE( + ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 6, Index)); + ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 5, Index)); + ASSERT_FALSE( + ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 8, Index)); + + ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, 4, Index)); } TEST(ShuffleVectorInst, isExtractSubvectorMask) {