diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -4093,8 +4093,8 @@ * Get the mask value at position Elt in the mask of a ShuffleVector * instruction. * - * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is undef - * at that position. + * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is + * poison at that position. */ int LLVMGetMaskValue(LLVMValueRef ShuffleVectorInst, unsigned Elt); diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1269,7 +1269,7 @@ APInt DemandedDstElts = APInt::getZero(Shuffle->getShuffleMask().size()); for (auto I : enumerate(Shuffle->getShuffleMask())) { - if (I.value() != UndefMaskElem) + if (I.value() != PoisonMaskElem) DemandedDstElts.setBit(I.index()); } return TargetTTI->getReplicationShuffleCost( diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -2002,7 +2002,7 @@ // ShuffleVectorInst Class //===----------------------------------------------------------------------===// -constexpr int UndefMaskElem = -1; +constexpr int PoisonMaskElem = -1; /// This instruction constructs a fixed permutation of two /// input vectors. @@ -2010,7 +2010,7 @@ /// For each element of the result vector, the shuffle mask selects an element /// from one of the input vectors to copy to the result. Non-negative elements /// in the mask represent an index into the concatenated pair of input vectors. -/// UndefMaskElem (-1) specifies that the result element is undefined. +/// PoisonMaskElem (-1) specifies that the result element is poison. /// /// For scalable vectors, all the elements of the mask must be 0 or -1. This /// requirement may be relaxed in the future. @@ -2068,16 +2068,16 @@ DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); /// Return the shuffle mask value of this instruction for the given element - /// index. Return UndefMaskElem if the element is undef. + /// index. Return PoisonMaskElem if the element is undef. int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; } /// Convert the input shuffle mask operand to a vector of integers. Undefined - /// elements of the mask are returned as UndefMaskElem. + /// elements of the mask are returned as PoisonMaskElem. static void getShuffleMask(const Constant *Mask, SmallVectorImpl &Result); /// Return the mask for this instruction as a vector of integers. Undefined - /// elements of the mask are returned as UndefMaskElem. + /// elements of the mask are returned as PoisonMaskElem. void getShuffleMask(SmallVectorImpl &Result) const { Result.assign(ShuffleMask.begin(), ShuffleMask.end()); } diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -5235,7 +5235,7 @@ ArrayRef Mask, Type *RetTy, const SimplifyQuery &Q, unsigned MaxRecurse) { - if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; })) + if (all_of(Mask, [](int Elem) { return Elem == PoisonMaskElem; })) return UndefValue::get(RetTy); auto *InVecTy = cast(Op0->getType()); diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -6438,7 +6438,7 @@ ArrayRef Mask = isa(Op) ? cast(Op)->getShuffleMask() : cast(Op)->getShuffleMask(); - return is_contained(Mask, UndefMaskElem); + return is_contained(Mask, PoisonMaskElem); } case Instruction::FNeg: case Instruction::PHI: diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -444,13 +444,13 @@ int Idx = I * SzDest + K; if (Idx == Sz) break; - if (Mask[Idx] >= Sz || Mask[Idx] == UndefMaskElem) + if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem) continue; int SrcRegIdx = Mask[Idx] / SzSrc; // Add a cost of PermuteTwoSrc for each new source register permute, // if we have more than one source registers. if (RegMasks[SrcRegIdx].empty()) - RegMasks[SrcRegIdx].assign(SzDest, UndefMaskElem); + RegMasks[SrcRegIdx].assign(SzDest, PoisonMaskElem); RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc; } } @@ -482,8 +482,8 @@ auto &&CombineMasks = [](MutableArrayRef FirstMask, ArrayRef SecondMask) { for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) { - if (SecondMask[Idx] != UndefMaskElem) { - assert(FirstMask[Idx] == UndefMaskElem && + if (SecondMask[Idx] != PoisonMaskElem) { + assert(FirstMask[Idx] == PoisonMaskElem && "Expected undefined mask element."); FirstMask[Idx] = SecondMask[Idx] + VF; } @@ -491,7 +491,7 @@ }; auto &&NormalizeMask = [](MutableArrayRef Mask) { for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { - if (Mask[Idx] != UndefMaskElem) + if (Mask[Idx] != PoisonMaskElem) Mask[Idx] = Idx; } }; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2383,7 +2383,7 @@ EVT EltVT = NewVT.getVectorElementType(); SmallVector Ops(NewElts, DAG.getUNDEF(EltVT)); for (unsigned I = 0; I < NewElts; ++I) { - if (Mask[I] == UndefMaskElem) + if (Mask[I] == PoisonMaskElem) continue; unsigned Idx = Mask[I]; if (Idx >= NewElts) @@ -2423,11 +2423,11 @@ // Use shuffles operands instead of shuffles themselves. // 1. Adjust mask. for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (Inputs[SrcRegIdx].isUndef()) { - Idx = UndefMaskElem; + Idx = PoisonMaskElem; continue; } auto *Shuffle = @@ -2435,8 +2435,8 @@ if (!Shuffle || !is_contained(P.second, SrcRegIdx)) continue; int MaskElt = Shuffle->getMaskElt(Idx % NewElts); - if (MaskElt == UndefMaskElem) { - Idx = UndefMaskElem; + if (MaskElt == PoisonMaskElem) { + Idx = PoisonMaskElem; continue; } Idx = MaskElt % NewElts + @@ -2455,11 +2455,11 @@ // Check if any concat_vectors can be simplified. SmallBitVector UsedSubVector(2 * std::size(Inputs)); for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (Inputs[SrcRegIdx].isUndef()) { - Idx = UndefMaskElem; + Idx = PoisonMaskElem; continue; } TargetLowering::LegalizeTypeAction TypeAction = @@ -2489,7 +2489,7 @@ if (!Pairs.empty() && Pairs.front().size() > 1) { // Adjust mask. for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; auto *It = find_if( @@ -2531,14 +2531,14 @@ !Shuffle->getOperand(1).isUndef()) { // Find the only used operand, if possible. for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (SrcRegIdx != I) continue; int MaskElt = Shuffle->getMaskElt(Idx % NewElts); - if (MaskElt == UndefMaskElem) { - Idx = UndefMaskElem; + if (MaskElt == PoisonMaskElem) { + Idx = PoisonMaskElem; continue; } int OpIdx = MaskElt / NewElts; @@ -2564,14 +2564,14 @@ // Found that operand is used already. // 1. Fix the mask for the reused operand. for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (SrcRegIdx != I) continue; int MaskElt = Shuffle->getMaskElt(Idx % NewElts); - if (MaskElt == UndefMaskElem) { - Idx = UndefMaskElem; + if (MaskElt == PoisonMaskElem) { + Idx = PoisonMaskElem; continue; } int MaskIdx = MaskElt / NewElts; @@ -2588,7 +2588,7 @@ Inputs[I] = Shuffle->getOperand(Op); // Adjust mask. for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (SrcRegIdx != I) @@ -2622,11 +2622,11 @@ auto &&UniqueConstantVec = UniqueConstantInputs.takeVector(); unsigned ConstNum = UniqueConstantVec.size(); for (int &Idx : Mask) { - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; unsigned SrcRegIdx = Idx / NewElts; if (Inputs[SrcRegIdx].isUndef()) { - Idx = UndefMaskElem; + Idx = PoisonMaskElem; continue; } const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]); @@ -2655,7 +2655,7 @@ // Build a shuffle mask for the output, discovering on the fly which // input vectors to use as shuffle operands. unsigned FirstMaskIdx = High * NewElts; - SmallVector Mask(NewElts * std::size(Inputs), UndefMaskElem); + SmallVector Mask(NewElts * std::size(Inputs), PoisonMaskElem); copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin()); assert(!Output && "Expected default initialized initial value."); TryPeekThroughShufflesInputs(Mask); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -425,7 +425,7 @@ bool FirstElt = true; if (all_of(Mask, [](int Elt) { return Elt == 0; })) { Out << "zeroinitializer"; - } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + } else if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) { Out << "poison"; } else { Out << "<"; @@ -435,7 +435,7 @@ else Out << ", "; Out << "i32 "; - if (Elt == UndefMaskElem) + if (Elt == PoisonMaskElem) Out << "poison"; else Out << Elt; diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -711,7 +711,7 @@ Type *EltTy = V1VTy->getElementType(); // Undefined shuffle mask -> undefined value. - if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) { return UndefValue::get(VectorType::get(EltTy, MaskEltCount)); } diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -3949,7 +3949,7 @@ return I->getMaskValue(Elt); } -int LLVMGetUndefMaskElem(void) { return UndefMaskElem; } +int LLVMGetUndefMaskElem(void) { return PoisonMaskElem; } LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) { Value *P = unwrap(AtomicInst); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -2164,8 +2164,8 @@ SmallVector NewMask(NumMaskElts); for (int i = 0; i != NumMaskElts; ++i) { int MaskElt = getMaskValue(i); - if (MaskElt == UndefMaskElem) { - NewMask[i] = UndefMaskElem; + if (MaskElt == PoisonMaskElem) { + NewMask[i] = PoisonMaskElem; continue; } assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask"); @@ -2186,11 +2186,11 @@ int V1Size = cast(V1->getType())->getElementCount().getKnownMinValue(); for (int Elem : Mask) - if (Elem != UndefMaskElem && Elem >= V1Size * 2) + if (Elem != PoisonMaskElem && Elem >= V1Size * 2) return false; if (isa(V1->getType())) - if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !all_equal(Mask)) + if ((Mask[0] != 0 && Mask[0] != PoisonMaskElem) || !all_equal(Mask)) return false; return true; @@ -2289,7 +2289,7 @@ } SmallVector MaskConst; for (int Elem : Mask) { - if (Elem == UndefMaskElem) + if (Elem == PoisonMaskElem) MaskConst.push_back(UndefValue::get(Int32Ty)); else MaskConst.push_back(ConstantInt::get(Int32Ty, Elem)); @@ -2627,7 +2627,7 @@ "Run out of mask?"); Mask = Mask.drop_front(ReplicationFactor); if (!all_of(CurrSubMask, [CurrElt](int MaskElt) { - return MaskElt == UndefMaskElem || MaskElt == CurrElt; + return MaskElt == PoisonMaskElem || MaskElt == CurrElt; })) return false; } @@ -2639,7 +2639,7 @@ bool ShuffleVectorInst::isReplicationMask(ArrayRef Mask, int &ReplicationFactor, int &VF) { // undef-less case is trivial. - if (!llvm::is_contained(Mask, UndefMaskElem)) { + if (!llvm::is_contained(Mask, PoisonMaskElem)) { ReplicationFactor = Mask.take_while([](int MaskElt) { return MaskElt == 0; }).size(); if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0) @@ -2657,7 +2657,7 @@ // Before doing that, let's perform basic correctness checking first. int Largest = -1; for (int MaskElt : Mask) { - if (MaskElt == UndefMaskElem) + if (MaskElt == PoisonMaskElem) continue; // Elements must be in non-decreasing order. if (MaskElt < Largest) @@ -2703,11 +2703,11 @@ return false; for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) { ArrayRef SubMask = Mask.slice(K, VF); - if (all_of(SubMask, [](int Idx) { return Idx == UndefMaskElem; })) + if (all_of(SubMask, [](int Idx) { return Idx == PoisonMaskElem; })) continue; SmallBitVector Used(VF, false); for_each(SubMask, [&Used, VF](int Idx) { - if (Idx != UndefMaskElem && Idx < VF) + if (Idx != PoisonMaskElem && Idx < VF) Used.set(Idx); }); if (!Used.all()) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -14744,9 +14744,9 @@ auto Mask = SVI->getShuffleMask(); // Sanity check if all the indices are NOT in range. - // If mask is `undef` or `poison`, `Mask` may be a vector of -1s. - // If all of them are `undef`, OOB read will happen later. - if (llvm::all_of(Mask, [](int Idx) { return Idx == UndefMaskElem; })) { + // If mask is `poison`, `Mask` may be a vector of -1s. + // If all of them are `poison`, OOB read will happen later. + if (llvm::all_of(Mask, [](int Idx) { return Idx == PoisonMaskElem; })) { return false; } // A 64bit st2 which does not start at element 0 will involved adding extra diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -3265,9 +3265,9 @@ unsigned NumSources = 0; for (unsigned E = 0; E < LTNumElts; E++) { int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E] - : UndefMaskElem; + : PoisonMaskElem; if (MaskElt < 0) { - NMask.push_back(UndefMaskElem); + NMask.push_back(PoisonMaskElem); continue; } diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1612,7 +1612,7 @@ LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E); unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements(); unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements(); - SmallVector NormalizedMask(NormalizedVF, UndefMaskElem); + SmallVector NormalizedMask(NormalizedVF, PoisonMaskElem); copy(Mask, NormalizedMask.begin()); unsigned PrevSrcReg = 0; ArrayRef PrevRegMask; @@ -1634,7 +1634,7 @@ return; } if (SrcReg != DestReg && - any_of(RegMask, [](int I) { return I != UndefMaskElem; })) { + any_of(RegMask, [](int I) { return I != PoisonMaskElem; })) { // Just a copy of the source register. Cost += TTI::TCC_Basic; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2728,7 +2728,7 @@ for (i = 0; i != SubVecNumElts; ++i) WidenMask.push_back(i); for (; i != VecNumElts; ++i) - WidenMask.push_back(UndefMaskElem); + WidenMask.push_back(PoisonMaskElem); Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask); @@ -3023,7 +3023,7 @@ int Sz = Mask.size(); SmallBitVector UsedIndices(Sz); for (int Idx : Mask) { - if (Idx == UndefMaskElem || UsedIndices.test(Idx)) + if (Idx == PoisonMaskElem || UsedIndices.test(Idx)) break; UsedIndices.set(Idx); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2496,7 +2496,7 @@ // in the case of a shuffle with no undefined mask elements. ArrayRef Mask; if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) && - !is_contained(Mask, UndefMaskElem) && + !is_contained(Mask, PoisonMaskElem) && cast(TVal)->isSelect()) { if (X == FVal) { // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X) @@ -2510,7 +2510,7 @@ } } if (match(FVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) && - !is_contained(Mask, UndefMaskElem) && + !is_contained(Mask, PoisonMaskElem) && cast(FVal)->isSelect()) { if (X == TVal) { // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1553,7 +1553,7 @@ SmallVector Elts; for (unsigned i = 0; i < VWidth; ++i) { if (UndefElts[i]) - Elts.push_back(UndefMaskElem); + Elts.push_back(PoisonMaskElem); else Elts.push_back(Shuffle->getMaskValue(i)); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -1344,7 +1344,7 @@ // (demanded elements analysis may unset it later). return nullptr; } else { - assert(OldMask[i] == UndefMaskElem && + assert(OldMask[i] == PoisonMaskElem && "Unexpected shuffle mask element for identity shuffle"); NewMask[i] = IdxC; } @@ -2145,7 +2145,7 @@ ConstantExpr::getShuffleVector(IdC, C, Mask); bool MightCreatePoisonOrUB = - is_contained(Mask, UndefMaskElem) && + is_contained(Mask, PoisonMaskElem) && (Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode)); if (MightCreatePoisonOrUB) NewC = InstCombiner::getSafeVectorConstantForBinop(BOpcode, NewC, true); @@ -2159,7 +2159,7 @@ // An undef shuffle mask element may propagate as an undef constant element in // the new binop. That would produce poison where the original code might not. // If we already made a safe constant, then there's no danger. - if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB) + if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB) NewBO->dropPoisonGeneratingFlags(); return NewBO; } @@ -2194,7 +2194,7 @@ cast(Shuf.getType())->getNumElements(); SmallVector NewMask(NumMaskElts, 0); for (unsigned i = 0; i != NumMaskElts; ++i) - if (Mask[i] == UndefMaskElem) + if (Mask[i] == PoisonMaskElem) NewMask[i] = Mask[i]; return new ShuffleVectorInst(NewIns, NewMask); @@ -2279,7 +2279,7 @@ // mask element, the result is undefined, but it is not poison or undefined // behavior. That is not necessarily true for div/rem/shift. bool MightCreatePoisonOrUB = - is_contained(Mask, UndefMaskElem) && + is_contained(Mask, PoisonMaskElem) && (Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc)); if (MightCreatePoisonOrUB) NewC = InstCombiner::getSafeVectorConstantForBinop(BOpc, NewC, @@ -2330,7 +2330,7 @@ NewI->andIRFlags(B1); if (DropNSW) NewI->setHasNoSignedWrap(false); - if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB) + if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB) NewI->dropPoisonGeneratingFlags(); } return replaceInstUsesWith(Shuf, NewBO); @@ -2366,7 +2366,7 @@ SrcType->getScalarSizeInBits() / DestType->getScalarSizeInBits(); ArrayRef Mask = Shuf.getShuffleMask(); for (unsigned i = 0, e = Mask.size(); i != e; ++i) { - if (Mask[i] == UndefMaskElem) + if (Mask[i] == PoisonMaskElem) continue; uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio; assert(LSBIndex <= INT32_MAX && "Overflowed 32-bits"); @@ -2552,7 +2552,7 @@ for (unsigned i = 0; i != NumElts; ++i) { int ExtractMaskElt = Shuf.getMaskValue(i); int MaskElt = Mask[i]; - NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt; + NewMask[i] = ExtractMaskElt == PoisonMaskElem ? ExtractMaskElt : MaskElt; } return new ShuffleVectorInst(X, Y, NewMask); } diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1707,9 +1707,9 @@ // TODO: Allow arbitrary shuffles by shuffling after binop? // That might be legal, but we have to deal with poison. if (LShuf->isSelect() && - !is_contained(LShuf->getShuffleMask(), UndefMaskElem) && + !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) && RShuf->isSelect() && - !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) { + !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) { // Example: // LHS = shuffle V1, V2, <0, 5, 6, 3> // RHS = shuffle V2, V1, <0, 5, 6, 3> diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp --- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp +++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp @@ -658,7 +658,7 @@ if (isConstantIntVector(Mask)) { unsigned MemIndex = 0; VResult = PoisonValue::get(VecType); - SmallVector ShuffleMask(VectorWidth, UndefMaskElem); + SmallVector ShuffleMask(VectorWidth, PoisonMaskElem); for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { Value *InsertElt; if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) { diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -349,7 +349,7 @@ UseMask MaskArg) { SmallBitVector UseMask(VF, true); for (auto [Idx, Value] : enumerate(Mask)) { - if (Value == UndefMaskElem) { + if (Value == PoisonMaskElem) { if (MaskArg == UseMask::UndefsAsMask) UseMask.reset(Idx); continue; @@ -467,7 +467,7 @@ Value *Vec2 = nullptr; enum ShuffleMode { Unknown, Select, Permute }; ShuffleMode CommonShuffleMode = Unknown; - Mask.assign(VL.size(), UndefMaskElem); + Mask.assign(VL.size(), PoisonMaskElem); for (unsigned I = 0, E = VL.size(); I < E; ++I) { // Undef can be represented as an undef element in a vector. if (isa(VL[I])) @@ -644,7 +644,7 @@ !isa(EI->getIndexOperand()) || is_contained(UndefVectorExtracts, I)) continue; - if (Mask[I] == UndefMaskElem && !isa(GatheredExtracts[I])) + if (Mask[I] == PoisonMaskElem && !isa(GatheredExtracts[I])) std::swap(VL[I], GatheredExtracts[I]); } return Res; @@ -958,12 +958,12 @@ Mask.append(SubMask.begin(), SubMask.end()); return; } - SmallVector NewMask(SubMask.size(), UndefMaskElem); + SmallVector NewMask(SubMask.size(), PoisonMaskElem); int TermValue = std::min(Mask.size(), SubMask.size()); for (int I = 0, E = SubMask.size(); I < E; ++I) { if ((!ExtendingManyInputs && (SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue)) || - SubMask[I] == UndefMaskElem) + SubMask[I] == PoisonMaskElem) continue; NewMask[I] = Mask[SubMask[I]]; } @@ -1010,7 +1010,7 @@ SmallVectorImpl &Mask) { Mask.clear(); const unsigned E = Indices.size(); - Mask.resize(E, UndefMaskElem); + Mask.resize(E, PoisonMaskElem); for (unsigned I = 0; I < E; ++I) Mask[Indices[I]] = I; } @@ -1023,7 +1023,7 @@ UndefValue::get(Scalars.front()->getType())); Prev.swap(Scalars); for (unsigned I = 0, E = Prev.size(); I < E; ++I) - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) Scalars[Mask[I]] = Prev[I]; } @@ -2538,8 +2538,8 @@ std::equal(VL.begin(), VL.end(), Mask.begin(), [Scalars](Value *V, int Idx) { return (isa(V) && - Idx == UndefMaskElem) || - (Idx != UndefMaskElem && V == Scalars[Idx]); + Idx == PoisonMaskElem) || + (Idx != PoisonMaskElem && V == Scalars[Idx]); }); }; if (!ReorderIndices.empty()) { @@ -3719,7 +3719,7 @@ SmallVector Prev(Reuses.begin(), Reuses.end()); Prev.swap(Reuses); for (unsigned I = 0, E = Prev.size(); I < E; ++I) - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) Reuses[Mask[I]] = Prev[I]; } @@ -3743,7 +3743,7 @@ } Order.assign(Mask.size(), Mask.size()); for (unsigned I = 0, E = Mask.size(); I < E; ++I) - if (MaskOrder[I] != UndefMaskElem) + if (MaskOrder[I] != PoisonMaskElem) Order[MaskOrder[I]] = I; fixupOrderingIndices(Order); } @@ -4078,14 +4078,14 @@ std::optional Idx = getExtractIndex(cast(V)); return Idx && *Idx < Sz; })) { - SmallVector ReorderMask(Sz, UndefMaskElem); + SmallVector ReorderMask(Sz, PoisonMaskElem); if (TE.ReorderIndices.empty()) std::iota(ReorderMask.begin(), ReorderMask.end(), 0); else inversePermutation(TE.ReorderIndices, ReorderMask); for (unsigned I = 0; I < VF; ++I) { int &Idx = ReusedMask[I]; - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; Value *V = TE.Scalars[ReorderMask[Idx]]; std::optional EI = getExtractIndex(cast(V)); @@ -4100,7 +4100,7 @@ for (unsigned K = 0; K < VF; K += Sz) { OrdersType CurrentOrder(TE.ReorderIndices); SmallVector SubMask{ArrayRef(ReusedMask).slice(K, Sz)}; - if (SubMask.front() == UndefMaskElem) + if (SubMask.front() == PoisonMaskElem) std::iota(SubMask.begin(), SubMask.end(), 0); reorderOrder(CurrentOrder, SubMask); transform(CurrentOrder, It, [K](unsigned Pos) { return Pos + K; }); @@ -4437,7 +4437,7 @@ unsigned E = Order.size(); OrdersType CurrentOrder(E, E); transform(Mask, CurrentOrder.begin(), [E](int Idx) { - return Idx == UndefMaskElem ? E : static_cast(Idx); + return Idx == PoisonMaskElem ? E : static_cast(Idx); }); fixupOrderingIndices(CurrentOrder); ++OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second; @@ -4462,10 +4462,10 @@ continue; SmallVector Mask; inversePermutation(BestOrder, Mask); - SmallVector MaskOrder(BestOrder.size(), UndefMaskElem); + SmallVector MaskOrder(BestOrder.size(), PoisonMaskElem); unsigned E = BestOrder.size(); transform(BestOrder, MaskOrder.begin(), [E](unsigned I) { - return I < E ? static_cast(I) : UndefMaskElem; + return I < E ? static_cast(I) : PoisonMaskElem; }); // Do an actual reordering, if profitable. for (std::unique_ptr &TE : VectorizableTree) { @@ -4676,7 +4676,7 @@ unsigned E = Order.size(); OrdersType CurrentOrder(E, E); transform(Mask, CurrentOrder.begin(), [E](int Idx) { - return Idx == UndefMaskElem ? E : static_cast(Idx); + return Idx == PoisonMaskElem ? E : static_cast(Idx); }); fixupOrderingIndices(CurrentOrder); OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second += @@ -4755,10 +4755,10 @@ VisitedOps.clear(); SmallVector Mask; inversePermutation(BestOrder, Mask); - SmallVector MaskOrder(BestOrder.size(), UndefMaskElem); + SmallVector MaskOrder(BestOrder.size(), PoisonMaskElem); unsigned E = BestOrder.size(); transform(BestOrder, MaskOrder.begin(), [E](unsigned I) { - return I < E ? static_cast(I) : UndefMaskElem; + return I < E ? static_cast(I) : PoisonMaskElem; }); for (const std::pair &Op : Data.second) { TreeEntry *TE = Op.second; @@ -5167,7 +5167,7 @@ for (Value *V : VL) { if (isConstant(V)) { ReuseShuffleIndicies.emplace_back( - isa(V) ? UndefMaskElem : UniqueValues.size()); + isa(V) ? PoisonMaskElem : UniqueValues.size()); UniqueValues.emplace_back(V); continue; } @@ -6298,7 +6298,7 @@ SmallVectorImpl *OpScalars = nullptr, SmallVectorImpl *AltScalars = nullptr) { unsigned Sz = VL.size(); - Mask.assign(Sz, UndefMaskElem); + Mask.assign(Sz, PoisonMaskElem); SmallVector OrderMask; if (!ReorderIndices.empty()) inversePermutation(ReorderIndices, OrderMask); @@ -6318,9 +6318,9 @@ } } if (!ReusesIndices.empty()) { - SmallVector NewMask(ReusesIndices.size(), UndefMaskElem); + SmallVector NewMask(ReusesIndices.size(), PoisonMaskElem); transform(ReusesIndices, NewMask.begin(), [&Mask](int Idx) { - return Idx != UndefMaskElem ? Mask[Idx] : UndefMaskElem; + return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem; }); Mask.swap(NewMask); } @@ -6440,13 +6440,13 @@ static void combineMasks(unsigned LocalVF, SmallVectorImpl &Mask, ArrayRef ExtMask) { unsigned VF = Mask.size(); - SmallVector NewMask(ExtMask.size(), UndefMaskElem); + SmallVector NewMask(ExtMask.size(), PoisonMaskElem); for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) { - if (ExtMask[I] == UndefMaskElem) + if (ExtMask[I] == PoisonMaskElem) continue; int MaskedIdx = Mask[ExtMask[I] % VF]; NewMask[I] = - MaskedIdx == UndefMaskElem ? UndefMaskElem : MaskedIdx % LocalVF; + MaskedIdx == PoisonMaskElem ? PoisonMaskElem : MaskedIdx % LocalVF; } Mask.swap(NewMask); } @@ -6533,9 +6533,9 @@ if (auto *SVOpTy = dyn_cast(SV->getOperand(0)->getType())) LocalVF = SVOpTy->getNumElements(); - SmallVector ExtMask(Mask.size(), UndefMaskElem); + SmallVector ExtMask(Mask.size(), PoisonMaskElem); for (auto [Idx, I] : enumerate(Mask)) { - if (I == UndefMaskElem || + if (I == PoisonMaskElem || static_cast(I) >= SV->getShuffleMask().size()) continue; ExtMask[Idx] = SV->getMaskValue(I); @@ -6551,11 +6551,11 @@ if (!IsOp1Undef && !IsOp2Undef) { // Update mask and mark undef elems. for (int &I : Mask) { - if (I == UndefMaskElem) + if (I == PoisonMaskElem) continue; if (SV->getMaskValue(I % SV->getShuffleMask().size()) == - UndefMaskElem) - I = UndefMaskElem; + PoisonMaskElem) + I = PoisonMaskElem; } break; } @@ -6577,8 +6577,8 @@ "Expected masks of same sizes."); // Clear known poison elements. for (auto [I, Idx] : enumerate(Mask)) - if (Idx == UndefMaskElem) - IdentityMask[I] = UndefMaskElem; + if (Idx == PoisonMaskElem) + IdentityMask[I] = PoisonMaskElem; Mask.swap(IdentityMask); auto *Shuffle = dyn_cast(V); return SinglePermute && @@ -6614,8 +6614,8 @@ Value *Op2 = V2; int VF = cast(V1->getType())->getElementCount().getKnownMinValue(); - SmallVector CombinedMask1(Mask.size(), UndefMaskElem); - SmallVector CombinedMask2(Mask.size(), UndefMaskElem); + SmallVector CombinedMask1(Mask.size(), PoisonMaskElem); + SmallVector CombinedMask2(Mask.size(), PoisonMaskElem); for (int I = 0, E = Mask.size(); I < E; ++I) { if (Mask[I] < VF) CombinedMask1[I] = Mask[I]; @@ -6633,9 +6633,9 @@ // again. if (auto *SV1 = dyn_cast(Op1)) if (auto *SV2 = dyn_cast(Op2)) { - SmallVector ExtMask1(Mask.size(), UndefMaskElem); + SmallVector ExtMask1(Mask.size(), PoisonMaskElem); for (auto [Idx, I] : enumerate(CombinedMask1)) { - if (I == UndefMaskElem) + if (I == PoisonMaskElem) continue; ExtMask1[Idx] = SV1->getMaskValue(I); } @@ -6643,9 +6643,9 @@ cast(SV1->getOperand(1)->getType()) ->getNumElements(), ExtMask1, UseMask::SecondArg); - SmallVector ExtMask2(CombinedMask2.size(), UndefMaskElem); + SmallVector ExtMask2(CombinedMask2.size(), PoisonMaskElem); for (auto [Idx, I] : enumerate(CombinedMask2)) { - if (I == UndefMaskElem) + if (I == PoisonMaskElem) continue; ExtMask2[Idx] = SV2->getMaskValue(I); } @@ -6685,8 +6685,8 @@ ->getElementCount() .getKnownMinValue()); for (int I = 0, E = Mask.size(); I < E; ++I) { - if (CombinedMask2[I] != UndefMaskElem) { - assert(CombinedMask1[I] == UndefMaskElem && + if (CombinedMask2[I] != PoisonMaskElem) { + assert(CombinedMask1[I] == PoisonMaskElem && "Expected undefined mask element"); CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF); } @@ -6880,25 +6880,25 @@ // Process extracts in blocks of EltsPerVector to check if the source vector // operand can be re-used directly. If not, add the cost of creating a // shuffle to extract the values into a vector register. - SmallVector RegMask(EltsPerVector, UndefMaskElem); + SmallVector RegMask(EltsPerVector, PoisonMaskElem); for (auto *V : VL) { ++Idx; // Reached the start of a new vector registers. if (Idx % EltsPerVector == 0) { - RegMask.assign(EltsPerVector, UndefMaskElem); + RegMask.assign(EltsPerVector, PoisonMaskElem); AllConsecutive = true; continue; } // Need to exclude undefs from analysis. - if (isa(V) || Mask[Idx] == UndefMaskElem) + if (isa(V) || Mask[Idx] == PoisonMaskElem) continue; // Check all extracts for a vector register on the target directly // extract values in order. unsigned CurrentIdx = *getExtractIndex(cast(V)); - if (!isa(VL[Idx - 1]) && Mask[Idx - 1] != UndefMaskElem) { + if (!isa(VL[Idx - 1]) && Mask[Idx - 1] != PoisonMaskElem) { unsigned PrevIdx = *getExtractIndex(cast(VL[Idx - 1])); AllConsecutive &= PrevIdx + 1 == CurrentIdx && CurrentIdx % EltsPerVector == Idx % EltsPerVector; @@ -6945,7 +6945,7 @@ DenseMap ExtractVectorsTys; for (auto [I, V] : enumerate(VL)) { // Ignore non-extractelement scalars. - if (isa(V) || (!Mask.empty() && Mask[I] == UndefMaskElem)) + if (isa(V) || (!Mask.empty() && Mask[I] == PoisonMaskElem)) continue; // If all users of instruction are going to be vectorized and this // instruction itself is not going to be vectorized, consider this @@ -7167,7 +7167,7 @@ } // Remove shuffled elements from list of gathers. for (int I = 0, Sz = Mask.size(); I < Sz; ++I) { - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) GatheredScalars[I] = PoisonValue::get(ScalarTy); } LLVM_DEBUG(dbgs() << "SLP: shuffled " << Entries.size() @@ -7398,7 +7398,7 @@ unsigned NumOfParts = TTI->getNumberOfParts(SrcVecTy); - SmallVector InsertMask(NumElts, UndefMaskElem); + SmallVector InsertMask(NumElts, PoisonMaskElem); unsigned OffsetBeg = *getInsertIndex(VL.front()); unsigned OffsetEnd = OffsetBeg; InsertMask[OffsetBeg] = 0; @@ -7435,13 +7435,13 @@ SmallVector Mask; if (!E->ReorderIndices.empty()) { inversePermutation(E->ReorderIndices, Mask); - Mask.append(InsertVecSz - Mask.size(), UndefMaskElem); + Mask.append(InsertVecSz - Mask.size(), PoisonMaskElem); } else { - Mask.assign(VecSz, UndefMaskElem); + Mask.assign(VecSz, PoisonMaskElem); std::iota(Mask.begin(), std::next(Mask.begin(), InsertVecSz), 0); } bool IsIdentity = true; - SmallVector PrevMask(InsertVecSz, UndefMaskElem); + SmallVector PrevMask(InsertVecSz, PoisonMaskElem); Mask.swap(PrevMask); for (unsigned I = 0; I < NumScalars; ++I) { unsigned InsertIdx = *getInsertIndex(VL[PrevMask[I]]); @@ -7484,14 +7484,14 @@ InsertVecTy); } else { for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I) - Mask[I] = InMask.test(I) ? UndefMaskElem : I; + Mask[I] = InMask.test(I) ? PoisonMaskElem : I; for (unsigned I = OffsetBeg - Offset, End = OffsetEnd - Offset; I <= End; ++I) - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) Mask[I] = I + VecSz; for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I) Mask[I] = - ((I >= InMask.size()) || InMask.test(I)) ? UndefMaskElem : I; + ((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I; Cost += TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, InsertVecTy, Mask); } } @@ -8133,8 +8133,8 @@ ResizeAction(ShuffleMask.begin()->first, Mask, /*ForSingleMask=*/false); SmallBitVector IsBasePoison = isUndefVector(Base, UseMask); for (unsigned Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { - if (Mask[Idx] == UndefMaskElem) - Mask[Idx] = IsBasePoison.test(Idx) ? UndefMaskElem : Idx; + if (Mask[Idx] == PoisonMaskElem) + Mask[Idx] = IsBasePoison.test(Idx) ? PoisonMaskElem : Idx; else Mask[Idx] = (Res.second ? Idx : Mask[Idx]) + VF; } @@ -8163,8 +8163,8 @@ // can shuffle them directly. ArrayRef SecMask = VMIt->second; for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) { - if (SecMask[I] != UndefMaskElem) { - assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars."); + if (SecMask[I] != PoisonMaskElem) { + assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars."); Mask[I] = SecMask[I] + Vec1VF; } } @@ -8177,12 +8177,12 @@ ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false); ArrayRef SecMask = VMIt->second; for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) { - if (Mask[I] != UndefMaskElem) { - assert(SecMask[I] == UndefMaskElem && "Multiple uses of scalars."); + if (Mask[I] != PoisonMaskElem) { + assert(SecMask[I] == PoisonMaskElem && "Multiple uses of scalars."); if (Res1.second) Mask[I] = I; - } else if (SecMask[I] != UndefMaskElem) { - assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars."); + } else if (SecMask[I] != PoisonMaskElem) { + assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars."); Mask[I] = (Res2.second ? I : SecMask[I]) + VF; } } @@ -8199,11 +8199,11 @@ ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false); ArrayRef SecMask = VMIt->second; for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) { - if (SecMask[I] != UndefMaskElem) { - assert((Mask[I] == UndefMaskElem || IsBaseNotUndef) && + if (SecMask[I] != PoisonMaskElem) { + assert((Mask[I] == PoisonMaskElem || IsBaseNotUndef) && "Multiple uses of scalars."); Mask[I] = (Res.second ? I : SecMask[I]) + VF; - } else if (Mask[I] != UndefMaskElem) { + } else if (Mask[I] != PoisonMaskElem) { Mask[I] = I; } } @@ -8298,7 +8298,7 @@ (void)ShuffleMasks.emplace_back(); SmallVectorImpl &Mask = ShuffleMasks.back()[ScalarTE]; if (Mask.empty()) - Mask.assign(FTy->getNumElements(), UndefMaskElem); + Mask.assign(FTy->getNumElements(), PoisonMaskElem); // Find the insertvector, vectorized in tree, if any. Value *Base = VU; while (auto *IEBase = dyn_cast(Base)) { @@ -8312,7 +8312,7 @@ do { IEBase = cast(Base); int Idx = *getInsertIndex(IEBase); - assert(Mask[Idx] == UndefMaskElem && + assert(Mask[Idx] == PoisonMaskElem && "InsertElementInstruction used already."); Mask[Idx] = Idx; Base = IEBase->getOperand(0); @@ -8332,7 +8332,7 @@ int InIdx = *InsertIdx; SmallVectorImpl &Mask = ShuffleMasks[VecId][ScalarTE]; if (Mask.empty()) - Mask.assign(FTy->getNumElements(), UndefMaskElem); + Mask.assign(FTy->getNumElements(), PoisonMaskElem); Mask[InIdx] = EU.Lane; DemandedElts[VecId].setBit(InIdx); continue; @@ -8371,7 +8371,7 @@ (all_of(Mask, [VF](int Idx) { return Idx < 2 * static_cast(VF); }) && !ShuffleVectorInst::isIdentityMask(Mask)))) { - SmallVector OrigMask(VecVF, UndefMaskElem); + SmallVector OrigMask(VecVF, PoisonMaskElem); std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)), OrigMask.begin()); C = TTI->getShuffleCost( @@ -8457,7 +8457,7 @@ // No need to check for the topmost gather node. if (TE == VectorizableTree.front().get()) return std::nullopt; - Mask.assign(VL.size(), UndefMaskElem); + Mask.assign(VL.size(), PoisonMaskElem); assert(TE->UserTreeIndices.size() == 1 && "Expected only single user of the gather node."); // TODO: currently checking only for Scalars in the tree entry, need to count @@ -8620,7 +8620,7 @@ // Clear undef scalars. for (int I = 0, Sz = VL.size(); I < Sz; ++I) if (isa(VL[I])) - Mask[I] = UndefMaskElem; + Mask[I] = PoisonMaskElem; return TargetTransformInfo::SK_PermuteSingleSrc; } // No perfect match, just shuffle, so choose the first tree node from the @@ -9069,7 +9069,7 @@ if (isa(VL[I])) continue; if (auto *SV = dyn_cast(Root)) { - if (SV->getMaskValue(I) == UndefMaskElem) + if (SV->getMaskValue(I) == PoisonMaskElem) continue; } } @@ -9180,7 +9180,7 @@ int V2VF = cast(V2->getType())->getNumElements(); int VF = std::max(V1VF, V2VF); int MinVF = std::min(V1VF, V2VF); - SmallVector IdentityMask(VF, UndefMaskElem); + SmallVector IdentityMask(VF, PoisonMaskElem); std::iota(IdentityMask.begin(), std::next(IdentityMask.begin(), MinVF), 0); Value *&Op = MinVF == V1VF ? V1 : V2; @@ -9211,7 +9211,7 @@ static void transformMaskAfterShuffle(MutableArrayRef CommonMask, ArrayRef Mask) { for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (Mask[Idx] != UndefMaskElem) + if (Mask[Idx] != PoisonMaskElem) CommonMask[Idx] = Idx; } @@ -9224,7 +9224,7 @@ Value *VecBase = nullptr; for (int I = 0, Sz = Mask.size(); I < Sz; ++I) { int Idx = Mask[I]; - if (Idx == UndefMaskElem) + if (Idx == PoisonMaskElem) continue; auto *EI = cast(E->Scalars[I]); VecBase = EI->getVectorOperand(); @@ -9272,7 +9272,7 @@ } V1 = createShuffle(V1, V2, Mask); for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (Mask[Idx] != UndefMaskElem) + if (Mask[Idx] != PoisonMaskElem) CommonMask[Idx] = Idx + Sz; InVectors.front() = Vec; if (InVectors.size() == 2) @@ -9285,7 +9285,7 @@ if (InVectors.empty()) { if (!isa(V1->getType())) { V1 = createShuffle(V1, nullptr, CommonMask); - CommonMask.assign(Mask.size(), UndefMaskElem); + CommonMask.assign(Mask.size(), PoisonMaskElem); transformMaskAfterShuffle(CommonMask, Mask); } InVectors.push_back(V1); @@ -9307,7 +9307,7 @@ transformMaskAfterShuffle(CommonMask, CommonMask); } for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (CommonMask[Idx] == UndefMaskElem && Mask[Idx] != UndefMaskElem) + if (CommonMask[Idx] == PoisonMaskElem && Mask[Idx] != PoisonMaskElem) CommonMask[Idx] = V->getType() != V1->getType() ? Idx + Sz @@ -9325,7 +9325,7 @@ // Check if second vector is required if the used elements are already // used from the first one. for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem) { + if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) { InVectors.push_back(V1); break; } @@ -9334,7 +9334,7 @@ if (auto *FTy = dyn_cast(V1->getType())) VF = FTy->getNumElements(); for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem) + if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF); } /// Adds another one input vector and the mask for the shuffling. @@ -9359,13 +9359,13 @@ Vec = createShuffle(Vec, nullptr, CommonMask); } for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) - if (CommonMask[Idx] != UndefMaskElem) + if (CommonMask[Idx] != PoisonMaskElem) CommonMask[Idx] = Idx; assert(VF > 0 && "Expected vector length for the final value before action."); unsigned VecVF = cast(Vec->getType())->getNumElements(); if (VecVF < VF) { - SmallVector ResizeMask(VF, UndefMaskElem); + SmallVector ResizeMask(VF, PoisonMaskElem); std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0); Vec = createShuffle(Vec, nullptr, ResizeMask); } @@ -9376,9 +9376,9 @@ if (CommonMask.empty()) { CommonMask.assign(ExtMask.begin(), ExtMask.end()); } else { - SmallVector NewMask(ExtMask.size(), UndefMaskElem); + SmallVector NewMask(ExtMask.size(), PoisonMaskElem); for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) { - if (ExtMask[I] == UndefMaskElem) + if (ExtMask[I] == PoisonMaskElem) continue; NewMask[I] = CommonMask[ExtMask[I]]; } @@ -9449,18 +9449,18 @@ // ... (use %2) // %shuffle = shuffle <2 x> %2, poison, <2 x> {2, 0} // br %block - SmallVector UniqueIdxs(VF, UndefMaskElem); + SmallVector UniqueIdxs(VF, PoisonMaskElem); SmallSet UsedIdxs; int Pos = 0; for (int Idx : VE->ReuseShuffleIndices) { - if (Idx != static_cast(VF) && Idx != UndefMaskElem && + if (Idx != static_cast(VF) && Idx != PoisonMaskElem && UsedIdxs.insert(Idx).second) UniqueIdxs[Idx] = Pos; ++Pos; } assert(VF >= UsedIdxs.size() && "Expected vectorization factor " "less than original vector size."); - UniqueIdxs.append(VF - UsedIdxs.size(), UndefMaskElem); + UniqueIdxs.append(VF - UsedIdxs.size(), PoisonMaskElem); V = FinalShuffle(V, UniqueIdxs); } else { assert(VF < cast(V->getType())->getNumElements() && @@ -9543,7 +9543,7 @@ if (It == VectorizableTree.end()) return false; unsigned I = - *find_if_not(Mask, [](int Idx) { return Idx == UndefMaskElem; }); + *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; }); int Sz = Mask.size(); if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) && ShuffleVectorInst::isIdentityMask(Mask)) @@ -9603,10 +9603,10 @@ // Restore the mask for previous partially matched values. for (auto [I, V] : enumerate(E->Scalars)) { if (isa(V)) { - Mask[I] = UndefMaskElem; + Mask[I] = PoisonMaskElem; continue; } - if (Mask[I] == UndefMaskElem) + if (Mask[I] == PoisonMaskElem) Mask[I] = Entries.front()->findLaneForValue(V); } ShuffleBuilder.add(Entries.front()->VectorizedValue, Mask); @@ -9622,7 +9622,7 @@ } // Remove shuffled elements from list of gathers. for (int I = 0, Sz = Mask.size(); I < Sz; ++I) { - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) GatheredScalars[I] = PoisonValue::get(ScalarTy); } } @@ -9669,7 +9669,7 @@ if (NumNonConsts == 1) { // Restore single insert element. if (IsSplat) { - ReuseMask.assign(VF, UndefMaskElem); + ReuseMask.assign(VF, PoisonMaskElem); std::swap(Scalars.front(), Scalars[SinglePos]); if (!UndefPos.empty() && UndefPos.front() == 0) Scalars.front() = UndefValue::get(ScalarTy); @@ -9708,7 +9708,7 @@ // Replace undefs by the poisons, emit broadcast and then emit // freeze. for_each(UndefPos, [&](int I) { - ReuseMask[I] = UndefMaskElem; + ReuseMask[I] = PoisonMaskElem; if (isa(Scalars[I])) Scalars[I] = PoisonValue::get(ScalarTy); }); @@ -9726,9 +9726,9 @@ // Find input vectors. Value *Vec2 = nullptr; for (unsigned I = 0, Sz = ExtractMask.size(); I < Sz; ++I) { - if (ExtractMask[I] == UndefMaskElem || - (!Mask.empty() && Mask[I] != UndefMaskElem)) { - ExtractMask[I] = UndefMaskElem; + if (ExtractMask[I] == PoisonMaskElem || + (!Mask.empty() && Mask[I] != PoisonMaskElem)) { + ExtractMask[I] = PoisonMaskElem; continue; } if (isa(E->Scalars[I])) @@ -9815,7 +9815,7 @@ } // Generate constants for final shuffle and build a mask for them. if (!all_of(GatheredScalars, PoisonValue::classof)) { - SmallVector BVMask(GatheredScalars.size(), UndefMaskElem); + SmallVector BVMask(GatheredScalars.size(), PoisonMaskElem); TryPackScalars(GatheredScalars, BVMask, /*IsRootPoison=*/true); Value *BV = gather(GatheredScalars); ShuffleBuilder.add(BV, BVMask); @@ -9835,14 +9835,14 @@ }); } else if (!allConstant(GatheredScalars)) { // Gather unique scalars and all constants. - SmallVector ReuseMask(GatheredScalars.size(), UndefMaskElem); + SmallVector ReuseMask(GatheredScalars.size(), PoisonMaskElem); TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true); Vec = gather(GatheredScalars); ShuffleBuilder.add(Vec, ReuseMask); Vec = ShuffleBuilder.finalize(E->ReuseShuffleIndices); } else { // Gather all constants. - SmallVector Mask(E->Scalars.size(), UndefMaskElem); + SmallVector Mask(E->Scalars.size(), PoisonMaskElem); for (auto [I, V] : enumerate(E->Scalars)) { if (!isa(V)) Mask[I] = I; @@ -9986,14 +9986,14 @@ SmallVector Mask; if (!E->ReorderIndices.empty()) { inversePermutation(E->ReorderIndices, Mask); - Mask.append(NumElts - NumScalars, UndefMaskElem); + Mask.append(NumElts - NumScalars, PoisonMaskElem); } else { - Mask.assign(NumElts, UndefMaskElem); + Mask.assign(NumElts, PoisonMaskElem); std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0); } // Create InsertVector shuffle if necessary bool IsIdentity = true; - SmallVector PrevMask(NumElts, UndefMaskElem); + SmallVector PrevMask(NumElts, PoisonMaskElem); Mask.swap(PrevMask); for (unsigned I = 0; I < NumScalars; ++I) { Value *Scalar = E->Scalars[PrevMask[I]]; @@ -10009,9 +10009,9 @@ } } - SmallVector InsertMask(NumElts, UndefMaskElem); + SmallVector InsertMask(NumElts, PoisonMaskElem); for (unsigned I = 0; I < NumElts; I++) { - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) InsertMask[Offset + I] = I; } SmallBitVector UseMask = @@ -10026,7 +10026,7 @@ isUndefVector(FirstInsert->getOperand(0), UseMask); if (!IsFirstPoison.all()) { for (unsigned I = 0; I < NumElts; I++) { - if (InsertMask[I] == UndefMaskElem && !IsFirstPoison.test(I)) + if (InsertMask[I] == PoisonMaskElem && !IsFirstPoison.test(I)) InsertMask[I] = I + NumElts; } } @@ -10044,8 +10044,8 @@ SmallBitVector IsFirstPoison = isUndefVector(FirstInsert->getOperand(0), UseMask); for (unsigned I = 0; I < NumElts; I++) { - if (InsertMask[I] == UndefMaskElem) - InsertMask[I] = IsFirstPoison.test(I) ? UndefMaskElem : I; + if (InsertMask[I] == PoisonMaskElem) + InsertMask[I] = IsFirstPoison.test(I) ? PoisonMaskElem : I; else InsertMask[I] += NumElts; } @@ -10707,7 +10707,7 @@ ShuffledInserts.size() - 1); SmallVectorImpl &Mask = It->ValueMasks[Vec]; if (Mask.empty()) - Mask.assign(FTy->getNumElements(), UndefMaskElem); + Mask.assign(FTy->getNumElements(), PoisonMaskElem); // Find the insertvector, vectorized in tree, if any. Value *Base = VU; while (auto *IEBase = dyn_cast(Base)) { @@ -10720,7 +10720,7 @@ do { IEBase = cast(Base); int IEIdx = *getInsertIndex(IEBase); - assert(Mask[Idx] == UndefMaskElem && + assert(Mask[Idx] == PoisonMaskElem && "InsertElementInstruction used already."); Mask[IEIdx] = IEIdx; Base = IEBase->getOperand(0); @@ -10738,7 +10738,7 @@ } SmallVectorImpl &Mask = It->ValueMasks[Vec]; if (Mask.empty()) - Mask.assign(FTy->getNumElements(), UndefMaskElem); + Mask.assign(FTy->getNumElements(), PoisonMaskElem); Mask[Idx] = ExternalUse.Lane; It->InsertElements.push_back(cast(User)); continue; @@ -10780,8 +10780,8 @@ } auto CreateShuffle = [&](Value *V1, Value *V2, ArrayRef Mask) { - SmallVector CombinedMask1(Mask.size(), UndefMaskElem); - SmallVector CombinedMask2(Mask.size(), UndefMaskElem); + SmallVector CombinedMask1(Mask.size(), PoisonMaskElem); + SmallVector CombinedMask2(Mask.size(), PoisonMaskElem); int VF = cast(V1->getType())->getNumElements(); for (int I = 0, E = Mask.size(); I < E; ++I) { if (Mask[I] < VF) @@ -10806,9 +10806,9 @@ return std::make_pair(Vec, true); } if (!ForSingleMask) { - SmallVector ResizeMask(VF, UndefMaskElem); + SmallVector ResizeMask(VF, PoisonMaskElem); for (unsigned I = 0; I < VF; ++I) { - if (Mask[I] != UndefMaskElem) + if (Mask[I] != PoisonMaskElem) ResizeMask[Mask[I]] = Mask[I]; } Vec = CreateShuffle(Vec, nullptr, ResizeMask); @@ -11011,14 +11011,14 @@ // registers. unsigned LastUndefsCnt = 0; for (int I = 0, E = NewMask.size(); I < E; ++I) { - if (SM1[I] == UndefMaskElem) + if (SM1[I] == PoisonMaskElem) ++LastUndefsCnt; else LastUndefsCnt = 0; - if (NewMask[I] != UndefMaskElem && SM1[I] != UndefMaskElem && + if (NewMask[I] != PoisonMaskElem && SM1[I] != PoisonMaskElem && NewMask[I] != SM1[I]) return false; - if (NewMask[I] == UndefMaskElem) + if (NewMask[I] == PoisonMaskElem) NewMask[I] = SM1[I]; } // Check if the last undefs actually change the final number of used vector @@ -13804,7 +13804,7 @@ // 7>, if elements 4th and 6th elements have even number of repeats. SmallVector Mask( cast(VectorizedValue->getType())->getNumElements(), - UndefMaskElem); + PoisonMaskElem); std::iota(Mask.begin(), Mask.end(), 0); bool NeedShuffle = false; for (unsigned I = 0, VF = VL.size(); I < VF; ++I) { diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -244,7 +244,7 @@ // still need a shuffle to change the vector size. auto *Ty = cast(I.getType()); unsigned OutputNumElts = Ty->getNumElements(); - SmallVector Mask(OutputNumElts, UndefMaskElem); + SmallVector Mask(OutputNumElts, PoisonMaskElem); assert(OffsetEltIndex < MinVecNumElts && "Address offset too big"); Mask[0] = OffsetEltIndex; if (OffsetEltIndex) @@ -480,7 +480,7 @@ // to the new element index. Example for OldIndex == 2 and NewIndex == 0: // ShufMask = { 2, undef, undef, undef } auto *VecTy = cast(Vec->getType()); - SmallVector ShufMask(VecTy->getNumElements(), UndefMaskElem); + SmallVector ShufMask(VecTy->getNumElements(), PoisonMaskElem); ShufMask[NewIndex] = OldIndex; return Builder.CreateShuffleVector(Vec, ShufMask, "shift"); } @@ -914,7 +914,7 @@ auto *CmpTy = cast(CmpInst::makeCmpResultType(X->getType())); InstructionCost NewCost = TTI.getCmpSelInstrCost( CmpOpcode, X->getType(), CmpInst::makeCmpResultType(X->getType()), Pred); - SmallVector ShufMask(VecTy->getNumElements(), UndefMaskElem); + SmallVector ShufMask(VecTy->getNumElements(), PoisonMaskElem); ShufMask[CheapIndex] = ExpensiveIndex; NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy, ShufMask); @@ -1593,12 +1593,12 @@ V2B.push_back(GetBaseMaskValue(SVI1B, V2[I].first)); } while (V1A.size() < NumElts) { - V1A.push_back(UndefMaskElem); - V1B.push_back(UndefMaskElem); + V1A.push_back(PoisonMaskElem); + V1B.push_back(PoisonMaskElem); } while (V2A.size() < NumElts) { - V2A.push_back(UndefMaskElem); - V2B.push_back(UndefMaskElem); + V2A.push_back(PoisonMaskElem); + V2B.push_back(PoisonMaskElem); } auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {