diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h --- a/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -37,6 +37,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/PatternMatch.h" namespace llvm { @@ -133,7 +134,9 @@ bool isUndefValue(Value *V) const { if (!CanUseUndef) return false; - return isa(V); + + using namespace PatternMatch; + return match(V, m_Undef()); } }; diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -88,8 +88,50 @@ /// Matches any compare instruction and ignore it. inline class_match m_Cmp() { return class_match(); } -/// Match an arbitrary undef constant. -inline class_match m_Undef() { return class_match(); } +struct undef_match { + static bool check(const Value *V) { + if (isa(V)) + return true; + + const auto *CA = dyn_cast(V); + if (!CA) + return false; + + SmallPtrSet Seen; + SmallVector Worklist; + + // Returns: is never undef? + auto CheckValue = [&](const ConstantAggregate *CA) { + for (const Value *Op : CA->operand_values()) { + if (isa(Op)) + continue; + + const auto *CA = dyn_cast(Op); + if (!CA) + return true; + if (Seen.insert(CA).second) + Worklist.emplace_back(CA); + } + + return false; + }; + + if (CheckValue(CA)) + return false; + + while (!Worklist.empty()) { + if (CheckValue(Worklist.pop_back_val())) + return false; + } + return true; + } + template bool match(ITy *V) { return check(V); } +}; + +/// Match an arbitrary undef constant. This matches poison as well. +/// If this is an aggregate and contains a non-aggregate element that is +/// neither undef nor poison, the aggregate is not matched. +inline auto m_Undef() { return undef_match(); } /// Match an arbitrary poison constant. inline class_match m_Poison() { return class_match(); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -675,7 +675,7 @@ static Instruction *shrinkSplatShuffle(TruncInst &Trunc, InstCombiner::BuilderTy &Builder) { auto *Shuf = dyn_cast(Trunc.getOperand(0)); - if (Shuf && Shuf->hasOneUse() && isa(Shuf->getOperand(1)) && + if (Shuf && Shuf->hasOneUse() && match(Shuf->getOperand(1), m_Undef()) && is_splat(Shuf->getShuffleMask()) && Shuf->getType() == Shuf->getOperand(0)->getType()) { // trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask @@ -708,7 +708,7 @@ Value *ScalarOp = InsElt->getOperand(1); Value *Index = InsElt->getOperand(2); - if (isa(VecOp)) { + if (match(VecOp, m_Undef())) { // trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index // fptrunc (inselt undef, X, Index) --> inselt undef, (fptrunc X), Index UndefValue *NarrowUndef = UndefValue::get(DestTy); @@ -2698,7 +2698,7 @@ ShufElts.getKnownMinValue() % 2 == 0 && Shuf->hasOneUse() && Shuf->isReverse()) { assert(ShufOp0->getType() == SrcTy && "Unexpected shuffle mask"); - assert(isa(ShufOp1) && "Unexpected shuffle op"); + assert(match(ShufOp1, m_Undef()) && "Unexpected shuffle op"); Function *Bswap = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::bswap, DestTy); Value *ScalarX = Builder.CreateBitCast(ShufOp0, DestTy); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -1065,7 +1065,7 @@ return nullptr; V = IV->getAggregateOperand(); } - if (!isa(V) ||!U) + if (!match(V, m_Undef()) || !U) return nullptr; auto *UT = cast(U->getType()); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2596,7 +2596,7 @@ // don't simplify it so loop unswitch can know the equality comparison // may have an undef operand. This is a workaround for PR31652 caused by // descrepancy about branch on undef between LoopUnswitch and GVN. - if (isa(TrueVal) || isa(FalseVal)) { + if (match(TrueVal, m_Undef()) || match(FalseVal, m_Undef())) { if (llvm::any_of(SI.users(), [&](User *U) { ICmpInst *CI = dyn_cast(U); if (CI && CI->isEquality()) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1056,7 +1056,7 @@ APInt EltMask(APInt::getAllOnesValue(VWidth)); assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!"); - if (isa(V)) { + if (match(V, m_Undef())) { // If the entire vector is undef or poison, just return this info. UndefElts = EltMask; return nullptr; @@ -1157,7 +1157,7 @@ // merge the undef bits here since gepping with either an undef base or // index results in undef. for (unsigned i = 0; i < I->getNumOperands(); i++) { - if (isa(I->getOperand(i))) { + if (match(I->getOperand(i), m_Undef())) { // If the entire vector is undefined, just return this info. UndefElts = EltMask; return nullptr; @@ -1226,7 +1226,7 @@ // operand. if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) && DemandedElts.isAllOnesValue()) { - if (!isa(I->getOperand(1))) { + if (!match(I->getOperand(1), m_Undef())) { I->setOperand(1, UndefValue::get(I->getOperand(1)->getType())); MadeChange = true; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -474,7 +474,7 @@ "Invalid CollectSingleShuffleElements"); unsigned NumElts = cast(V->getType())->getNumElements(); - if (isa(V)) { + if (match(V, m_Undef())) { Mask.assign(NumElts, -1); return true; } @@ -630,7 +630,7 @@ assert(V->getType()->isVectorTy() && "Invalid shuffle!"); unsigned NumElts = cast(V->getType())->getNumElements(); - if (isa(V)) { + if (match(V, m_Undef())) { Mask.assign(NumElts, -1); return std::make_pair( PermittedRHS ? UndefValue::get(PermittedRHS->getType()) : V, nullptr); @@ -1102,7 +1102,7 @@ // insert into every element. // TODO: If the base vector is not undef, it might be better to create a splat // and then a select-shuffle (blend) with the base vector. - if (!isa(FirstIE->getOperand(0))) + if (!match(FirstIE->getOperand(0), m_Undef())) if (!ElementPresent.all()) return nullptr; @@ -1164,7 +1164,7 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) { // Check if the vector operand of this insert is an identity shuffle. auto *Shuf = dyn_cast(InsElt.getOperand(0)); - if (!Shuf || !isa(Shuf->getOperand(1)) || + if (!Shuf || !match(Shuf->getOperand(1), m_Undef()) || !(Shuf->isIdentityWithExtract() || Shuf->isIdentityWithPadding())) return nullptr; @@ -1633,7 +1633,7 @@ assert(V->getType()->isVectorTy() && "can't reorder non-vector elements"); Type *EltTy = V->getType()->getScalarType(); Type *I32Ty = IntegerType::getInt32Ty(V->getContext()); - if (isa(V)) + if (match(V, m_Undef())) return UndefValue::get(FixedVectorType::get(EltTy, Mask.size())); if (isa(V)) @@ -1886,7 +1886,7 @@ // Canonicalize to choose from operand 0 first unless operand 1 is undefined. // Commuting undef to operand 0 conflicts with another canonicalization. unsigned NumElts = cast(Shuf.getType())->getNumElements(); - if (!isa(Shuf.getOperand(1)) && + if (!match(Shuf.getOperand(1), m_Undef()) && Shuf.getMaskValue(0) >= (int)NumElts) { // TODO: Can we assert that both operands of a shuffle-select are not undef // (otherwise, it would have been folded by instsimplify? @@ -2083,7 +2083,7 @@ /// Try to combine 2 shuffles into 1 shuffle by concatenating a shuffle mask. static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) { Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1); - if (!Shuf.isIdentityWithExtract() || !isa(Op1)) + if (!Shuf.isIdentityWithExtract() || !match(Op1, m_Undef())) return nullptr; Value *X, *Y; @@ -2231,10 +2231,10 @@ !isPowerOf2_32( cast(Shuffle0->getType())->getNumElements()) || !isPowerOf2_32(cast(X->getType())->getNumElements()) || - isa(X) || isa(Y)) + match(X, m_Undef()) || match(Y, m_Undef())) return nullptr; - assert(isa(Shuffle0->getOperand(1)) && - isa(Shuffle1->getOperand(1)) && + assert(match(Shuffle0->getOperand(1), m_Undef()) && + match(Shuffle1->getOperand(1), m_Undef()) && "Unexpected operand for identity shuffle"); // This is a shuffle of 2 widening shuffles. We can shuffle the narrow source @@ -2342,7 +2342,8 @@ // shuffle x, x, mask --> shuffle x, undef, mask' if (LHS == RHS) { - assert(!isa(RHS) && "Shuffle with 2 undef ops not simplified?"); + assert(!match(RHS, m_Undef()) && + "Shuffle with 2 undef ops not simplified?"); // Remap any references to RHS to use LHS. SmallVector Elts; for (unsigned i = 0; i != VWidth; ++i) { @@ -2356,7 +2357,7 @@ } // shuffle undef, x, mask --> shuffle x, undef, mask' - if (isa(LHS)) { + if (match(LHS, m_Undef())) { SVI.commute(); return &SVI; } @@ -2391,7 +2392,7 @@ if (Instruction *I = foldIdentityPaddedShuffles(SVI)) return I; - if (isa(RHS) && canEvaluateShuffled(LHS, Mask)) { + if (match(RHS, m_Undef()) && canEvaluateShuffled(LHS, Mask)) { Value *V = evaluateInDifferentElementOrder(LHS, Mask); return replaceInstUsesWith(SVI, V); } @@ -2530,10 +2531,10 @@ ShuffleVectorInst* LHSShuffle = dyn_cast(LHS); ShuffleVectorInst* RHSShuffle = dyn_cast(RHS); if (LHSShuffle) - if (!isa(LHSShuffle->getOperand(1)) && !isa(RHS)) + if (!match(LHSShuffle->getOperand(1), m_Undef()) && !match(RHS, m_Undef())) LHSShuffle = nullptr; if (RHSShuffle) - if (!isa(RHSShuffle->getOperand(1))) + if (!match(RHSShuffle->getOperand(1), m_Undef())) RHSShuffle = nullptr; if (!LHSShuffle && !RHSShuffle) return MadeChange ? &SVI : nullptr; @@ -2556,7 +2557,7 @@ Value* newRHS = RHS; if (LHSShuffle) { // case 1 - if (isa(RHS)) { + if (match(RHS, m_Undef())) { newLHS = LHSOp0; newRHS = LHSOp1; } @@ -2614,7 +2615,7 @@ // // If the value selected is an undef value, explicitly specify it // with a -1 mask value. (case 1) - if (isa(RHS)) + if (match(RHS, m_Undef())) eltMask = -1; // If RHS is going to be replaced (case 3 or 4), calculate the // new mask value for the element. @@ -2623,8 +2624,8 @@ // If the value selected is an undef value, explicitly specify it // with a -1 mask value. if (eltMask >= (int)RHSOp0Width) { - assert(isa(RHSShuffle->getOperand(1)) - && "should have been check above"); + assert(match(RHSShuffle->getOperand(1), m_Undef()) && + "should have been check above"); eltMask = -1; } } else diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1682,7 +1682,7 @@ Constant *MaybeUndef = ConstOp1 ? ConstantExpr::get(Opcode, UndefScalar, CElt) : ConstantExpr::get(Opcode, CElt, UndefScalar); - if (!isa(MaybeUndef)) { + if (!match(MaybeUndef, m_Undef())) { MayChange = false; break; } diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll --- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll @@ -85,8 +85,7 @@ ; This should turn into a single shuffle. define <4 x float> @test8(<4 x float> %x, <4 x float> %y) { ; CHECK-LABEL: @test8( -; CHECK-NEXT: [[T132:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> -; CHECK-NEXT: [[T134:%.*]] = shufflevector <4 x float> [[T132]], <4 x float> [[Y:%.*]], <4 x i32> +; CHECK-NEXT: [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> ; CHECK-NEXT: ret <4 x float> [[T134]] ; %t4 = extractelement <4 x float> %x, i32 1 diff --git a/llvm/test/Transforms/InstSimplify/icmp-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-constant.ll --- a/llvm/test/Transforms/InstSimplify/icmp-constant.ll +++ b/llvm/test/Transforms/InstSimplify/icmp-constant.ll @@ -1069,8 +1069,7 @@ define <2 x i1> @heterogeneous_constvector(<2 x i8> %x) { ; CHECK-LABEL: @heterogeneous_constvector( -; CHECK-NEXT: [[C:%.*]] = icmp ult <2 x i8> [[X:%.*]], -; CHECK-NEXT: ret <2 x i1> [[C]] +; CHECK-NEXT: ret <2 x i1> zeroinitializer ; %c = icmp ult <2 x i8> %x, ret <2 x i1> %c diff --git a/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll b/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll --- a/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/scalarization.ll @@ -27,7 +27,7 @@ ; CHECK-NEXT: [[DOTSCALAR6:%.*]] = add i32 [[DOTSCALAR5]], [[DIV9]] ; CHECK-NEXT: [[DOTSCALAR7:%.*]] = add i32 [[DOTSCALAR6]], [[MUL21]] ; CHECK-NEXT: [[DOTSCALAR8:%.*]] = add i32 [[DOTSCALAR7]], 317425 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[DOTSCALAR8]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> , i32 [[DOTSCALAR8]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[ADD29:%.*]] = add <4 x i32> [[TMP2]], [[NUM:%.*]] ; CHECK-NEXT: ret <4 x i32> [[ADD29]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll --- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll @@ -425,7 +425,7 @@ ; CHECK-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4 ; CHECK-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8 ; CHECK-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16 -; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x i32> , i32 [[AB1]], i32 1 +; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x i32> , i32 [[AB1]], i32 1 ; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2 ; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3 ; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5 diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp --- a/llvm/unittests/IR/PatternMatch.cpp +++ b/llvm/unittests/IR/PatternMatch.cpp @@ -1022,6 +1022,37 @@ EXPECT_TRUE(A == Val); } +TEST_F(PatternMatchTest, UndefPoisonMix) { + Type *ScalarTy = IRB.getInt8Ty(); + ArrayType *ArrTy = ArrayType::get(ScalarTy, 2); + StructType *StTy = StructType::get(ScalarTy, ScalarTy); + StructType *StTy2 = StructType::get(ScalarTy, StTy); + StructType *StTy3 = StructType::get(StTy, ScalarTy); + Constant *Zero = ConstantInt::getNullValue(ScalarTy); + UndefValue *U = UndefValue::get(ScalarTy); + UndefValue *P = PoisonValue::get(ScalarTy); + + EXPECT_TRUE(match(ConstantVector::get({U, P}), m_Undef())); + EXPECT_TRUE(match(ConstantVector::get({P, U}), m_Undef())); + + EXPECT_TRUE(match(ConstantArray::get(ArrTy, {U, P}), m_Undef())); + EXPECT_TRUE(match(ConstantArray::get(ArrTy, {P, U}), m_Undef())); + + auto *UP = ConstantStruct::get(StTy, {U, P}); + EXPECT_TRUE(match(ConstantStruct::get(StTy2, {U, UP}), m_Undef())); + EXPECT_TRUE(match(ConstantStruct::get(StTy2, {P, UP}), m_Undef())); + EXPECT_TRUE(match(ConstantStruct::get(StTy3, {UP, U}), m_Undef())); + EXPECT_TRUE(match(ConstantStruct::get(StTy3, {UP, P}), m_Undef())); + + EXPECT_FALSE(match(ConstantStruct::get(StTy, {U, Zero}), m_Undef())); + EXPECT_FALSE(match(ConstantStruct::get(StTy, {Zero, U}), m_Undef())); + EXPECT_FALSE(match(ConstantStruct::get(StTy, {P, Zero}), m_Undef())); + EXPECT_FALSE(match(ConstantStruct::get(StTy, {Zero, P}), m_Undef())); + + EXPECT_FALSE(match(ConstantStruct::get(StTy2, {Zero, UP}), m_Undef())); + EXPECT_FALSE(match(ConstantStruct::get(StTy3, {UP, Zero}), m_Undef())); +} + TEST_F(PatternMatchTest, VectorUndefInt) { Type *ScalarTy = IRB.getInt8Ty(); Type *VectorTy = FixedVectorType::get(ScalarTy, 4);