[PatternMatch] don't match a scalar select of bool vectors as a logical-and or logical-or

[spatel]

Summary

Most folds based on these matchers already check to make sure the condition type is the same as the select type, and it seems unlikely that a fold would want to handle a scalar-select-of-vectors pattern (there are no regression tests for it).

This is a preliminary step for fixing #issue 58552. The fold(s) responsible for that crash (D101807, D101375) don't use the matchers yet, but they probably should.

[nikic]

LGTM

llvm/include/llvm/IR/PatternMatch.h

if (I->getOpcode() == Opcode) {
       return (L.match(Op0) && R.match(Op1)) ||
              (Commutable && L.match(Op1) && R.match(Op0));
     }
 
     if (auto *Select = dyn_cast<SelectInst>(I)) {
       auto *Cond = Select->getCondition();
       auto *TVal = Select->getTrueValue();
       auto *FVal = Select->getFalseValue();
+
+      // Don't match a scalar select of bool vectors.
+      // Transforms expect a single type for operands if this matches.
+      if (Cond->getType() != Select->getType())
+        return false;
+
       if (Opcode == Instruction::And) {
         auto *C = dyn_cast<Constant>(FVal);
         if (C && C->isNullValue())
           return (L.match(Cond) && R.match(TVal)) ||
                  (Commutable && L.match(TVal) && R.match(Cond));
       } else {
         assert(Opcode == Instruction::Or);
         auto *C = dyn_cast<Constant>(TVal);

llvm/unittests/IR/PatternMatch.cpp

TEST_F(PatternMatchTest, VectorLogicalSelects) {
   Value *MixedTypeAnd = IRB.CreateSelect(Scalar, Vector, F);
 
   // select <3 x i1> Vector, <3 x i1> <i1 1, i1 1, i1 1>, <3 x i1> Vector
   Value *VecOr = IRB.CreateSelect(Vector, T, Vector);
 
   // select i1 Scalar, <3 x i1> <i1 1, i1 1, i1 1>, <3 x i1> Vector
   Value *MixedTypeOr = IRB.CreateSelect(Scalar, T, Vector);
 
+  // We allow matching a real vector logical select,
+  // but not a scalar select of vector bools.
   EXPECT_TRUE(match(VecAnd, m_LogicalAnd(m_Value(), m_Value())));
-  EXPECT_TRUE(match(MixedTypeAnd, m_LogicalAnd(m_Value(), m_Value())));
+  EXPECT_FALSE(match(MixedTypeAnd, m_LogicalAnd(m_Value(), m_Value())));
   EXPECT_TRUE(match(VecOr, m_LogicalOr(m_Value(), m_Value())));
-  EXPECT_TRUE(match(MixedTypeOr, m_LogicalOr(m_Value(), m_Value())));
+  EXPECT_FALSE(match(MixedTypeOr, m_LogicalOr(m_Value(), m_Value())));
 }
 
 TEST_F(PatternMatchTest, VScale) {
   DataLayout DL = M->getDataLayout();
 
   Type *VecTy = ScalableVectorType::get(IRB.getInt8Ty(), 1);
   Type *VecPtrTy = VecTy->getPointerTo();
   Value *NullPtrVec = Constant::getNullValue(VecPtrTy);