TableGen/GlobalISel: Partially handle immAllOnesV/immAllZerosV

[arsenm]

Summary

These should really match either G_BUILD_VECTOR or
G_BUILD_VECTOR_TRUNC, but there doesn't seem to be an existing
mechanism for matching alternative opcodes. There is GIM_SwitchOpcode,
but it seems to assume it's oly only used for matcher optimization.

I could also omit any opcode check and rely on the matcher directly
checking the opcode, but the table optimizer currently assumes there
has to be an opcode check.

Also doesn't try to handle undef elements like the DAG version.

[aemerson]

Should we just have a G_SPLAT/G_BROADCAST opcode that we can canonicalize to? I find the 3 different representations annoying to deal with.

[arsenm]

In D85202#2193974, @aemerson wrote:

Should we just have a G_SPLAT/G_BROADCAST opcode that we can canonicalize to? I find the 3 different representations annoying to deal with.

I count 2, not sure what the 3rd would be. I've always thought that vector constants should be legal in the DAG, but the prevailing opinion seemed to be that a build_vector of constants is somehow better

[aemerson]

I was thinking of the in-reg vector splat with a shuffle. LGTM anyway.

[foad]

In D85202#2193988, @arsenm wrote:

I've always thought that vector constants should be legal in the DAG

Me too.

[arsenm]

5c5e6d951eec7ce89ed49f8220ce8442c1d20472

llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h

enum {
   /// Check the size of the memory access for the given machine memory operand
   /// against the size of an operand.
   /// - InsnID - Instruction ID
   /// - MMOIdx - MMO index
   /// - OpIdx - The operand index to compare the MMO against
   GIM_CheckMemorySizeEqualToLLT,
   GIM_CheckMemorySizeLessThanLLT,
   GIM_CheckMemorySizeGreaterThanLLT,
+
+  /// Check if this is a vector that can be treated as a vector splat
+  /// constant. This is valid for both G_BUILD_VECTOR as well as
+  /// G_BUILD_VECTOR_TRUNC. For AllOnes refers to individual bits, so a -1
+  /// element.
+  /// - InsnID - Instruction ID
+  GIM_CheckIsBuildVectorAllOnes,
+  GIM_CheckIsBuildVectorAllZeros,
+
   /// Check a generic C++ instruction predicate
   /// - InsnID - Instruction ID
   /// - PredicateID - The ID of the predicate function to call
   GIM_CheckCxxInsnPredicate,
 
   /// Check the type for the specified operand
   /// - InsnID - Instruction ID
   /// - OpIdx - Operand index

llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

case GIM_CheckAPFloatImmPredicate: {
       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 
       if (!testImmPredicate_APFloat(Predicate, Value))
         if (handleReject() == RejectAndGiveUp)
           return false;
       break;
     }
+    case GIM_CheckIsBuildVectorAllOnes:
+    case GIM_CheckIsBuildVectorAllZeros: {
+      int64_t InsnID = MatchTable[CurrentIdx++];
+
+      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
+                      dbgs() << CurrentIdx
+                             << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
+                             << InsnID << "])\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+
+      const MachineInstr *MI = State.MIs[InsnID];
+      assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
+              MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
+             "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
+
+      if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
+        if (!isBuildVectorAllOnes(*MI, MRI)) {
+          if (handleReject() == RejectAndGiveUp)
+            return false;
+        }
+      } else {
+        if (!isBuildVectorAllZeros(*MI, MRI)) {
+          if (handleReject() == RejectAndGiveUp)
+            return false;
+        }
+      }
+
+      break;
+    }
     case GIM_CheckCxxInsnPredicate: {
       int64_t InsnID = MatchTable[CurrentIdx++];
       int64_t Predicate = MatchTable[CurrentIdx++];
       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
                       dbgs()
                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
                           << InsnID << "], Predicate=" << Predicate << ")\n");
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

llvm/include/llvm/CodeGen/GlobalISel/Utils.h

 /// In the worst case, this returns LLT::scalar(1)
 LLVM_READNONE
 LLT getGCDType(LLT OrigTy, LLT TargetTy);
 
 /// \returns The splat index of a G_SHUFFLE_VECTOR \p MI when \p MI is a splat.
 /// If \p MI is not a splat, returns None.
 Optional<int> getSplatIndex(MachineInstr &MI);
 
+/// Return true if the specified instruction is a G_BUILD_VECTOR or
+/// G_BUILD_VECTOR_TRUNC where all of the elements are 0 or undef.
+bool isBuildVectorAllZeros(const MachineInstr &MI,
+                           const MachineRegisterInfo &MRI);
+
+/// Return true if the specified instruction is a G_BUILD_VECTOR or
+/// G_BUILD_VECTOR_TRUNC where all of the elements are ~0 or undef.
+bool isBuildVectorAllOnes(const MachineInstr &MI,
+                          const MachineRegisterInfo &MRI);
+
 } // End namespace llvm.
 #endif

llvm/lib/CodeGen/GlobalISel/Utils.cpp

 //===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file This file implements the utility functions used by the GlobalISel
 /// pipeline.
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/IR/Constants.h"
 
 #define DEBUG_TYPE "globalisel-utils"
 
 using namespace llvm;
+using namespace MIPatternMatch;
 
 Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,
                                    const TargetInstrInfo &TII,
                                    const RegisterBankInfo &RBI, Register Reg,
                                    const TargetRegisterClass &RegClass) {
   if (!RBI.constrainGenericRegister(Reg, RegClass, MRI))
     return MRI.createVirtualRegister(&RegClass);
 
Optional<int> llvm::getSplatIndex(MachineInstr &MI) {
   // as the first non-undef value.
   int SplatValue = *FirstDefinedIdx;
   if (any_of(make_range(std::next(FirstDefinedIdx), Mask.end()),
              [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
     return None;
 
   return SplatValue;
 }
+
+static bool isBuildVectorOp(unsigned Opcode) {
+  return Opcode == TargetOpcode::G_BUILD_VECTOR ||
+         Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;
+}
+
+// TODO: Handle mixed undef elements.
+static bool isBuildVectorConstantSplat(const MachineInstr &MI,
+                                       const MachineRegisterInfo &MRI,
+                                       int64_t SplatValue) {
+  if (!isBuildVectorOp(MI.getOpcode()))
+    return false;
+
+  const unsigned NumOps = MI.getNumOperands();
+  for (unsigned I = 1; I != NumOps; ++I) {
+    Register Element = MI.getOperand(I).getReg();
+    int64_t ElementValue;
+    if (!mi_match(Element, MRI, m_ICst(ElementValue)) ||
+        ElementValue != SplatValue)
+      return false;
+  }
+
+  return true;
+}
+
+bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,
+                                 const MachineRegisterInfo &MRI) {
+  return isBuildVectorConstantSplat(MI, MRI, 0);
+}
+
+bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,
+                                const MachineRegisterInfo &MRI) {
+  return isBuildVectorConstantSplat(MI, MRI, -1);
+}

llvm/test/CodeGen/AArch64/arm64-vabs.ll

; CHECK: sabdl2.2d
   ret <2 x i64> %res1
 }
 
 define <2 x i32> @abspattern1(<2 x i32> %a) nounwind {
 ; CHECK-LABEL: abspattern1:
 ; DAG: abs.2s
 ; DAG-NEXT: ret
 
+; GISEL: neg.2s
 ; GISEL: cmge.2s
-; GISEL: sub.2s
 ; GISEL: fcsel
 ; GISEL: fcsel
         %tmp1neg = sub <2 x i32> zeroinitializer, %a
         %b = icmp sge <2 x i32> %a, zeroinitializer
         %abs = select <2 x i1> %b, <2 x i32> %a, <2 x i32> %tmp1neg
         ret <2 x i32> %abs
 }
 
; CHECK-NEXT: ret
         ret <16 x i8> %abs
 }
 
 define <2 x i64> @abspattern7(<2 x i64> %a) nounwind {
 ; CHECK-LABEL: abspattern7:
 ; DAG: abs.2d
 ; DAG-NEXT: ret
 
+; GISEL: neg.2d
 ; GISEL: cmge.2d
-; GISEL: sub.2d
 ; GISEL: fcsel
 ; GISEL: fcsel
         %tmp1neg = sub <2 x i64> zeroinitializer, %a
         %b = icmp sle <2 x i64> %a, zeroinitializer
         %abs = select <2 x i1> %b, <2 x i64> %tmp1neg, <2 x i64> %a
         ret <2 x i64> %abs
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll

; GCN-NEXT:    ; return to shader part epilog
   %and = and i64 %src0, %not.src1
   %cast = bitcast i64 %and to <2 x float>
   ret <2 x float> %cast
 }
 
 define amdgpu_ps <2 x i32> @s_andn2_v2i32(<2 x i32> inreg %src0, <2 x i32> inreg %src1) {
 ; GCN-LABEL: s_andn2_v2i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b32 s0, -1
-; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_xor_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:    s_andn2_b64 s[0:1], s[2:3], s[4:5]
 ; GCN-NEXT:    ; return to shader part epilog
   %not.src1 = xor <2 x i32> %src1, <i32 -1, i32 -1>
   %and = and <2 x i32> %src0, %not.src1
   ret <2 x i32> %and
 }
 
 define amdgpu_ps <2 x i32> @s_andn2_v2i32_commute(<2 x i32> inreg %src0, <2 x i32> inreg %src1) {
 ; GCN-LABEL: s_andn2_v2i32_commute:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b32 s0, -1
-; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_xor_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    s_andn2_b64 s[0:1], s[2:3], s[4:5]
 ; GCN-NEXT:    ; return to shader part epilog
   %not.src1 = xor <2 x i32> %src1, <i32 -1, i32 -1>
   %and = and <2 x i32> %not.src1, %src0
   ret <2 x i32> %and
 }
 
 define amdgpu_ps i16 @s_andn2_i16(i16 inreg %src0, i16 inreg %src1) {
 ; GFX6-LABEL: s_andn2_i16:

llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll

; GCN-NEXT:    ; return to shader part epilog
   %or = or i64 %src0, %not.src1
   %cast = bitcast i64 %or to <2 x float>
   ret <2 x float> %cast
 }
 
 define amdgpu_ps <2 x i32> @s_orn2_v2i32(<2 x i32> inreg %src0, <2 x i32> inreg %src1) {
 ; GCN-LABEL: s_orn2_v2i32:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b32 s0, -1
-; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_xor_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT:    s_orn2_b64 s[0:1], s[2:3], s[4:5]
 ; GCN-NEXT:    ; return to shader part epilog
   %not.src1 = xor <2 x i32> %src1, <i32 -1, i32 -1>
   %or = or <2 x i32> %src0, %not.src1
   ret <2 x i32> %or
 }
 
 define amdgpu_ps <2 x i32> @s_orn2_v2i32_commute(<2 x i32> inreg %src0, <2 x i32> inreg %src1) {
 ; GCN-LABEL: s_orn2_v2i32_commute:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b32 s0, -1
-; GCN-NEXT:    s_mov_b32 s1, s0
-; GCN-NEXT:    s_xor_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    s_orn2_b64 s[0:1], s[2:3], s[4:5]
 ; GCN-NEXT:    ; return to shader part epilog
   %not.src1 = xor <2 x i32> %src1, <i32 -1, i32 -1>
   %or = or <2 x i32> %not.src1, %src0
   ret <2 x i32> %or
 }
 
 define amdgpu_ps i16 @s_orn2_i16(i16 inreg %src0, i16 inreg %src1) {
 ; GFX6-LABEL: s_orn2_i16:

llvm/test/TableGen/Common/GlobalISelEmitterCommon.td

 def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
 def GPR32Op : RegisterOperand<GPR32>;
 def F0 : Register<"f0"> { let Namespace = "MyTarget"; }
 def FPR32 : RegisterClass<"MyTarget", [f32], 32, (add F0)>;
 def FPR32Op : RegisterOperand<FPR32>;
 def B0 : Register<"b0"> { let Namespace = "MyTarget"; }
 def GPR8 : RegisterClass<"MyTarget", [i8], 8, (add B0)>;
 
+
+def V0 : Register<"v0"> { let Namespace = "MyTarget"; }
+def VecReg128 : RegisterClass<"MyTarget", [v4i32], 128, (add V0)>;
+
 def p0 : PtrValueType <i32, 0>;
 
 class I<dag OOps, dag IOps, list<dag> Pat>
   : Instruction {
   let Namespace = "MyTarget";
   let OutOperandList = OOps;
   let InOperandList = IOps;
   let Pattern = Pat;
 }

llvm/test/TableGen/GlobalISelEmitter-immAllZeroOne.td

+// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -optimize-match-table=false -I %p/../../include -I %p/Common %s -o - | FileCheck -check-prefixes=GISEL-NOOPT,GISEL %s
+// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -optimize-match-table=true -I %p/../../include -I %p/Common %s -o - | FileCheck -check-prefixes=GISEL-OPT,GISEL %s
+
+include "llvm/Target/Target.td"
+include "GlobalISelEmitterCommon.td"
+
+
+// GISEL-OPT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
+// GISEL-OPT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR,
+// GISEL-OPT: GIM_CheckIsBuildVectorAllZeros, /*MI*/1,
+
+// GISEL-OPT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
+// GISEL-OPT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR,
+// GISEL-OPT: GIM_CheckIsBuildVectorAllOnes, /*MI*/1,
+
+
+
+// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LSHR,
+// GISEL-NOOPT: // MIs[0] Operand 2
+// GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
+// GISEL-NOOPT-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
+// GISEL-NOOPT-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR,
+// GISEL-NOOPT-NEXT: GIM_CheckIsBuildVectorAllOnes, /*MI*/1,
+// GISEL-NOOPT-NEXT: // MIs[1] Operand 0
+// GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_v4s16,
+// GISEL-NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
+// GISEL-NOOPT-NEXT: // (srl:{ *:[v4i32] } v4i32:{ *:[v4i32] }:$src0, immAllOnesV:{ *:[v4i16] })  =>  (VFOOONES:{ *:[v4i32] } v4i32:{ *:[v4i32] }:$src0)
+def VFOOONES : I<(outs VecReg128:$dst), (ins VecReg128:$src0),
+  [(set v4i32:$dst, (srl v4i32:$src0, (v4i16 immAllOnesV)))]
+>;
+
+
+// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SHL,
+// GISEL-NOOPT: // MIs[0] Operand 2
+// GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
+// GISEL-NOOPT-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
+// GISEL-NOOPT-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR,
+// GISEL-NOOPT-NEXT: GIM_CheckIsBuildVectorAllZeros, /*MI*/1,
+// GISEL-NOOPT-NEXT: // MIs[1] Operand 0
+// GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_v4s16,
+// GISEL-NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
+// GISEL-NOOPT-NEXT: // (shl:{ *:[v4i32] } v4i32:{ *:[v4i32] }:$src0, immAllZerosV:{ *:[v4i16] })  =>  (VFOOZERO:{ *:[v4i32] } v4i32:{ *:[v4i32] }:$src0)
+def VFOOZERO : I<(outs VecReg128:$dst), (ins VecReg128:$src0),
+  [(set v4i32:$dst, (shl v4i32:$src0, (v4i16 immAllZerosV)))]
+>;

llvm/utils/TableGen/GlobalISelEmitter.cpp

enum PredicateKind {
     IPM_NumOperands,
     IPM_ImmPredicate,
     IPM_Imm,
     IPM_AtomicOrderingMMO,
     IPM_MemoryLLTSize,
     IPM_MemoryVsLLTSize,
     IPM_MemoryAddressSpace,
     IPM_MemoryAlignment,
+    IPM_VectorSplatImm,
     IPM_GenericPredicate,
     OPM_SameOperand,
     OPM_ComplexPattern,
     OPM_IntrinsicID,
     OPM_CmpPredicate,
     OPM_Instruction,
     OPM_Int,
     OPM_LiteralInt,
Table << MatchTable::Opcode(Relation == EqualTo
                                           : "GIM_CheckMemorySizeLessThanLLT")
           << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
           << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
           << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
           << MatchTable::LineBreak;
   }
 };
 
+// Matcher for immAllOnesV/immAllZerosV
+class VectorSplatImmPredicateMatcher : public InstructionPredicateMatcher {
+public:
+  enum SplatKind {
+    AllZeros,
+    AllOnes
+  };
+
+private:
+  SplatKind Kind;
+
+public:
+  VectorSplatImmPredicateMatcher(unsigned InsnVarID, SplatKind K)
+      : InstructionPredicateMatcher(IPM_VectorSplatImm, InsnVarID), Kind(K) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_VectorSplatImm;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B) &&
+           Kind == static_cast<const VectorSplatImmPredicateMatcher &>(B).Kind;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override {
+    if (Kind == AllOnes)
+      Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllOnes");
+    else
+      Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllZeros");
+
+    Table << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID);
+    Table << MatchTable::LineBreak;
+  }
+};
+
 /// Generates code to check an arbitrary C++ instruction predicate.
 class GenericInstructionPredicateMatcher : public InstructionPredicateMatcher {
 protected:
   TreePredicateFn Predicate;
 
 public:
   GenericInstructionPredicateMatcher(unsigned InsnVarID,
                                      TreePredicateFn Predicate)
protected:
   unsigned InsnVarID;
 
   /// PhysRegInputs - List list has an entry for each explicitly specified
   /// physreg input to the pattern.  The first elt is the Register node, the
   /// second is the recorded slot number the input pattern match saved it in.
   SmallVector<std::pair<Record *, unsigned>, 2> PhysRegInputs;
 
 public:
-  InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName)
-      : Rule(Rule), SymbolicName(SymbolicName) {
+  InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName,
+                     bool NumOpsCheck = true)
+      : Rule(Rule), NumOperandsCheck(NumOpsCheck), SymbolicName(SymbolicName) {
     // We create a new instruction matcher.
     // Get a new ID for that instruction.
     InsnVarID = Rule.implicitlyDefineInsnVar(*this);
   }
 
   /// Construct a new instruction predicate and add it to the matcher.
   template <class Kind, class... Args>
   Optional<Kind *> addPredicate(Args &&... args) {

 ///   (G_ADD $src1, $src2)
 /// subpattern.
 class InstructionOperandMatcher : public OperandPredicateMatcher {
 protected:
   std::unique_ptr<InstructionMatcher> InsnMatcher;
 
 public:
   InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                            RuleMatcher &Rule, StringRef SymbolicName)
+                            RuleMatcher &Rule, StringRef SymbolicName,
+                            bool NumOpsCheck = true)
       : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx),
-        InsnMatcher(new InstructionMatcher(Rule, SymbolicName)) {}
+        InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)) {}
 
   static bool classof(const PredicateMatcher *P) {
     return P->getKind() == OPM_Instruction;
   }
 
   InstructionMatcher &getInsnMatcher() const { return *InsnMatcher; }
 
   void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const {
if (ChildRec->isSubClassOf("ImmLeaf")) {
       return failedImport(
           "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
     }
 
     // Place holder for SRCVALUE nodes. Nothing to do here.
     if (ChildRec->getName() == "srcvalue")
       return Error::success();
 
+    const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
+    if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
+      auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
+          InsnMatcher.getRuleMatcher(), SrcChild->getName(), false);
+      InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
+
+      ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
+      InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
+          &Target.getInstruction(RK.getDef("G_BUILD_VECTOR")));
+
+      // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
+      // theoretically not emit any opcode check, but getOpcodeMatcher currently
+      // has to succeed.
+      OperandMatcher &OM =
+          InsnOperand.getInsnMatcher().addOperand(0, "", TempOpIdx);
+      if (auto Error =
+              OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */))
+        return failedImport(toString(std::move(Error)) +
+                            " for result of Src pattern operator");
+
+      InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
+          ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
+                      : VectorSplatImmPredicateMatcher::AllZeros);
+      return Error::success();
+    }
+
     return failedImport(
         "Src pattern child def is an unsupported tablegen class");
   }
 
   return failedImport("Src pattern child is an unsupported kind");
 }
 
 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(