[AArch64] Add support for various operations on nxv1i1 types.

[sdesmalen]

Summary

The supported operations are:

• Logical operations (and, or, xor, bic)
• Logical reductions (and, or, xor, [us]min, [us]max)
• Conversions to/from svbool_t
• Predicate count (CNTP)

[sdesmalen]

Fixed bug in lowering of aarch64_sve_cntp.

[sdesmalen]

Move the lowering of nxv1i1 llvm.aarch64.sve.cntp to the operation that
lowered to that intrinsic (VECREDUCE_OR), so that all the intrinsics only
support legal types.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

20975

Can this be Pg because you've updated getPTrue so getPredicateForVector should do the right thing?

llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll

112–114

Given this patch fixes the xor reductions is it worth merging this patch with D129081?

[sdesmalen]

Use Pg instead of creating a new predicate for VECREDUCE_XOR.

[sdesmalen]

llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll

112–114

This patch already has quite a lot going on, so I'd rather keep them separate if that's alright. I did regenerate the tests for the other patch after rebasing it on top of this patch.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll

112–114

Stacking the patches works for me. Thanks.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case Intrinsic::aarch64_sve_whilelo:
     if (Op.getValueType() == MVT::nxv1i1) {
       SDValue ID =
           DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, dl, MVT::i64);
       return getPUNPKLO(DAG, dl,
                         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::nxv2i1,
                                     ID, Op.getOperand(1), Op.getOperand(2)));
     }
     return SDValue();
+  case Intrinsic::aarch64_sve_cntp:
+    if (Op.getOperand(1).getValueType() == MVT::nxv1i1) {
+      // The other (unpacked lanes) may not be zeroed, so zero
+      // them explicitly here.
+      SDValue Op1 = getSVESafeBitCast(MVT::nxv2i1, Op.getOperand(1), DAG);
+      SDValue Op2 = getSVESafeBitCast(MVT::nxv2i1, Op.getOperand(2), DAG);
+      Op1 = DAG.getNode(ISD::AND, dl, MVT::nxv2i1,
+                         DAG.getConstant(1, dl, MVT::nxv2i1), Op1);
+      return DAG.getNode(
+          ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
+          DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, dl, MVT::i64),
+          Op1, Op2);
+    }
+    return SDValue();
   case Intrinsic::localaddress: {
     const auto &MF = DAG.getMachineFunction();
     const auto *RegInfo = Subtarget->getRegisterInfo();
     unsigned Reg = RegInfo->getLocalAddressRegister(MF);
     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
                               Op.getSimpleValueType());
   }
 
case ISD::VECREDUCE_AND: {
     return getPTest(DAG, VT, Pg, Op, AArch64CC::NONE_ACTIVE);
   }
   case ISD::VECREDUCE_XOR: {
     SDValue ID =
         DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64);
     SDValue Cntp =
         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, ID, Pg, Op);
     return DAG.getAnyExtOrTrunc(Cntp, DL, VT);
   }
   }
 
   return SDValue();
 }
 
 SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
                                                    SDValue ScalarOp,
                                                    SelectionDAG &DAG) const {

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVEorSME] in {
   def : Pat<(AArch64ptest (nxv16i1 PPR:$pg), (nxv16i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv8i1 PPR:$pg), (nxv8i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv4i1 PPR:$pg), (nxv4i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv2i1 PPR:$pg), (nxv2i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
+  def : Pat<(AArch64ptest (nxv1i1 PPR:$pg), (nxv1i1 PPR:$src)),
+            (PTEST_PP PPR:$pg, PPR:$src)>;
 
   let AddedComplexity = 1 in {
   class LD1RPat<ValueType vt, SDPatternOperator operator,
                 Instruction load, Instruction ptrue, ValueType index_vt, ComplexPattern CP, Operand immtype> :
         Pat<(vt (splat_vector (index_vt (operator (CP GPR64:$base, immtype:$offset))))),
             (load (ptrue 31), GPR64:$base, $offset)>;
   }
 
let Predicates = [HasSVEorSME] in {
   def : Pat<(nxv16i1 (and PPR:$Ps1, PPR:$Ps2)),
             (AND_PPzPP (PTRUE_B 31), PPR:$Ps1, PPR:$Ps2)>;
   def : Pat<(nxv8i1 (and PPR:$Ps1, PPR:$Ps2)),
             (AND_PPzPP (PTRUE_H 31), PPR:$Ps1, PPR:$Ps2)>;
   def : Pat<(nxv4i1 (and PPR:$Ps1, PPR:$Ps2)),
             (AND_PPzPP (PTRUE_S 31), PPR:$Ps1, PPR:$Ps2)>;
   def : Pat<(nxv2i1 (and PPR:$Ps1, PPR:$Ps2)),
             (AND_PPzPP (PTRUE_D 31), PPR:$Ps1, PPR:$Ps2)>;
+  // Emulate .Q operation using a PTRUE_D when the other lanes don't matter.
+  def : Pat<(nxv1i1 (and PPR:$Ps1, PPR:$Ps2)),
+            (AND_PPzPP (PTRUE_D 31), PPR:$Ps1, PPR:$Ps2)>;
 
   // Add more complex addressing modes here as required
   multiclass pred_load<ValueType Ty, ValueType PredTy, SDPatternOperator Load,
                        Instruction RegRegInst, Instruction RegImmInst, ComplexPattern AddrCP> {
     let AddedComplexity = 1 in {
       def _reg_reg_z : Pat<(Ty (Load (AddrCP GPR64:$base, GPR64:$offset), (PredTy PPR:$gp), (SVEDup0Undef))),
                            (RegRegInst PPR:$gp, GPR64:$base, GPR64:$offset)>;
     }

llvm/lib/Target/AArch64/SVEInstrFormats.td

multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,
   def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1,
                                !cast<Instruction>(NAME), PTRUE_B>;
   def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1,
                                !cast<Instruction>(NAME), PTRUE_H>;
   def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1,
                                !cast<Instruction>(NAME), PTRUE_S>;
   def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1,
                                !cast<Instruction>(NAME), PTRUE_D>;
+  // Emulate .Q operation using a PTRUE_D when the other lanes don't matter.
+  def : SVE_2_Op_AllActive_Pat<nxv1i1, op_nopred, nxv1i1, nxv1i1,
+                               !cast<Instruction>(NAME), PTRUE_D>;
 }
 
 // An instance of sve_int_pred_log_and but uses op_nopred's first operand as the
 // general predicate.
 multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op,
                                SDPatternOperator op_nopred> :
   sve_int_pred_log<opc, asm, op> {
   def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)),
             (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;
   def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)),
             (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;
   def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)),
             (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;
   def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)),
             (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;
+  // Emulate .Q operation using a PTRUE_D when the other lanes don't matter.
+  def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)),
+            (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Logical Mask Immediate Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_log_imm<bits<2> opc, string asm>
 : I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),

llvm/test/CodeGen/AArch64/sve-int-log.ll

 ; CHECK-LABEL: and_b_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    ret
   %res = and <vscale x 16 x i8> %a, zeroinitializer
   ret <vscale x 16 x i8> %res
 }
 
+define <vscale x 1 x i1> @and_pred_q(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) {
+; CHECK-LABEL: and_pred_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = and <vscale x 1 x i1> %a, %b
+  ret <vscale x 1 x i1> %res
+}
+
 define <vscale x 2 x i1> @and_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
 ; CHECK-LABEL: and_pred_d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %res = and <vscale x 2 x i1> %a, %b
   ret <vscale x 2 x i1> %res
 }

 ; CHECK-NEXT:    bic z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %allones = shufflevector <vscale x 16 x i8> insertelement(<vscale x 16 x i8> undef, i8 -1, i32 0), <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %not_b = xor <vscale x 16 x i8> %b, %allones
   %res = and <vscale x 16 x i8> %a, %not_b
   ret <vscale x 16 x i8> %res
 }
 
+define <vscale x 1 x i1> @bic_pred_q(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) {
+; CHECK-LABEL: bic_pred_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic p0.b, p0/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %allones = shufflevector <vscale x 1 x i1> insertelement(<vscale x 1 x i1> undef, i1 true, i32 0), <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer
+  %not_b = xor <vscale x 1 x i1> %b, %allones
+  %res = and <vscale x 1 x i1> %a, %not_b
+  ret <vscale x 1 x i1> %res
+}
+
 define <vscale x 2 x i1> @bic_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
 ; CHECK-LABEL: bic_pred_d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    bic p0.b, p0/z, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %allones = shufflevector <vscale x 2 x i1> insertelement(<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer
   %not_b = xor <vscale x 2 x i1> %b, %allones
   %res = and <vscale x 2 x i1> %a, %not_b

 define <vscale x 16 x i8> @or_b_zero(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: or_b_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %res = or <vscale x 16 x i8> %a, zeroinitializer
   ret <vscale x 16 x i8> %res
 }
 
+define <vscale x 1 x i1> @or_pred_q(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) {
+; CHECK-LABEL: or_pred_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = or <vscale x 1 x i1> %a, %b
+  ret <vscale x 1 x i1> %res
+}
+
 define <vscale x 2 x i1> @or_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
 ; CHECK-LABEL: or_pred_d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %res = or <vscale x 2 x i1> %a, %b
   ret <vscale x 2 x i1> %res
 }

 define <vscale x 16 x i8> @xor_b_zero(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: xor_b_zero:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %res = xor <vscale x 16 x i8> %a, zeroinitializer
   ret <vscale x 16 x i8> %res
 }
 
+define <vscale x 1 x i1> @xor_pred_q(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) {
+; CHECK-LABEL: xor_pred_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.d
+; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = xor <vscale x 1 x i1> %a, %b
+  ret <vscale x 1 x i1> %res
+}
+
 define <vscale x 2 x i1> @xor_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
 ; CHECK-LABEL: xor_pred_d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p2.d
 ; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %res = xor <vscale x 2 x i1> %a, %b
   ret <vscale x 2 x i1> %res

llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll

 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.and.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_and_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_and_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p2.h, p1.b
+; CHECK-NEXT:    eor p0.b, p1/z, p0.b, p2.b
+; CHECK-NEXT:    ptest p2, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.and.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; ORV
 
 define i1 @reduce_or_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_or_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.or.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_or_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_or_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    ptest p1, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.or.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; XORV
 
 define i1 @reduce_xor_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_xor_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    cntp x8, p1, p0.b
 ; CHECK-NEXT:    and w0, w8, #0x1

 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    cntp x8, p1, p0.d
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.xor.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_xor_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_xor_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    cntp x8, p1, p0.d
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.xor.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; SMAXV
 
 define i1 @reduce_smax_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_smax_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b

 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.smax.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_smax_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_smax_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p2.h, p1.b
+; CHECK-NEXT:    eor p0.b, p1/z, p0.b, p2.b
+; CHECK-NEXT:    ptest p2, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.smax.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; SMINV
 
 define i1 @reduce_smin_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_smin_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.smin.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_smin_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_smin_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    ptest p1, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.smin.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; UMAXV
 
 define i1 @reduce_umax_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_umax_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptest p0, p0.b
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.umax.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_umax_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_umax_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    ptest p1, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.umax.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 ; UMINV
 
 define i1 @reduce_umin_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: reduce_umin_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b

 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.umin.i1.nxv4i1(<vscale x 4 x i1> %vec)
   ret i1 %res
 }
 
+define i1 @reduce_umin_nxv1i1(<vscale x 1 x i1> %vec) {
+; CHECK-LABEL: reduce_umin_nxv1i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p2.h, p1.b
+; CHECK-NEXT:    eor p0.b, p1/z, p0.b, p2.b
+; CHECK-NEXT:    ptest p2, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %res = call i1 @llvm.vector.reduce.umin.i1.nxv1i1(<vscale x 1 x i1> %vec)
+  ret i1 %res
+}
+
 define i1 @reduce_umin_nxv2i1(<vscale x 2 x i1> %vec) {
 ; CHECK-LABEL: reduce_umin_nxv2i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    not p0.b, p1/z, p0.b
 ; CHECK-NEXT:    ptest p1, p0.b
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.umin.i1.nxv2i1(<vscale x 2 x i1> %vec)
   ret i1 %res
 }
 
 declare i1 @llvm.vector.reduce.and.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.and.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.and.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.and.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.and.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.or.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.or.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.or.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.or.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.or.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.xor.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.xor.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.xor.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.xor.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.xor.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.smin.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.smin.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.smin.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.smin.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.smin.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.smax.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.smax.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.smax.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.smax.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.smax.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.umin.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.umin.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.umin.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.umin.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.umin.i1.nxv1i1(<vscale x 1 x i1> %vec)
 
 declare i1 @llvm.vector.reduce.umax.i1.nxv16i1(<vscale x 16 x i1> %vec)
 declare i1 @llvm.vector.reduce.umax.i1.nxv8i1(<vscale x 8 x i1> %vec)
 declare i1 @llvm.vector.reduce.umax.i1.nxv4i1(<vscale x 4 x i1> %vec)
 declare i1 @llvm.vector.reduce.umax.i1.nxv2i1(<vscale x 2 x i1> %vec)
+declare i1 @llvm.vector.reduce.umax.i1.nxv1i1(<vscale x 1 x i1> %vec)

llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
 ; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
+define <vscale x 16 x i1> @reinterpret_bool_from_q(<vscale x 1 x i1> %arg) {
+; CHECK-LABEL: reinterpret_bool_from_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv1i1(<vscale x 1 x i1> %arg)
+  ret <vscale x 16 x i1> %res
+}
+
 ;
 ; Converting from svbool_t
 ;
 
 define <vscale x 16 x i1> @reinterpret_bool_to_b(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_b:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret

 define <vscale x 2 x i1> @reinterpret_bool_to_d(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %pg)
   ret <vscale x 2 x i1> %out
 }
 
+define <vscale x 1 x i1> @reinterpret_bool_to_q(<vscale x 16 x i1> %pg) {
+; CHECK-LABEL: reinterpret_bool_to_q:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %out = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pg)
+  ret <vscale x 1 x i1> %out
+}
+
 ; Reinterpreting a ptrue should not introduce an `and` instruction.
 define <vscale x 16 x i1> @reinterpret_ptrue() {
 ; CHECK-LABEL: reinterpret_ptrue:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ret
   %in = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %out = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %in)

 declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 immarg)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 immarg)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1>)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv1i1(<vscale x 1 x i1>)
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv16i1(<vscale x 16 x i1>)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1>)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1>)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1>)
+declare <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1>)