[DAGCombiner] Fold vecreduce_or/and if operand is insert_subvector.

[sdesmalen]

Summary

Fold:

vecreduce_or(insert_subvec(zeroinitializer, vec))
-> vecreduce_or(vec)

vecreduce_and(insert_subvec(allones, vec))
-> vecreduce_and(vec)

vecreduce_and/or(insert_subvec(undef, vec))
-> vecreduce_and/or(vec)

This is useful for SVE which uses insert/extract subvector
to convert fixed-width to/from scalable vectors.

[bsmith]

LGTM with minor nits.

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

22522–22523

Nit: This comment doesn't mention the AND case.

llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll

51

Nit: Should be _ones?

[sdesmalen]

In D118919#3294184, @bsmith wrote:

LGTM with minor nits.

Thanks!

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) {
     unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND
         ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX;
     if (!TLI.isOperationLegalOrCustom(Opcode, VT) &&
         TLI.isOperationLegalOrCustom(NewOpcode, VT) &&
         DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits())
       return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0);
   }
 
+  // vecreduce_or(insert_subvector(zero or undef, val)) -> vecreduce_or(val)
+  // vecreduce_and(insert_subvector(ones or undef, val)) -> vecreduce_and(val)
+  if (N0.getOpcode() == ISD::INSERT_SUBVECTOR &&
+      TLI.isTypeLegal(N0.getOperand(1).getValueType())) {
+    SDValue Vec = N0.getOperand(0);
+    SDValue Subvec = N0.getOperand(1);
+    if ((Opcode == ISD::VECREDUCE_OR &&
+         (N0.getOperand(0).isUndef() || isNullOrNullSplat(Vec))) ||
+        (Opcode == ISD::VECREDUCE_AND &&
+         (N0.getOperand(0).isUndef() || isAllOnesOrAllOnesSplat(Vec))))
+      return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), Subvec);
+  }
+
   return SDValue();
 }
 
 SDValue DAGCombiner::visitVPOp(SDNode *N) {
   // VP operations in which all vector elements are disabled - either by
   // determining that the mask is all false or that the EVL is 0 - can be
   // eliminated.
   bool AreAllEltsDisabled = false;

llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
+
+;
+; OR reductions
+;
+
+define i1 @reduce_or_insert_subvec_into_zero(<vscale x 4 x i1> %in) {
+; CHECK-LABEL: reduce_or_insert_subvec_into_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptest p0, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> zeroinitializer, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+define i1 @reduce_or_insert_subvec_into_poison(<vscale x 4 x i1> %in) {
+; CHECK-LABEL: reduce_or_insert_subvec_into_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptest p0, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> poison, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+define i1 @reduce_or_insert_subvec_into_nonzero(<vscale x 4 x i1> %in, <vscale x 16 x i1> %vec) {
+; CHECK-LABEL: reduce_or_insert_subvec_into_nonzero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    punpklo p2.h, p1.b
+; CHECK-NEXT:    punpkhi p1.h, p1.b
+; CHECK-NEXT:    punpkhi p2.h, p2.b
+; CHECK-NEXT:    uzp1 p0.h, p0.h, p2.h
+; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT:    ptest p0, p0.b
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> %vec, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+;
+; AND reductions
+;
+
+define i1 @reduce_and_insert_subvec_into_ones(<vscale x 4 x i1> %in) {
+; CHECK-LABEL: reduce_and_insert_subvec_into_ones:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    not p0.b, p1/z, p0.b
+; CHECK-NEXT:    ptest p1, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %allones.ins = insertelement <vscale x 16 x i1> poison, i1 1, i32 0
+  %allones = shufflevector <vscale x 16 x i1> %allones.ins,  <vscale x 16 x i1> poison,  <vscale x 16 x i32> zeroinitializer
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> %allones, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+define i1 @reduce_and_insert_subvec_into_poison(<vscale x 4 x i1> %in) {
+; CHECK-LABEL: reduce_and_insert_subvec_into_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    not p0.b, p1/z, p0.b
+; CHECK-NEXT:    ptest p1, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> poison, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+define i1 @reduce_and_insert_subvec_into_var(<vscale x 4 x i1> %in, <vscale x 16 x i1> %vec) {
+; CHECK-LABEL: reduce_and_insert_subvec_into_var:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    punpklo p3.h, p1.b
+; CHECK-NEXT:    punpkhi p1.h, p1.b
+; CHECK-NEXT:    punpkhi p3.h, p3.b
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    uzp1 p0.h, p0.h, p3.h
+; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT:    not p0.b, p2/z, p0.b
+; CHECK-NEXT:    ptest p2, p0.b
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %t = call <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> %vec, <vscale x 4 x i1> %in, i64 0)
+  %res = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> %t)
+  ret i1 %res
+}
+
+declare i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1>)
+declare i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1>)
+declare <vscale x 16 x i1> @llvm.experimental.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1>, <vscale x 4 x i1>, i64)