[LegalizeTypes] Replace vecreduce_xor/or/and with vecreduce_add/umax/umin if not legal

[bsmith]

Summary

This is done during type legalization since the target representation of
these nodes may not be valid until after type legalization, and after
type legalization the fact that these are dealing with i1 types may be
lost.

[bsmith]

• Add support for i1 vecreduce_and -> vecreduce_umin.

[paulwalker-arm]

Looks good to me but with a couple of observations.

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

2273

Perhaps this should be else if? Same goes for the next if block.

llvm/test/CodeGen/AArch64/reduce-and.ll

169–171 ↗

(On Diff #441698)

Are the _ins_ tests strictly necessary? It's not immediately obvious how they relate to your code changes.

[bsmith]

llvm/test/CodeGen/AArch64/reduce-and.ll

169–171 ↗

(On Diff #441698)

The rest of the tests in these files cause a zero_extend to be generated, hence would be covered by the current way this was being done (in an add dagcombine), these _ins_ tests cause an any_extend to be generated, which is the reason why this functionality was moved to type legalisation, hence it seemed reasonable to test that case also (i.e. I want to avoid someone moving this back to something like an add dagcombine and thinking everything is fine).

[paulwalker-arm]

llvm/test/CodeGen/AArch64/reduce-and.ll

169–171 ↗

(On Diff #441698)

Sounds reasonable. Thanks for the explanation.

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

case ISD::ZERO_EXTEND:
     return ZExtPromotedInteger(V);
   }
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
   SDLoc dl(N);
   SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
 
-  EVT EltVT = Op.getValueType().getVectorElementType();
-  EVT VT = N->getValueType(0);
+  EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
+  EVT InVT = Op.getValueType();
+  EVT EltVT = InVT.getVectorElementType();
+  EVT ResVT = N->getValueType(0);
+  unsigned Opcode = N->getOpcode();
 
-  if (VT.bitsGE(EltVT))
-    return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
+  // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
+  // vecreduce_xor is not legal
+  if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
+      !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) &&
+      TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT))
+    Opcode = ISD::VECREDUCE_ADD;
+
+  // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
+  // vecreduce_or is not legal
+  if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
+      !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) &&
+      TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT))
+    Opcode = ISD::VECREDUCE_UMAX;
+
+  if (ResVT.bitsGE(EltVT))
+    return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
 
   // Result size must be >= element size. If this is not the case after
   // promotion, also promote the result type and then truncate.
-  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
-  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
+  SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
+  return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
   SDLoc DL(N);
   SDValue Op = N->getOperand(OpNo);
   SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
 
   if (OpNo == 2) { // Mask

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue performANDCombine(SDNode *N,
   EVT VT = N->getValueType(0);
 
   if (SDValue R = performANDORCSELCombine(N, DAG))
     return R;
 
   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
 
-  // Although NEON has no EORV instruction, when only the least significant bit
-  // is required the operation is synonymous with ADDV.
-  if (LHS.getOpcode() == ISD::VECREDUCE_XOR && isOneConstant(RHS) &&
-      LHS.getOperand(0).getValueType().isFixedLengthVector() &&
-      LHS.hasOneUse()) {
-    SDLoc DL(N);
-    SDValue ADDV = DAG.getNode(ISD::VECREDUCE_ADD, DL, VT, LHS.getOperand(0));
-    return DAG.getNode(ISD::AND, DL, VT, ADDV, RHS);
-  }
-
   if (VT.isScalableVector())
     return performSVEAndCombine(N, DCI);
 
   // The combining code below works only for NEON vectors. In particular, it
   // does not work for SVE when dealing with vectors wider than 128 bits.
   if (!VT.is64BitVector() && !VT.is128BitVector())
     return SDValue();
 

llvm/test/CodeGen/AArch64/reduce-or.ll

 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %a)
   ret i1 %or_result
 }
 
 define i1 @test_redor_v2i1(<2 x i1> %a) {
 ; CHECK-LABEL: test_redor_v2i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    orr w8, w9, w8
+; CHECK-NEXT:    umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v2i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    mov s1, v0.s[1]
 ; GISEL-NEXT:    fmov w8, s0
 ; GISEL-NEXT:    fmov w9, s1
 ; GISEL-NEXT:    orr w8, w8, w9
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %a)
   ret i1 %or_result
 }
 
 define i1 @test_redor_v4i1(<4 x i1> %a) {
 ; CHECK-LABEL: test_redor_v4i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    orr w8, w8, w11
+; CHECK-NEXT:    umaxv h0, v0.4h
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v4i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    mov h1, v0.h[1]
 ; GISEL-NEXT:    mov h2, v0.h[2]

 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %a)
   ret i1 %or_result
 }
 
 define i1 @test_redor_v8i1(<8 x i1> %a) {
 ; CHECK-LABEL: test_redor_v8i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[6]
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    umov w10, v0.b[7]
-; CHECK-NEXT:    orr w8, w8, w11
-; CHECK-NEXT:    orr w8, w8, w12
-; CHECK-NEXT:    orr w8, w8, w13
-; CHECK-NEXT:    orr w8, w8, w9
-; CHECK-NEXT:    orr w8, w8, w10
+; CHECK-NEXT:    umaxv b0, v0.8b
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v8i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    mov b1, v0.b[1]
 ; GISEL-NEXT:    mov b2, v0.b[2]

 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %a)
   ret i1 %or_result
 }
 
 define i1 @test_redor_v16i1(<16 x i1> %a) {
 ; CHECK-LABEL: test_redor_v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[5]
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    umov w10, v0.b[6]
-; CHECK-NEXT:    orr w8, w8, w11
-; CHECK-NEXT:    umov w11, v0.b[7]
-; CHECK-NEXT:    orr w8, w8, w12
-; CHECK-NEXT:    orr w8, w8, w9
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    orr w8, w8, w11
+; CHECK-NEXT:    umaxv b0, v0.16b
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v16i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    mov b1, v0.b[1]
 ; GISEL-NEXT:    mov b2, v0.b[2]
 ; GISEL-NEXT:    mov b3, v0.b[3]

 ; GISEL-NEXT:    orr w9, w10, w11
 ; GISEL-NEXT:    orr w8, w8, w9
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %a)
   ret i1 %or_result
 }
 
+define <16 x i1> @test_redor_ins_v16i1(<16 x i1> %a) {
+; CHECK-LABEL: test_redor_ins_v16i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaxv b0, v0.16b
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_redor_ins_v16i1:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    mov b1, v0.b[1]
+; GISEL-NEXT:    mov b2, v0.b[2]
+; GISEL-NEXT:    mov b3, v0.b[3]
+; GISEL-NEXT:    mov b4, v0.b[4]
+; GISEL-NEXT:    mov b5, v0.b[5]
+; GISEL-NEXT:    mov b6, v0.b[6]
+; GISEL-NEXT:    mov b7, v0.b[7]
+; GISEL-NEXT:    fmov w8, s0
+; GISEL-NEXT:    fmov w9, s1
+; GISEL-NEXT:    fmov w10, s2
+; GISEL-NEXT:    fmov w11, s3
+; GISEL-NEXT:    mov b16, v0.b[8]
+; GISEL-NEXT:    mov b17, v0.b[9]
+; GISEL-NEXT:    mov b18, v0.b[10]
+; GISEL-NEXT:    mov b19, v0.b[11]
+; GISEL-NEXT:    orr w8, w8, w9
+; GISEL-NEXT:    orr w9, w10, w11
+; GISEL-NEXT:    fmov w10, s4
+; GISEL-NEXT:    fmov w11, s5
+; GISEL-NEXT:    fmov w12, s6
+; GISEL-NEXT:    fmov w13, s7
+; GISEL-NEXT:    mov b20, v0.b[12]
+; GISEL-NEXT:    mov b21, v0.b[13]
+; GISEL-NEXT:    mov b22, v0.b[14]
+; GISEL-NEXT:    mov b23, v0.b[15]
+; GISEL-NEXT:    orr w10, w10, w11
+; GISEL-NEXT:    orr w11, w12, w13
+; GISEL-NEXT:    fmov w12, s16
+; GISEL-NEXT:    fmov w13, s17
+; GISEL-NEXT:    fmov w14, s18
+; GISEL-NEXT:    fmov w15, s19
+; GISEL-NEXT:    fmov w16, s22
+; GISEL-NEXT:    fmov w17, s23
+; GISEL-NEXT:    orr w8, w8, w9
+; GISEL-NEXT:    orr w12, w12, w13
+; GISEL-NEXT:    orr w9, w10, w11
+; GISEL-NEXT:    orr w13, w14, w15
+; GISEL-NEXT:    fmov w14, s20
+; GISEL-NEXT:    fmov w15, s21
+; GISEL-NEXT:    orr w10, w12, w13
+; GISEL-NEXT:    orr w8, w8, w9
+; GISEL-NEXT:    orr w14, w14, w15
+; GISEL-NEXT:    orr w15, w16, w17
+; GISEL-NEXT:    orr w11, w14, w15
+; GISEL-NEXT:    orr w9, w10, w11
+; GISEL-NEXT:    orr w8, w8, w9
+; GISEL-NEXT:    fmov s0, w8
+; GISEL-NEXT:    ret
+  %or_result = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %a)
+  %ins = insertelement <16 x i1> poison, i1 %or_result, i64 0
+  ret <16 x i1> %ins
+}
+
 define i8 @test_redor_v1i8(<1 x i8> %a) {
 ; CHECK-LABEL: test_redor_v1i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    umov w0, v0.b[0]
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v1i8:

llvm/test/CodeGen/AArch64/reduce-xor.ll

 ; GISEL-NEXT:    eor w9, w10, w11
 ; GISEL-NEXT:    eor w8, w8, w9
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %a)
   ret i1 %or_result
 }
 
+define <16 x i1> @test_redxor_ins_v16i1(<16 x i1> %a) {
+; CHECK-LABEL: test_redxor_ins_v16i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: test_redxor_ins_v16i1:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    mov b1, v0.b[1]
+; GISEL-NEXT:    mov b2, v0.b[2]
+; GISEL-NEXT:    mov b3, v0.b[3]
+; GISEL-NEXT:    mov b4, v0.b[4]
+; GISEL-NEXT:    mov b5, v0.b[5]
+; GISEL-NEXT:    mov b6, v0.b[6]
+; GISEL-NEXT:    mov b7, v0.b[7]
+; GISEL-NEXT:    fmov w8, s0
+; GISEL-NEXT:    fmov w9, s1
+; GISEL-NEXT:    fmov w10, s2
+; GISEL-NEXT:    fmov w11, s3
+; GISEL-NEXT:    mov b16, v0.b[8]
+; GISEL-NEXT:    mov b17, v0.b[9]
+; GISEL-NEXT:    mov b18, v0.b[10]
+; GISEL-NEXT:    mov b19, v0.b[11]
+; GISEL-NEXT:    eor w8, w8, w9
+; GISEL-NEXT:    eor w9, w10, w11
+; GISEL-NEXT:    fmov w10, s4
+; GISEL-NEXT:    fmov w11, s5
+; GISEL-NEXT:    fmov w12, s6
+; GISEL-NEXT:    fmov w13, s7
+; GISEL-NEXT:    mov b20, v0.b[12]
+; GISEL-NEXT:    mov b21, v0.b[13]
+; GISEL-NEXT:    mov b22, v0.b[14]
+; GISEL-NEXT:    mov b23, v0.b[15]
+; GISEL-NEXT:    eor w10, w10, w11
+; GISEL-NEXT:    eor w11, w12, w13
+; GISEL-NEXT:    fmov w12, s16
+; GISEL-NEXT:    fmov w13, s17
+; GISEL-NEXT:    fmov w14, s18
+; GISEL-NEXT:    fmov w15, s19
+; GISEL-NEXT:    fmov w16, s22
+; GISEL-NEXT:    fmov w17, s23
+; GISEL-NEXT:    eor w8, w8, w9
+; GISEL-NEXT:    eor w12, w12, w13
+; GISEL-NEXT:    eor w9, w10, w11
+; GISEL-NEXT:    eor w13, w14, w15
+; GISEL-NEXT:    fmov w14, s20
+; GISEL-NEXT:    fmov w15, s21
+; GISEL-NEXT:    eor w10, w12, w13
+; GISEL-NEXT:    eor w8, w8, w9
+; GISEL-NEXT:    eor w14, w14, w15
+; GISEL-NEXT:    eor w15, w16, w17
+; GISEL-NEXT:    eor w11, w14, w15
+; GISEL-NEXT:    eor w9, w10, w11
+; GISEL-NEXT:    eor w8, w8, w9
+; GISEL-NEXT:    fmov s0, w8
+; GISEL-NEXT:    ret
+  %xor_result = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %a)
+  %ins = insertelement <16 x i1> poison, i1 %xor_result, i64 0
+  ret <16 x i1> %ins
+}
+
 define i8 @test_redxor_v1i8(<1 x i8> %a) {
 ; CHECK-LABEL: test_redxor_v1i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    umov w0, v0.b[0]
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v1i8:

llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll

 ; CHECK-NEXT:    ret
   %b = call i32 @llvm.vector.reduce.umax.v3i32(<3 x i32> %a)
   ret i32 %b
 }
 
 define i1 @test_v4i1(<4 x i1> %a) nounwind {
 ; CHECK-LABEL: test_v4i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    orr w8, w8, w11
+; CHECK-NEXT:    umaxv h0, v0.4h
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
   %b = call i1 @llvm.vector.reduce.umax.v4i1(<4 x i1> %a)
   ret i1 %b
 }
 
 define i24 @test_v4i24(<4 x i24> %a) nounwind {
 ; CHECK-LABEL: test_v4i24: