[RISCV] Add DAG combine to optimize vXi64 all ones/zeros fixed vector on RV32.

[craig.topper]

Summary

vXi64 build_vectors are converted to v2Xi32 during type legalization
on RV32. This causes the use of SEW=32 splat instead of SEW=64.

This patch adds a DAG combine to detect the bitcasted pattern
and turn it into a SEW=64 splat.

We could maybe fix this in type legalization, but I'm not sure
we want to lose all zeroes/ones vector knowledge that early since
it's used by negate and not idioms for example.

[frasercrmck]

Sorry for putting this off for so long. I do think we need to do something about this. I've tried to think about alternatives with a wider scope (presumably requiring type legalization) but you're right that all-zeroes and all-ones is important. I suppose I'm a bit a paralyzed on what to do.

[craig.topper]

Though we may not having any example tests right now, this pattern can probably also be created by DAG.getConstant after type legalization due to the NodesMustHaveLegalTypes check.

[craig.topper]

Ping

[frasercrmck]

Sorry for dropping the ball on this. I still haven't managed to look into it further. Is this a pattern that only shows up for all-zeros/all-ones, or is that the only cases that this patch is interested in?

Just wondering: instead of generating this bitcast/v2Xi32 pattern, could we lower this to SPLAT_VECTOR_PARTS and teach LLVM about this node a little better where required? I feel like the last discussion I read was that SPLATs should be open to fixed-length vectors too. That would at least keep it generic for a little longer, and catch all kinds of splat rather than just ones and zeros.

I'm happy to look into this further if you think it's worthwhile. It doesn't have to hold up this patch.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   // We can use any register for comparisons
   setHasMultipleConditionRegisters();
 
   setTargetDAGCombine(ISD::SETCC);
   if (Subtarget.hasStdExtZbp()) {
     setTargetDAGCombine(ISD::OR);
   }
-  if (Subtarget.hasStdExtV())
+  if (Subtarget.hasStdExtV()) {
     setTargetDAGCombine(ISD::FCOPYSIGN);
+    setTargetDAGCombine(ISD::BITCAST);
+  }
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                             LLVMContext &Context,
                                             EVT VT) const {
   if (!VT.isVector())
     return getPointerTy(DL);
   if (Subtarget.hasStdExtV() &&
case ISD::FCOPYSIGN: {
     In2 = In2.getOperand(0);
     if (In2.getOpcode() != ISD::FNEG)
       break;
     SDLoc DL(N);
     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
   }
+  case ISD::BITCAST: {
+    // Look for (vXi64 (bitcast (vXi32 (extract_subvector
+    //                                  (nxvXi32 vmv_v_x_vl, 0/-1), 0))))
+    // Turn it into (vXi64 (extract_subvector (nxvXi64 vmv_v_x_vl, 0/-1), 0))
+    // This gets created during the legalization of all ones/zeros build_vector
+    // on RV32.
+    EVT VT = N->getValueType(0);
+    if (!VT.isFixedLengthVector() || VT.getVectorElementType() != MVT::i64)
+      return SDValue();
+
+    SDValue Src = N->getOperand(0);
+    EVT SrcVT = Src.getValueType();
+
+    if (!SrcVT.isFixedLengthVector() ||
+        SrcVT.getVectorElementType() != MVT::i32)
+      return SDValue();
+
+    // Look for scalable vector to fixed vector cast.
+    if (Src.getOpcode() != ISD::EXTRACT_SUBVECTOR || !Src.hasOneUse() ||
+        !Src.getOperand(0).getValueType().isScalableVector() ||
+        !isNullConstant(Src.getOperand(1)))
+      return SDValue();
+
+    Src = Src.getOperand(0);
+    SrcVT = Src.getValueType();
+    if (SrcVT.getVectorElementType() != MVT::i32)
+      return SDValue();
+
+    // Look for VMV_V_X_VL splat of a constant.
+    if (Src.getOpcode() != RISCVISD::VMV_V_X_VL || !Src.hasOneUse() ||
+        !isa<ConstantSDNode>(Src.getOperand(1)))
+      return SDValue();
+
+    // Splatted value should be all ones/zeros.
+    SDValue Scalar = Src.getOperand(0);
+    if (!isNullConstant(Scalar) && !isAllOnesConstant(Scalar))
+      return SDValue();
+
+    SDLoc DL(N);
+    SrcVT =
+        EVT::getVectorVT(*DAG.getContext(), MVT::i64,
+                         SrcVT.getVectorElementCount().divideCoefficientBy(2));
+    SDValue VL = DAG.getConstant(Src.getConstantOperandVal(1) / 2, DL,
+                                 Src.getOperand(1).getValueType());
+    Src = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, SrcVT, Scalar, VL);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
+                       DAG.getIntPtrConstant(0, DL));
+  }
   }
 
   return SDValue();
 }
 
 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
     const SDNode *N, CombineLevel Level) const {
   // The following folds are only desirable if `(OP _, c1 << c2)` can be

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll

 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    addi a3, a0, 16
 ; RV32-NEXT:    vsetivli a4, 2, e32,m1,ta,mu
 ; RV32-NEXT:    vle32.v v25, (a3)
 ; RV32-NEXT:    vse32.v v25, (sp)
 ; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
 ; RV32-NEXT:    vle64.v v26, (a0)
-; RV32-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
 ; RV32-NEXT:    vmv.v.i v28, 0
 ; RV32-NEXT:    vsetivli a3, 2, e64,m2,tu,mu
 ; RV32-NEXT:    vslideup.vi v28, v26, 0
 ; RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
 ; RV32-NEXT:    vle32.v v26, (sp)
 ; RV32-NEXT:    vsetivli a3, 4, e64,m2,tu,mu
 ; RV32-NEXT:    vslideup.vi v28, v26, 2
 ; RV32-NEXT:    vsetvli a3, zero, e64,m2,ta,mu

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll

 ; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = insertelement <4 x i64> undef, i64 -1, i32 0
   %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer
   store <4 x i64> %b, <4 x i64>* %x
   ret void
 }
 
-; This requires a bitcast on RV32 due to type legalization rewriting the
-; build_vector to v8i32.
-; FIXME: We should prevent this and use the implicit sign extension of vmv.v.x
-; with SEW=64 on RV32.
 define void @splat_allones_with_use_v4i64(<4 x i64>* %x) {
-; LMULMAX8-RV32-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX8-RV32:       # %bb.0:
-; LMULMAX8-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX8-RV32-NEXT:    vle64.v v26, (a0)
-; LMULMAX8-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
-; LMULMAX8-RV32-NEXT:    vmv.v.i v28, -1
-; LMULMAX8-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX8-RV32-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX8-RV32-NEXT:    vse64.v v26, (a0)
-; LMULMAX8-RV32-NEXT:    ret
-;
-; LMULMAX2-RV32-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX2-RV32:       # %bb.0:
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle64.v v26, (a0)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.i v28, -1
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX2-RV32-NEXT:    vse64.v v26, (a0)
-; LMULMAX2-RV32-NEXT:    ret
-;
-; LMULMAX1-RV32-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v27, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v27
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vse64.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX8-RV64-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX8-RV64:       # %bb.0:
-; LMULMAX8-RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX8-RV64-NEXT:    vle64.v v26, (a0)
-; LMULMAX8-RV64-NEXT:    vadd.vi v26, v26, -1
-; LMULMAX8-RV64-NEXT:    vse64.v v26, (a0)
-; LMULMAX8-RV64-NEXT:    ret
+; LMULMAX8-LABEL: splat_allones_with_use_v4i64:
+; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX8-NEXT:    vle64.v v26, (a0)
+; LMULMAX8-NEXT:    vadd.vi v26, v26, -1
+; LMULMAX8-NEXT:    vse64.v v26, (a0)
+; LMULMAX8-NEXT:    ret
 ;
-; LMULMAX2-RV64-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV64-NEXT:    vle64.v v26, (a0)
-; LMULMAX2-RV64-NEXT:    vadd.vi v26, v26, -1
-; LMULMAX2-RV64-NEXT:    vse64.v v26, (a0)
-; LMULMAX2-RV64-NEXT:    ret
+; LMULMAX2-LABEL: splat_allones_with_use_v4i64:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-NEXT:    vle64.v v26, (a0)
+; LMULMAX2-NEXT:    vadd.vi v26, v26, -1
+; LMULMAX2-NEXT:    vse64.v v26, (a0)
+; LMULMAX2-NEXT:    ret
 ;
-; LMULMAX1-RV64-LABEL: splat_allones_with_use_v4i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vadd.vi v25, v25, -1
-; LMULMAX1-RV64-NEXT:    vadd.vi v26, v26, -1
-; LMULMAX1-RV64-NEXT:    vse64.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    ret
+; LMULMAX1-LABEL: splat_allones_with_use_v4i64:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    vle64.v v25, (a1)
+; LMULMAX1-NEXT:    vle64.v v26, (a0)
+; LMULMAX1-NEXT:    vadd.vi v25, v25, -1
+; LMULMAX1-NEXT:    vadd.vi v26, v26, -1
+; LMULMAX1-NEXT:    vse64.v v26, (a0)
+; LMULMAX1-NEXT:    vse64.v v25, (a1)
+; LMULMAX1-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = add <4 x i64> %a, <i64 -1, i64 -1, i64 -1, i64 -1>
   store <4 x i64> %b, <4 x i64>* %x
   ret void
 }
 
 ; This test used to crash at LMUL=8 when inserting a v16i64 subvector into
 ; nxv8i64 at index 0: the v16i64 type was used to get the LMUL, the size of

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll

; CHECK-NEXT:    ret
   %b = insertelement <4 x i32> undef, i32 -1, i32 0
   %c = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %d = add <4 x i32> %a, %c
   store <4 x i32> %d, <4 x i32>* %x
   ret void
 }
 
 define void @add_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: add_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: add_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vadd.vi v25, v25, -1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: add_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vadd.vi v25, v25, -1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %d = add <2 x i64> %a, %c
   store <2 x i64> %d, <2 x i64>* %x
   ret void
 }
 
; CHECK-NEXT:    ret
   %b = insertelement <4 x i32> undef, i32 -1, i32 0
   %c = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %d = sub <4 x i32> %a, %c
   store <4 x i32> %d, <4 x i32>* %x
   ret void
 }
 
 define void @sub_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: sub_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: sub_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi a1, zero, -1
-; LMULMAX1-RV64-NEXT:    vsub.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: sub_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    addi a1, zero, -1
+; CHECK-NEXT:    vsub.vx v25, v25, a1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %d = sub <2 x i64> %a, %c
   store <2 x i64> %d, <2 x i64>* %x
   ret void
 }
 
; CHECK-NEXT:    ret
   %b = insertelement <4 x i32> undef, i32 -1, i32 0
   %c = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %d = xor <4 x i32> %a, %c
   store <4 x i32> %d, <4 x i32>* %x
   ret void
 }
 
 define void @xor_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: xor_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vxor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: xor_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vxor.vi v25, v25, -1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: xor_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vxor.vi v25, v25, -1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %d = xor <2 x i64> %a, %c
   store <2 x i64> %d, <2 x i64>* %x
   ret void
 }