[RISCV] Lower CONCAT_VECTORS to INSERT_SUBVECTOR nodes

[frasercrmck]

Summary

The default expansion of CONCAT_VECTORS goes through the stack. This
patch avoids that penalty by custom-lowering CONCAT_VECTORS to a series
of INSERT_SUBVECTOR nodes. Futher optimizations are possible, but this
is a good start.

[craig.topper]

LGTM

[frasercrmck]

• rebase

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (Subtarget.useRVVForFixedLengthVectors()) {
         for (MVT OtherVT : MVT::fixedlen_vector_valuetypes())
           setTruncStoreAction(VT, OtherVT, Expand);
 
         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+        setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
 
         setOperationAction(ISD::LOAD, VT, Custom);
         setOperationAction(ISD::STORE, VT, Custom);
 
         setOperationAction(ISD::SETCC, VT, Custom);
 
         setOperationAction(ISD::TRUNCATE, VT, Custom);
 
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::INSERT_SUBVECTOR:
     return lowerINSERT_SUBVECTOR(Op, DAG);
   case ISD::EXTRACT_SUBVECTOR:
     return lowerEXTRACT_SUBVECTOR(Op, DAG);
   case ISD::BUILD_VECTOR:
     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
   case ISD::VECTOR_SHUFFLE:
     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
+  case ISD::CONCAT_VECTORS: {
+    // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
+    // better than going through the stack, as the default expansion does.
+    SDLoc DL(Op);
+    MVT VT = Op.getSimpleValueType();
+    assert(VT.isFixedLengthVector() && "Unexpected CONCAT_VECTORS lowering");
+    unsigned NumOpElts =
+        Op.getOperand(0).getSimpleValueType().getVectorNumElements();
+    SDValue Vec = DAG.getUNDEF(VT);
+    for (const auto &OpIdx : enumerate(Op->ops()))
+      Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, OpIdx.value(),
+                        DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
+    return Vec;
+  }
   case ISD::LOAD:
     return lowerFixedLengthVectorLoadToRVV(Op, DAG);
   case ISD::STORE:
     return lowerFixedLengthVectorStoreToRVV(Op, DAG);
   case ISD::SETCC:
     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
   case ISD::ADD:
     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll

 ; LMULMAX8-NEXT:    vsetivli a0, 8, e8,mf2,ta,mu
 ; LMULMAX8-NEXT:    vnsrl.wi v26, v25, 0
 ; LMULMAX8-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
 ; LMULMAX8-NEXT:    vse8.v v26, (a1)
 ; LMULMAX8-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: fp2si_v8f64_v8i8:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi sp, sp, -16
-; LMULMAX1-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX1-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vle64.v v25, (a2)
+; LMULMAX1-NEXT:    vle64.v v26, (a0)
 ; LMULMAX1-NEXT:    addi a2, a0, 32
-; LMULMAX1-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-NEXT:    addi a2, a0, 48
 ; LMULMAX1-NEXT:    vle64.v v27, (a2)
-; LMULMAX1-NEXT:    addi a0, a0, 16
+; LMULMAX1-NEXT:    addi a0, a0, 48
 ; LMULMAX1-NEXT:    vle64.v v28, (a0)
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
 ; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v29, v27
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v27, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v29, v27, 0
-; LMULMAX1-NEXT:    addi a0, sp, 6
-; LMULMAX1-NEXT:    vsetivli a2, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v29, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v27, 0
+; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vmv1r.v v30, v27
+; LMULMAX1-NEXT:    vslideup.vi v30, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
-; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v27, v28
+; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v29, v28
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v28, v27, 0
+; LMULMAX1-NEXT:    vnsrl.wi v28, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v27, v28, 0
-; LMULMAX1-NEXT:    addi a0, sp, 2
-; LMULMAX1-NEXT:    vsetivli a2, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v27, (a0)
+; LMULMAX1-NEXT:    vnsrl.wi v29, v28, 0
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v30, v29, 2
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
-; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v27, v26
+; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v28, v26
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v26, v27, 0
+; LMULMAX1-NEXT:    vnsrl.wi v26, v28, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v27, v26, 0
-; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 4
-; LMULMAX1-NEXT:    vse8.v v27, (a0)
-; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 4
-; LMULMAX1-NEXT:    vle8.v v26, (a0)
-; LMULMAX1-NEXT:    addi a0, sp, 12
-; LMULMAX1-NEXT:    vse8.v v26, (a0)
+; LMULMAX1-NEXT:    vnsrl.wi v28, v26, 0
+; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v28, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
 ; LMULMAX1-NEXT:    vfncvt.rtz.x.f.w v26, v25
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v25, v26, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v26, v25, 0
-; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v26, (sp)
-; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vle8.v v25, (sp)
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v26, 2
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v25, 0
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v25, v27, 0
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v25, v30, 4
 ; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vle8.v v25, (a0)
 ; LMULMAX1-NEXT:    vse8.v v25, (a1)
-; LMULMAX1-NEXT:    addi sp, sp, 16
 ; LMULMAX1-NEXT:    ret
   %a = load <8 x double>, <8 x double>* %x
   %d = fptosi <8 x double> %a to <8 x i8>
   store <8 x i8> %d, <8 x i8>* %y
   ret void
 }
 
 define void @fp2ui_v8f64_v8i8(<8 x double>* %x, <8 x i8>* %y) {
 ; LMULMAX8-LABEL: fp2ui_v8f64_v8i8:
 ; LMULMAX8:       # %bb.0:
 ; LMULMAX8-NEXT:    vsetivli a2, 8, e64,m4,ta,mu
 ; LMULMAX8-NEXT:    vle64.v v28, (a0)
 ; LMULMAX8-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
 ; LMULMAX8-NEXT:    vfncvt.rtz.xu.f.w v26, v28
 ; LMULMAX8-NEXT:    vsetivli a0, 8, e16,m1,ta,mu
 ; LMULMAX8-NEXT:    vnsrl.wi v25, v26, 0
 ; LMULMAX8-NEXT:    vsetivli a0, 8, e8,mf2,ta,mu
 ; LMULMAX8-NEXT:    vnsrl.wi v26, v25, 0
 ; LMULMAX8-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
 ; LMULMAX8-NEXT:    vse8.v v26, (a1)
 ; LMULMAX8-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: fp2ui_v8f64_v8i8:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi sp, sp, -16
-; LMULMAX1-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX1-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vle64.v v25, (a2)
+; LMULMAX1-NEXT:    vle64.v v26, (a0)
 ; LMULMAX1-NEXT:    addi a2, a0, 32
-; LMULMAX1-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-NEXT:    addi a2, a0, 48
 ; LMULMAX1-NEXT:    vle64.v v27, (a2)
-; LMULMAX1-NEXT:    addi a0, a0, 16
+; LMULMAX1-NEXT:    addi a0, a0, 48
 ; LMULMAX1-NEXT:    vle64.v v28, (a0)
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
 ; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v29, v27
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v27, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v29, v27, 0
-; LMULMAX1-NEXT:    addi a0, sp, 6
-; LMULMAX1-NEXT:    vsetivli a2, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v29, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v27, 0
+; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vmv1r.v v30, v27
+; LMULMAX1-NEXT:    vslideup.vi v30, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
-; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v27, v28
+; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v29, v28
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v28, v27, 0
+; LMULMAX1-NEXT:    vnsrl.wi v28, v29, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v27, v28, 0
-; LMULMAX1-NEXT:    addi a0, sp, 2
-; LMULMAX1-NEXT:    vsetivli a2, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v27, (a0)
+; LMULMAX1-NEXT:    vnsrl.wi v29, v28, 0
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v30, v29, 2
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
-; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v27, v26
+; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v28, v26
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v26, v27, 0
+; LMULMAX1-NEXT:    vnsrl.wi v26, v28, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
-; LMULMAX1-NEXT:    vnsrl.wi v27, v26, 0
-; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 4
-; LMULMAX1-NEXT:    vse8.v v27, (a0)
-; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 4
-; LMULMAX1-NEXT:    vle8.v v26, (a0)
-; LMULMAX1-NEXT:    addi a0, sp, 12
-; LMULMAX1-NEXT:    vse8.v v26, (a0)
+; LMULMAX1-NEXT:    vnsrl.wi v28, v26, 0
+; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v28, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
 ; LMULMAX1-NEXT:    vfncvt.rtz.xu.f.w v26, v25
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v25, v26, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v26, v25, 0
-; LMULMAX1-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v26, (sp)
-; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vle8.v v25, (sp)
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v26, 2
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v25, 0
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v25, v27, 0
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v25, v30, 4
 ; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vle8.v v25, (a0)
 ; LMULMAX1-NEXT:    vse8.v v25, (a1)
-; LMULMAX1-NEXT:    addi sp, sp, 16
 ; LMULMAX1-NEXT:    ret
   %a = load <8 x double>, <8 x double>* %x
   %d = fptoui <8 x double> %a to <8 x i8>
   store <8 x i8> %d, <8 x i8>* %y
   ret void
 }

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-exttrunc.ll

 ; LMULMAX2-NEXT:    vsetivli a0, 8, e8,mf2,ta,mu
 ; LMULMAX2-NEXT:    vnsrl.wi v26, v25, 0
 ; LMULMAX2-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
 ; LMULMAX2-NEXT:    vse8.v v26, (a1)
 ; LMULMAX2-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: trunc_v8i8_v8i32:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi sp, sp, -16
-; LMULMAX1-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX1-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-NEXT:    addi a2, a0, 16
-; LMULMAX1-NEXT:    vle32.v v25, (a2)
+; LMULMAX1-NEXT:    vle32.v v25, (a0)
+; LMULMAX1-NEXT:    addi a0, a0, 16
 ; LMULMAX1-NEXT:    vle32.v v26, (a0)
 ; LMULMAX1-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v27, v25, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 4, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v25, v27, 0
-; LMULMAX1-NEXT:    addi a0, sp, 12
-; LMULMAX1-NEXT:    vsetivli a2, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v27, 0
+; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v25, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v25, v26, 0
 ; LMULMAX1-NEXT:    vsetivli a0, 4, e8,mf4,ta,mu
 ; LMULMAX1-NEXT:    vnsrl.wi v26, v25, 0
-; LMULMAX1-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vse8.v v26, (a0)
+; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,tu,mu
+; LMULMAX1-NEXT:    vslideup.vi v27, v26, 4
 ; LMULMAX1-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
-; LMULMAX1-NEXT:    addi a0, sp, 8
-; LMULMAX1-NEXT:    vle8.v v25, (a0)
-; LMULMAX1-NEXT:    vse8.v v25, (a1)
-; LMULMAX1-NEXT:    addi sp, sp, 16
+; LMULMAX1-NEXT:    vse8.v v27, (a1)
 ; LMULMAX1-NEXT:    ret
   %a = load <8 x i32>, <8 x i32>* %x
   %b = trunc <8 x i32> %a to <8 x i8>
   store <8 x i8> %b, <8 x i8>* %z
   ret void
 }