[RISCV] Support CONCAT_VECTORS on scalable masks

[frasercrmck]

Summary

This patch is a simple fix which registers CONCAT_VECTORS as
custom-lowered for scalable mask vectors. This follows the pattern of
all other scalable-vector types, as the default expansion of
CONCAT_VECTORS cannot handle scalable types, and even if it did it'd go
through the stack and generate worse code.

[craig.topper]

LGTM

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (!Subtarget.is64Bit()) {
       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
     }
 
     for (MVT VT : BoolVecVTs) {
       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
 
       // Mask VTs are custom-expanded into a series of standard nodes
       setOperationAction(ISD::TRUNCATE, VT, Custom);
+      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
 
       setOperationAction(ISD::SELECT, VT, Expand);
       setOperationAction(ISD::SELECT_CC, VT, Expand);

llvm/test/CodeGen/RISCV/rvv/setcc-fp-rv32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; FIXME: The scalar/vector operations ('fv' tests) should swap operands and
 ; condition codes accordingly in order to generate a 'vf' instruction.
 
 define <vscale x 8 x i1> @fcmp_oeq_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb) {
 ; CHECK-LABEL: fcmp_oeq_vv_nxv8f16:
 ; CHECK:       # %bb.0:

 ; CHECK-NEXT:    vmor.mm v0, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x double> undef, double %b, i32 0
   %splat = shufflevector <vscale x 8 x double> %head, <vscale x 8 x double> undef, <vscale x 8 x i32> zeroinitializer
   %vc = fcmp uno <vscale x 8 x double> %va, %splat
   ret <vscale x 8 x i1> %vc
 }
 
+; This fcmp/setcc is split and so we find a scalable-vector mask CONCAT_VECTOR
+; node. Ensure we correctly (custom) lower this.
+define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
+; CHECK-LABEL: fcmp_oeq_vf_nx16f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fcvt.d.w ft0, zero
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vmfeq.vf v25, v16, ft0
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v0, v25, a0
+; CHECK-NEXT:    ret
+  %vc = fcmp oeq <vscale x 16 x double> %va, zeroinitializer
+  ret <vscale x 16 x i1> %vc
+}
+
 attributes #0 = { "no-nans-fp-math"="true" }

llvm/test/CodeGen/RISCV/rvv/setcc-fp-rv64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 ; FIXME: The scalar/vector operations ('fv' tests) should swap operands and
 ; condition codes accordingly in order to generate a 'vf' instruction.
 
 define <vscale x 8 x i1> @fcmp_oeq_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x half> %vb) {
 ; CHECK-LABEL: fcmp_oeq_vv_nxv8f16:
 ; CHECK:       # %bb.0:

 ; CHECK-NEXT:    vmor.mm v0, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x double> undef, double %b, i32 0
   %splat = shufflevector <vscale x 8 x double> %head, <vscale x 8 x double> undef, <vscale x 8 x i32> zeroinitializer
   %vc = fcmp uno <vscale x 8 x double> %va, %splat
   ret <vscale x 8 x i1> %vc
 }
 
+; This fcmp/setcc is split and so we find a scalable-vector mask CONCAT_VECTOR
+; node. Ensure we correctly (custom) lower this.
+define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
+; CHECK-LABEL: fcmp_oeq_vf_nx16f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.d.x ft0, zero
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vmfeq.vf v25, v16, ft0
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v0, v25, a0
+; CHECK-NEXT:    ret
+  %vc = fcmp oeq <vscale x 16 x double> %va, zeroinitializer
+  ret <vscale x 16 x i1> %vc
+}
+
 attributes #0 = { "no-nans-fp-math"="true" }

llvm/test/CodeGen/RISCV/rvv/setcc-integer-rv32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 8 x i1> @icmp_eq_vv_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb) {
 ; CHECK-LABEL: icmp_eq_vv_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    ret
   %vc = icmp eq <vscale x 8 x i8> %va, %vb
; CHECK-NEXT:    ret
   %heada = insertelement <vscale x 8 x i8> undef, i8 5, i32 0
   %splata = shufflevector <vscale x 8 x i8> %heada, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
   %headb = insertelement <vscale x 8 x i8> undef, i8 2, i32 0
   %splatb = shufflevector <vscale x 8 x i8> %headb, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
   %vc = icmp eq <vscale x 8 x i8> %splata, %splatb
   ret <vscale x 8 x i1> %vc
 }
 
+; This icmp/setcc is split and so we find a scalable-vector mask CONCAT_VECTOR
+; node. Ensure we correctly (custom) lower this.
+define <vscale x 16 x i1> @icmp_eq_vi_nx16i64(<vscale x 16 x i64> %va) {
+; CHECK-LABEL: icmp_eq_vi_nx16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli a2, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vmseq.vi v25, v16, 0
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v0, v25, a0
+; CHECK-NEXT:    ret
+  %vc = icmp eq <vscale x 16 x i64> %va, zeroinitializer
+  ret <vscale x 16 x i1> %vc
+}

llvm/test/CodeGen/RISCV/rvv/setcc-integer-rv64.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 8 x i1> @icmp_eq_vv_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb) {
 ; CHECK-LABEL: icmp_eq_vv_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, mu
 ; CHECK-NEXT:    vmseq.vv v0, v8, v9
 ; CHECK-NEXT:    ret
   %vc = icmp eq <vscale x 8 x i8> %va, %vb

 ; CHECK-NEXT:    vmsle.vi v0, v8, 5
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 5, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
   %vc = icmp sle <vscale x 8 x i64> %va, %splat
   ret <vscale x 8 x i1> %vc
 }
 
+; This icmp/setcc is split and so we find a scalable-vector mask CONCAT_VECTOR
+; node. Ensure we correctly (custom) lower this.
+define <vscale x 16 x i1> @icmp_eq_vi_nx16i64(<vscale x 16 x i64> %va) {
+; CHECK-LABEL: icmp_eq_vi_nx16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli a2, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vmseq.vi v25, v16, 0
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v0, v25, a0
+; CHECK-NEXT:    ret
+  %vc = icmp eq <vscale x 16 x i64> %va, zeroinitializer
+  ret <vscale x 16 x i1> %vc
+}