diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -6145,8 +6145,20 @@
SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
// Pad the vector with the neutral element.
- unsigned OrigElts = OrigVT.getVectorNumElements();
- unsigned WideElts = WideVT.getVectorNumElements();
+ unsigned OrigElts = OrigVT.getVectorMinNumElements();
+ unsigned WideElts = WideVT.getVectorMinNumElements();
+
+ if (WideVT.isScalableVector()) {
+ unsigned GCD = greatestCommonDivisor(OrigElts, WideElts);
+ EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
+ ElementCount::getScalable(GCD));
+ SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
+ for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
+ Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
+ DAG.getVectorIdxConstant(Idx, dl));
+ return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
+ }
+
for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
DAG.getVectorIdxConstant(Idx, dl));
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v,+m -target-abi=ilp32d \
; RUN: -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v,+m -target-abi=lp64d \
; RUN: -verify-machineinstrs < %s | FileCheck %s
declare half @llvm.vector.reduce.fadd.nxv1f16(half, <vscale x 1 x half>)
@@ -1048,3 +1048,101 @@
%red = call reassoc nsz float @llvm.vector.reduce.fadd.nxv1f32(float %s, <vscale x 1 x float> %v)
ret float %red
}
+
+; Test Widen VECREDUCE_SEQ_FADD
+declare half @llvm.vector.reduce.fadd.nxv3f16(half, <vscale x 3 x half>)
+
+define half @vreduce_ord_fadd_nxv3f16(<vscale x 3 x half> %v, half %s) {
+; CHECK-LABEL: vreduce_ord_fadd_nxv3f16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: srli a0, a0, 3
+; CHECK-NEXT: slli a1, a0, 1
+; CHECK-NEXT: add a1, a1, a0
+; CHECK-NEXT: add a0, a1, a0
+; CHECK-NEXT: fmv.h.x ft0, zero
+; CHECK-NEXT: fneg.h ft0, ft0
+; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.v.f v9, ft0
+; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT: vslideup.vx v8, v9, a1
+; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.s.f v9, fa0
+; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, mu
+; CHECK-NEXT: vfredosum.vs v8, v8, v9
+; CHECK-NEXT: vfmv.f.s fa0, v8
+; CHECK-NEXT: ret
+ %red = call half @llvm.vector.reduce.fadd.nxv3f16(half %s, <vscale x 3 x half> %v)
+ ret half %red
+}
+
+declare half @llvm.vector.reduce.fadd.nxv6f16(half, <vscale x 6 x half>)
+
+define half @vreduce_ord_fadd_nxv6f16(<vscale x 6 x half> %v, half %s) {
+; CHECK-LABEL: vreduce_ord_fadd_nxv6f16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: srli a0, a0, 2
+; CHECK-NEXT: add a1, a0, a0
+; CHECK-NEXT: fmv.h.x ft0, zero
+; CHECK-NEXT: fneg.h ft0, ft0
+; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.v.f v10, ft0
+; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT: vslideup.vx v9, v10, a0
+; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.s.f v10, fa0
+; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, mu
+; CHECK-NEXT: vfredosum.vs v8, v8, v10
+; CHECK-NEXT: vfmv.f.s fa0, v8
+; CHECK-NEXT: ret
+ %red = call half @llvm.vector.reduce.fadd.nxv6f16(half %s, <vscale x 6 x half> %v)
+ ret half %red
+}
+
+declare half @llvm.vector.reduce.fadd.nxv10f16(half, <vscale x 10 x half>)
+
+define half @vreduce_ord_fadd_nxv10f16(<vscale x 10 x half> %v, half %s) {
+; CHECK-LABEL: vreduce_ord_fadd_nxv10f16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: srli a0, a0, 2
+; CHECK-NEXT: add a1, a0, a0
+; CHECK-NEXT: fmv.h.x ft0, zero
+; CHECK-NEXT: fneg.h ft0, ft0
+; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.v.f v12, ft0
+; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT: vslideup.vx v10, v12, a0
+; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT: vslideup.vi v11, v12, 0
+; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT: vslideup.vx v11, v12, a0
+; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.s.f v12, fa0
+; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, mu
+; CHECK-NEXT: vfredosum.vs v8, v8, v12
+; CHECK-NEXT: vfmv.f.s fa0, v8
+; CHECK-NEXT: ret
+ %red = call half @llvm.vector.reduce.fadd.nxv10f16(half %s, <vscale x 10 x half> %v)
+ ret half %red
+}
+
+declare half @llvm.vector.reduce.fadd.nxv12f16(half, <vscale x 12 x half>)
+
+define half @vreduce_ord_fadd_nxv12f16(<vscale x 12 x half> %v, half %s) {
+; CHECK-LABEL: vreduce_ord_fadd_nxv12f16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.s.f v12, fa0
+; CHECK-NEXT: fmv.h.x ft0, zero
+; CHECK-NEXT: fneg.h ft0, ft0
+; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, mu
+; CHECK-NEXT: vfmv.v.f v11, ft0
+; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, mu
+; CHECK-NEXT: vfredosum.vs v8, v8, v12
+; CHECK-NEXT: vfmv.f.s fa0, v8
+; CHECK-NEXT: ret
+ %red = call half @llvm.vector.reduce.fadd.nxv12f16(half %s, <vscale x 12 x half> %v)
+ ret half %red
+}