diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -6145,12 +6145,23 @@ SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); // Pad the vector with the neutral element. - unsigned OrigElts = OrigVT.getVectorNumElements(); - unsigned WideElts = WideVT.getVectorNumElements(); + unsigned OrigElts = OrigVT.getVectorMinNumElements(); + unsigned WideElts = WideVT.getVectorMinNumElements(); + + if (WideVT.isScalableVector()) { + unsigned GCD = greatestCommonDivisor(OrigElts, WideElts); + EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, + ElementCount::getScalable(GCD)); + SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem); + for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) + Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral, + DAG.getVectorIdxConstant(Idx, dl)); + return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); + } + for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, DAG.getVectorIdxConstant(Idx, dl)); - return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); } diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll --- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare half @llvm.vector.reduce.fadd.nxv1f16(half, ) @@ -1048,3 +1048,101 @@ %red = call reassoc nsz float @llvm.vector.reduce.fadd.nxv1f32(float %s, %v) ret float %red } + +; Test Widen VECREDUCE_SEQ_FADD +declare half @llvm.vector.reduce.fadd.nxv3f16(half, ) + +define half @vreduce_ord_fadd_nxv3f16( %v, half %s) { +; CHECK-LABEL: vreduce_ord_fadd_nxv3f16: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a0, vlenb +; CHECK-NEXT: srli a0, a0, 3 +; CHECK-NEXT: slli a1, a0, 1 +; CHECK-NEXT: add a1, a1, a0 +; CHECK-NEXT: add a0, a1, a0 +; CHECK-NEXT: fmv.h.x ft0, zero +; CHECK-NEXT: fneg.h ft0, ft0 +; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu +; CHECK-NEXT: vfmv.v.f v9, ft0 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; CHECK-NEXT: vslideup.vx v8, v9, a1 +; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu +; CHECK-NEXT: vfmv.s.f v9, fa0 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, mu +; CHECK-NEXT: vfredosum.vs v8, v8, v9 +; CHECK-NEXT: vfmv.f.s fa0, v8 +; CHECK-NEXT: ret + %red = call half @llvm.vector.reduce.fadd.nxv3f16(half %s, %v) + ret half %red +} + +declare half @llvm.vector.reduce.fadd.nxv6f16(half, ) + +define half @vreduce_ord_fadd_nxv6f16( %v, half %s) { +; CHECK-LABEL: vreduce_ord_fadd_nxv6f16: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a0, vlenb +; CHECK-NEXT: srli a0, a0, 2 +; CHECK-NEXT: add a1, a0, a0 +; CHECK-NEXT: fmv.h.x ft0, zero +; CHECK-NEXT: fneg.h ft0, ft0 +; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu +; CHECK-NEXT: vfmv.v.f v10, ft0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu +; CHECK-NEXT: vslideup.vx v9, v10, a0 +; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu +; CHECK-NEXT: vfmv.s.f v10, fa0 +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, mu +; CHECK-NEXT: vfredosum.vs v8, v8, v10 +; CHECK-NEXT: vfmv.f.s fa0, v8 +; CHECK-NEXT: ret + %red = call half @llvm.vector.reduce.fadd.nxv6f16(half %s, %v) + ret half %red +} + +declare half @llvm.vector.reduce.fadd.nxv10f16(half, ) + +define half @vreduce_ord_fadd_nxv10f16( %v, half %s) { +; CHECK-LABEL: vreduce_ord_fadd_nxv10f16: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a0, vlenb +; CHECK-NEXT: srli a0, a0, 2 +; CHECK-NEXT: add a1, a0, a0 +; CHECK-NEXT: fmv.h.x ft0, zero +; CHECK-NEXT: fneg.h ft0, ft0 +; CHECK-NEXT: vsetvli a2, zero, e16, m1, ta, mu +; CHECK-NEXT: vfmv.v.f v12, ft0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu +; CHECK-NEXT: vslideup.vx v10, v12, a0 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; CHECK-NEXT: vslideup.vi v11, v12, 0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, tu, mu +; CHECK-NEXT: vslideup.vx v11, v12, a0 +; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu +; CHECK-NEXT: vfmv.s.f v12, fa0 +; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, mu +; CHECK-NEXT: vfredosum.vs v8, v8, v12 +; CHECK-NEXT: vfmv.f.s fa0, v8 +; CHECK-NEXT: ret + %red = call half @llvm.vector.reduce.fadd.nxv10f16(half %s, %v) + ret half %red +} + +declare half @llvm.vector.reduce.fadd.nxv12f16(half, ) + +define half @vreduce_ord_fadd_nxv12f16( %v, half %s) { +; CHECK-LABEL: vreduce_ord_fadd_nxv12f16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, mu +; CHECK-NEXT: vfmv.s.f v12, fa0 +; CHECK-NEXT: fmv.h.x ft0, zero +; CHECK-NEXT: fneg.h ft0, ft0 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, mu +; CHECK-NEXT: vfmv.v.f v11, ft0 +; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, mu +; CHECK-NEXT: vfredosum.vs v8, v8, v12 +; CHECK-NEXT: vfmv.f.s fa0, v8 +; CHECK-NEXT: ret + %red = call half @llvm.vector.reduce.fadd.nxv12f16(half %s, %v) + ret half %red +}