[RISCV] Support the scalable-vector fadd reduction intrinsic

[frasercrmck]

Summary

This patch adds support for both the fadd reduction intrinsic, in both
the ordered and unordered modes.

The fmin and fmax intrinsics are not currently supported due to a
discrepancy between the LLVM semantics and the RVV ISA behaviour with
regards to signaling NaNs. This behaviour is likely fixed in version 2.3
of the RISC-V F/D/Q extension, but until then the intrinsics can be left
unsupported.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1718

Won't getSimpleVT) below already assert?

1719

getSimpleValueType()

1731

I think ISD::SCALAR_TO_VECTOR is what we would more commonly use.

llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll

31

Annoying that SelectionDAG breaks the scalar input out separately.

[jrtc27]

If something's not yet supported due to a spec deficiency that should be documented in the code with a link to the outstanding spec issue.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1731

i guess we don't have a true scalar_to_vector instruction in RVV, i.e. an instruction that just moves a scalar register without being dependent on the previous vector register. So I think INSERT_VECTOR_ELT is ok.

[frasercrmck]

In D95870#2536916, @jrtc27 wrote:

If something's not yet supported due to a spec deficiency that should be documented in the code with a link to the outstanding spec issue.

That's a good idea. My first thought would be RISCVTTIImpl::shouldExpandReduction, not that it can actually expand it right now? Regardless of our intentions, if we find such a reduction in the IR we crash.

[frasercrmck]

• simplify SimpleValueType access
• update documentation about missing fmin/fmax reductions

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1718

Yep, thanks. Done as part of using getSimpleValueType() below.

1731

Yeah, tbh, I can see how it could go either way. Since the upper elements of SCALAR_TO_VECTOR are undefined I suppose we could define it as equivalent to a splat. But I don't think that's worth doing, and INSERT_VECTOR_ELT better matches the semantics of the "scalar" operands of these reductions.

[frasercrmck]

• rebase

[craig.topper]

LGTM

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1731

Should we splat here like we do for the integer reductions?

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1731

I think we have a special case for splatting 0.0 so that could be beneficial.

[frasercrmck]

• address Craig's suggestions to use a splat instead of insertelt0
• could perhaps merge the int and fp reduction lowering code if we wanted to,

though it's not a perfect fit

[craig.topper]

LGTM

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
   VECREDUCE_ADD,
   VECREDUCE_UMAX,
   VECREDUCE_SMAX,
   VECREDUCE_UMIN,
   VECREDUCE_SMIN,
   VECREDUCE_AND,
   VECREDUCE_OR,
   VECREDUCE_XOR,
+  VECREDUCE_FADD,
+  VECREDUCE_SEQ_FADD,
 };
 } // namespace RISCVISD
 
 class RISCVTargetLowering : public TargetLowering {
   const RISCVSubtarget &Subtarget;
 
 public:
   explicit RISCVTargetLowering(const TargetMachine &TM,
private:
   SDValue lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
                              int64_t ExtTrueVal) const;
   SDValue lowerVectorMaskTrunc(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFPVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
 
   bool isEligibleForTailCallOptimization(
       CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
       const SmallVector<CCValAssign, 16> &ArgLocs) const;
 
   /// Generate error diagnostics if any register used by CC has been marked
   /// reserved.
   void validateCCReservedRegs(

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

const auto SetCommonVFPActions = [&](MVT VT) {
       setOperationAction(ISD::FP_ROUND, VT, Custom);
       setOperationAction(ISD::FP_EXTEND, VT, Custom);
       // Custom-lower insert/extract operations to simplify patterns.
       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
       // Expand various condition codes (explained above).
       for (auto CC : VFPCCToExpand)
         setCondCodeAction(CC, VT, Expand);
+
+      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
     };
 
     if (Subtarget.hasStdExtZfh())
       for (MVT VT : F16VecVTs)
         SetCommonVFPActions(VT);
 
     if (Subtarget.hasStdExtF())
       for (MVT VT : F32VecVTs)
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_SMAX:
   case ISD::VECREDUCE_UMIN:
   case ISD::VECREDUCE_SMIN:
   case ISD::VECREDUCE_AND:
   case ISD::VECREDUCE_OR:
   case ISD::VECREDUCE_XOR:
     return lowerVECREDUCE(Op, DAG);
+  case ISD::VECREDUCE_FADD:
+  case ISD::VECREDUCE_SEQ_FADD:
+    return lowerFPVECREDUCE(Op, DAG);
   }
 }
 
 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
                              SelectionDAG &DAG, unsigned Flags) {
   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
 }
 
SDValue IdentitySplat =
       DAG.getSplatVector(M1VT, DL, DAG.getConstant(IdentityVal, DL, VecEltVT));
   SDValue Reduction =
       DAG.getNode(RVVOpcode, DL, M1VT, Op.getOperand(0), IdentitySplat);
   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
                              DAG.getConstant(0, DL, Subtarget.getXLenVT()));
   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
 }
 
+// Given a reduction op, this function returns the matching reduction opcode,
+// the vector SDValue and the scalar SDValue required to lower this to a
+// RISCVISD node.
+static std::tuple<unsigned, SDValue, SDValue>
+getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
+  SDLoc DL(Op);
+  switch (Op.getOpcode()) {
+  default:
+    llvm_unreachable("Unhandled reduction");
+  case ISD::VECREDUCE_FADD:
+    return {RISCVISD::VECREDUCE_FADD, Op.getOperand(0),
+            DAG.getConstantFP(0.0, DL, EltVT)};
+  case ISD::VECREDUCE_SEQ_FADD:
+    return {RISCVISD::VECREDUCE_SEQ_FADD, Op.getOperand(1), Op.getOperand(0)};
+  }
+}
+
+SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  assert(Op.getValueType().isSimple() && "Unexpected vector-reduce lowering");
+  MVT VecEltVT = Op.getValueType().getSimpleVT();
+  // We have to perform a bit of a dance to get from our vector type to the
+  // correct LMUL=1 vector type. See above for an explanation.
+  unsigned NumElts = 64 / VecEltVT.getSizeInBits();
+  MVT M1VT = MVT::getScalableVectorVT(VecEltVT, NumElts);
+
+  unsigned RVVOpcode;
+  SDValue VectorVal, ScalarVal;
+  std::tie(RVVOpcode, VectorVal, ScalarVal) =
+      getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
+
+  SDValue ZeroIdx = DAG.getConstant(0, DL, Subtarget.getXLenVT());
+  SDValue ScalarInVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, M1VT,
+                                    DAG.getUNDEF(M1VT), ScalarVal, ZeroIdx);
+  SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, VectorVal, ScalarInVec);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction, ZeroIdx);
+}
+
 // Returns the opcode of the target-specific SDNode that implements the 32-bit
 // form of the given Opcode.
 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
   switch (Opcode) {
   default:
     llvm_unreachable("Unexpected opcode");
   case ISD::SHL:
     return RISCVISD::SLLW;
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(VECREDUCE_ADD)
   NODE_NAME_CASE(VECREDUCE_UMAX)
   NODE_NAME_CASE(VECREDUCE_SMAX)
   NODE_NAME_CASE(VECREDUCE_UMIN)
   NODE_NAME_CASE(VECREDUCE_SMIN)
   NODE_NAME_CASE(VECREDUCE_AND)
   NODE_NAME_CASE(VECREDUCE_OR)
   NODE_NAME_CASE(VECREDUCE_XOR)
+  NODE_NAME_CASE(VECREDUCE_FADD)
+  NODE_NAME_CASE(VECREDUCE_SEQ_FADD)
   }
   // clang-format on
   return nullptr;
 #undef NODE_NAME_CASE
 }
 
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.

llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td

 class SwapHelper<dag Prefix, dag A, dag B, dag Suffix, bit swap> {
    dag Value = !con(Prefix, !if(swap, B, A), !if(swap, A, B), Suffix);
 }
 
 def SDTRVVVecReduce : SDTypeProfile<1, 2, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<0, 2>
 ]>;
 
-foreach kind = ["ADD", "UMAX", "SMAX", "UMIN", "SMIN", "AND", "OR", "XOR"] in
+foreach kind = ["ADD", "UMAX", "SMAX", "UMIN", "SMIN", "AND", "OR", "XOR",
+                "FADD", "SEQ_FADD"] in
   def rvv_vecreduce_#kind : SDNode<"RISCVISD::VECREDUCE_"#kind, SDTRVVVecReduce>;
 
 multiclass VPatUSLoadStoreSDNode<LLVMType type,
                                  LLVMType mask_type,
                                  int sew,
                                  LMULInfo vlmul,
                                  OutPatFrag avl,
                                  RegisterClass reg_rs1,
foreach vtiToWti = AllWidenableIntToFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar fwti = vtiToWti.Wti;
     def : Pat<(vti.Vector (vop (fwti.Vector fwti.RegClass:$rs1))),
               (!cast<Instruction>(instruction_name#"_"#vti.LMul.MX)
                   fwti.RegClass:$rs1, vti.AVL, vti.SEW)>;
   }
 }
 
-multiclass VPatReductionSDNode<SDNode vop, string instruction_name> {
-  foreach vti = AllIntegerVectors in {
-    defvar vti_m1 = !cast<VTypeInfo>("VI" # vti.SEW # "M1");
+multiclass VPatReductionSDNode<SDNode vop, string instruction_name, bit is_float> {
+  foreach vti = !if(is_float, AllFloatVectors, AllIntegerVectors) in {
+    defvar vti_m1 = !cast<VTypeInfo>(!if(is_float, "VF", "VI") # vti.SEW # "M1");
     def: Pat<(vti_m1.Vector (vop (vti.Vector vti.RegClass:$rs1), VR:$rs2)),
         (!cast<Instruction>(instruction_name#"_VS_"#vti.LMul.MX)
             (vti_m1.Vector (IMPLICIT_DEF)),
             (vti.Vector vti.RegClass:$rs1),
             (vti_m1.Vector VR:$rs2),
             vti.AVL, vti.SEW)>;
   }
 }
foreach vti = AllIntegerVectors in {
 
   def : Pat<(vti.Vector (vselect (vti.Mask VMV0:$vm), (SplatPat_simm5 simm5:$rs1),
                                                       vti.RegClass:$rs2)),
             (!cast<Instruction>("PseudoVMERGE_VIM_"#vti.LMul.MX)
                  vti.RegClass:$rs2, simm5:$rs1, VMV0:$vm, vti.AVL, vti.SEW)>;
 }
 
 // 15.1. Vector Single-Width Integer Reduction Instructions
-defm "" : VPatReductionSDNode<rvv_vecreduce_ADD,  "PseudoVREDSUM">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_UMAX, "PseudoVREDMAXU">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_SMAX, "PseudoVREDMAX">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_UMIN, "PseudoVREDMINU">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_SMIN, "PseudoVREDMIN">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_AND,  "PseudoVREDAND">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_OR,   "PseudoVREDOR">;
-defm "" : VPatReductionSDNode<rvv_vecreduce_XOR,  "PseudoVREDXOR">;
+defm "" : VPatReductionSDNode<rvv_vecreduce_ADD,  "PseudoVREDSUM", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_UMAX, "PseudoVREDMAXU", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_SMAX, "PseudoVREDMAX", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_UMIN, "PseudoVREDMINU", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_SMIN, "PseudoVREDMIN", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_AND,  "PseudoVREDAND", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_OR,   "PseudoVREDOR", /*is_float*/0>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_XOR,  "PseudoVREDXOR", /*is_float*/0>;
+
+// 15.3. Vector Single-Width Floating-Point Reduction Instructions
+defm "" : VPatReductionSDNode<rvv_vecreduce_SEQ_FADD, "PseudoVFREDOSUM", /*is_float*/1>;
+defm "" : VPatReductionSDNode<rvv_vecreduce_FADD,     "PseudoVFREDSUM", /*is_float*/1>;
 
 // 16.1. Vector Mask-Register Logical Instructions
 foreach mti = AllMasks in {
   def : Pat<(mti.Mask (and VR:$rs1, VR:$rs2)),
             (!cast<Instruction>("PseudoVMAND_MM_"#mti.LMul.MX)
                  VR:$rs1, VR:$rs2, mti.AVL, mti.SEW)>;
   def : Pat<(mti.Mask (or VR:$rs1, VR:$rs2)),
             (!cast<Instruction>("PseudoVMOR_MM_"#mti.LMul.MX)

llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp

 bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
   // Currently, the ExpandReductions pass can't expand scalable-vector
   // reductions, but we still request expansion as RVV doesn't support certain
   // reductions and the SelectionDAG can't legalize them either.
   switch (II->getIntrinsicID()) {
   default:
     return false;
   case Intrinsic::vector_reduce_mul:
-  case Intrinsic::vector_reduce_fadd:
   case Intrinsic::vector_reduce_fmul:
   case Intrinsic::vector_reduce_fmax:
   case Intrinsic::vector_reduce_fmin:
     return true;
   }
 }

llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+declare half @llvm.vector.reduce.fadd.nxv1f16(half, <vscale x 1 x half>)
+
+define half @vreduce_fadd_nxv1f16(<vscale x 1 x half> %v, half %s) {
+; RV32-LABEL: vreduce_fadd_nxv1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.h.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.h fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.h.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.h fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc half @llvm.vector.reduce.fadd.nxv1f16(half %s, <vscale x 1 x half> %v)
+  ret half %red
+}
+
+define half @vreduce_ord_fadd_nxv1f16(<vscale x 1 x half> %v, half %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call half @llvm.vector.reduce.fadd.nxv1f16(half %s, <vscale x 1 x half> %v)
+  ret half %red
+}
+
+declare half @llvm.vector.reduce.fadd.nxv2f16(half, <vscale x 2 x half>)
+
+define half @vreduce_fadd_nxv2f16(<vscale x 2 x half> %v, half %s) {
+; RV32-LABEL: vreduce_fadd_nxv2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.h.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.h fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.h.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.h fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc half @llvm.vector.reduce.fadd.nxv2f16(half %s, <vscale x 2 x half> %v)
+  ret half %red
+}
+
+define half @vreduce_ord_fadd_nxv2f16(<vscale x 2 x half> %v, half %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call half @llvm.vector.reduce.fadd.nxv2f16(half %s, <vscale x 2 x half> %v)
+  ret half %red
+}
+
+declare half @llvm.vector.reduce.fadd.nxv4f16(half, <vscale x 4 x half>)
+
+define half @vreduce_fadd_nxv4f16(<vscale x 4 x half> %v, half %s) {
+; RV32-LABEL: vreduce_fadd_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.h.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.h fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.h.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.h fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc half @llvm.vector.reduce.fadd.nxv4f16(half %s, <vscale x 4 x half> %v)
+  ret half %red
+}
+
+define half @vreduce_ord_fadd_nxv4f16(<vscale x 4 x half> %v, half %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call half @llvm.vector.reduce.fadd.nxv4f16(half %s, <vscale x 4 x half> %v)
+  ret half %red
+}
+
+declare float @llvm.vector.reduce.fadd.nxv1f32(float, <vscale x 1 x float>)
+
+define float @vreduce_fadd_nxv1f32(<vscale x 1 x float> %v, float %s) {
+; RV32-LABEL: vreduce_fadd_nxv1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.w.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.s fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.w.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.s fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc float @llvm.vector.reduce.fadd.nxv1f32(float %s, <vscale x 1 x float> %v)
+  ret float %red
+}
+
+define float @vreduce_ord_fadd_nxv1f32(<vscale x 1 x float> %v, float %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call float @llvm.vector.reduce.fadd.nxv1f32(float %s, <vscale x 1 x float> %v)
+  ret float %red
+}
+
+declare float @llvm.vector.reduce.fadd.nxv2f32(float, <vscale x 2 x float>)
+
+define float @vreduce_fadd_nxv2f32(<vscale x 2 x float> %v, float %s) {
+; RV32-LABEL: vreduce_fadd_nxv2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.w.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.s fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.w.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.s fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc float @llvm.vector.reduce.fadd.nxv2f32(float %s, <vscale x 2 x float> %v)
+  ret float %red
+}
+
+define float @vreduce_ord_fadd_nxv2f32(<vscale x 2 x float> %v, float %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call float @llvm.vector.reduce.fadd.nxv2f32(float %s, <vscale x 2 x float> %v)
+  ret float %red
+}
+
+declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
+
+define float @vreduce_fadd_nxv4f32(<vscale x 4 x float> %v, float %s) {
+; RV32-LABEL: vreduce_fadd_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fmv.w.x ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.s fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.w.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.s fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float %s, <vscale x 4 x float> %v)
+  ret float %red
+}
+
+define float @vreduce_ord_fadd_nxv4f32(<vscale x 4 x float> %v, float %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call float @llvm.vector.reduce.fadd.nxv4f32(float %s, <vscale x 4 x float> %v)
+  ret float %red
+}
+
+declare double @llvm.vector.reduce.fadd.nxv1f64(double, <vscale x 1 x double>)
+
+define double @vreduce_fadd_nxv1f64(<vscale x 1 x double> %v, double %s) {
+; RV32-LABEL: vreduce_fadd_nxv1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.d fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.d fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc double @llvm.vector.reduce.fadd.nxv1f64(double %s, <vscale x 1 x double> %v)
+  ret double %red
+}
+
+define double @vreduce_ord_fadd_nxv1f64(<vscale x 1 x double> %v, double %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call double @llvm.vector.reduce.fadd.nxv1f64(double %s, <vscale x 1 x double> %v)
+  ret double %red
+}
+
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
+
+define double @vreduce_fadd_nxv2f64(<vscale x 2 x double> %v, double %s) {
+; RV32-LABEL: vreduce_fadd_nxv2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.d fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.d fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc double @llvm.vector.reduce.fadd.nxv2f64(double %s, <vscale x 2 x double> %v)
+  ret double %red
+}
+
+define double @vreduce_ord_fadd_nxv2f64(<vscale x 2 x double> %v, double %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call double @llvm.vector.reduce.fadd.nxv2f64(double %s, <vscale x 2 x double> %v)
+  ret double %red
+}
+
+declare double @llvm.vector.reduce.fadd.nxv4f64(double, <vscale x 4 x double>)
+
+define double @vreduce_fadd_nxv4f64(<vscale x 4 x double> %v, double %s) {
+; RV32-LABEL: vreduce_fadd_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, ft0
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vfredsum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s ft0, v25
+; RV32-NEXT:    fadd.d fa0, fa0, ft0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_fadd_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, ft0
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vfredsum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s ft0, v25
+; RV64-NEXT:    fadd.d fa0, fa0, ft0
+; RV64-NEXT:    ret
+  %red = call reassoc double @llvm.vector.reduce.fadd.nxv4f64(double %s, <vscale x 4 x double> %v)
+  ret double %red
+}
+
+define double @vreduce_ord_fadd_nxv4f64(<vscale x 4 x double> %v, double %s) {
+; RV32-LABEL: vreduce_ord_fadd_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.s.f v25, fa0
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vfredosum.vs v25, v8, v25
+; RV32-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV32-NEXT:    vfmv.f.s fa0, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vreduce_ord_fadd_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.s.f v25, fa0
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vfredosum.vs v25, v8, v25
+; RV64-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
+; RV64-NEXT:    vfmv.f.s fa0, v25
+; RV64-NEXT:    ret
+  %red = call double @llvm.vector.reduce.fadd.nxv4f64(double %s, <vscale x 4 x double> %v)
+  ret double %red
+}