[AArch64] Mark vecreduce_fminimum and vecreduce_fmaximum as legal

[dmgreen]

Summary

This adds some simple lowering of vecreduce_fminimum and vecreduce_fmaximum to fminv/fmaxv instructions, in the same way that vecreduce_fmax / vecreduce_fmin is lowered to fminnmv / fmaxnmv.

[SjoerdMeijer]

Looks like a good idea to me.

llvm/include/llvm/Target/TargetSelectionDAG.td

 def vecreduce_add  : SDNode<"ISD::VECREDUCE_ADD", SDTVecReduce>;
 def vecreduce_smax  : SDNode<"ISD::VECREDUCE_SMAX", SDTVecReduce>;
 def vecreduce_umax  : SDNode<"ISD::VECREDUCE_UMAX", SDTVecReduce>;
 def vecreduce_smin  : SDNode<"ISD::VECREDUCE_SMIN", SDTVecReduce>;
 def vecreduce_umin  : SDNode<"ISD::VECREDUCE_UMIN", SDTVecReduce>;
 def vecreduce_fadd  : SDNode<"ISD::VECREDUCE_FADD", SDTFPVecReduce>;
 def vecreduce_fmin  : SDNode<"ISD::VECREDUCE_FMIN", SDTFPVecReduce>;
 def vecreduce_fmax  : SDNode<"ISD::VECREDUCE_FMAX", SDTFPVecReduce>;
+def vecreduce_fminimum : SDNode<"ISD::VECREDUCE_FMINIMUM", SDTFPVecReduce>;
+def vecreduce_fmaximum : SDNode<"ISD::VECREDUCE_FMAXIMUM", SDTFPVecReduce>;
 
 def fadd       : SDNode<"ISD::FADD"       , SDTFPBinOp, [SDNPCommutative]>;
 def fsub       : SDNode<"ISD::FSUB"       , SDTFPBinOp>;
 def fmul       : SDNode<"ISD::FMUL"       , SDTFPBinOp, [SDNPCommutative]>;
 def fdiv       : SDNode<"ISD::FDIV"       , SDTFPBinOp>;
 def frem       : SDNode<"ISD::FREM"       , SDTFPBinOp>;
 def fma        : SDNode<"ISD::FMA"        , SDTFPTernaryOp, [SDNPCommutative]>;
 def fmad       : SDNode<"ISD::FMAD"       , SDTFPTernaryOp, [SDNPCommutative]>;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasNEON()) {
     }
 
     // Vector reductions
     for (MVT VT : { MVT::v4f16, MVT::v2f32,
                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
       if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
         setOperationAction(ISD::VECREDUCE_FMAX, VT, Legal);
         setOperationAction(ISD::VECREDUCE_FMIN, VT, Legal);
+        setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Legal);
+        setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Legal);
 
         setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
       }
     }
     for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
                     MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def AArch64facgt     : PatFrags<(ops node:$Rn, node:$Rm),
                                  (int_aarch64_neon_facgt node:$Rn, node:$Rm)]>;
 
 def AArch64fmaxnmv : PatFrags<(ops node:$Rn),
                               [(vecreduce_fmax node:$Rn),
                                (int_aarch64_neon_fmaxnmv node:$Rn)]>;
 def AArch64fminnmv : PatFrags<(ops node:$Rn),
                               [(vecreduce_fmin node:$Rn),
                                (int_aarch64_neon_fminnmv node:$Rn)]>;
+def AArch64fmaxv : PatFrags<(ops node:$Rn),
+                            [(vecreduce_fmaximum node:$Rn),
+                             (int_aarch64_neon_fmaxv node:$Rn)]>;
+def AArch64fminv : PatFrags<(ops node:$Rn),
+                            [(vecreduce_fminimum node:$Rn),
+                             (int_aarch64_neon_fminv node:$Rn)]>;
 
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
 def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stzg : SDNode<"AArch64ISD::STZG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64st2g : SDNode<"AArch64ISD::ST2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stz2g : SDNode<"AArch64ISD::STZ2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
 def SDT_AArch64unpk : SDTypeProfile<1, 1, [

 def : Pat<(f32 (int_aarch64_neon_faddv (v4f32 V128:$Rn))),
           (FADDPv2i32p (EXTRACT_SUBREG (FADDPv4f32 V128:$Rn, V128:$Rn), dsub))>;
 def : Pat<(f64 (int_aarch64_neon_faddv (v2f64 V128:$Rn))),
           (FADDPv2i64p V128:$Rn)>;
 def : Pat<(f32 (AArch64fmaxnmv (v2f32 V64:$Rn))),
           (FMAXNMPv2i32p V64:$Rn)>;
 def : Pat<(f64 (AArch64fmaxnmv (v2f64 V128:$Rn))),
           (FMAXNMPv2i64p V128:$Rn)>;
-def : Pat<(f32 (int_aarch64_neon_fmaxv (v2f32 V64:$Rn))),
+def : Pat<(f32 (AArch64fmaxv (v2f32 V64:$Rn))),
           (FMAXPv2i32p V64:$Rn)>;
-def : Pat<(f64 (int_aarch64_neon_fmaxv (v2f64 V128:$Rn))),
+def : Pat<(f64 (AArch64fmaxv (v2f64 V128:$Rn))),
           (FMAXPv2i64p V128:$Rn)>;
 def : Pat<(f32 (AArch64fminnmv (v2f32 V64:$Rn))),
           (FMINNMPv2i32p V64:$Rn)>;
 def : Pat<(f64 (AArch64fminnmv (v2f64 V128:$Rn))),
           (FMINNMPv2i64p V128:$Rn)>;
-def : Pat<(f32 (int_aarch64_neon_fminv (v2f32 V64:$Rn))),
+def : Pat<(f32 (AArch64fminv (v2f32 V64:$Rn))),
           (FMINPv2i32p V64:$Rn)>;
-def : Pat<(f64 (int_aarch64_neon_fminv (v2f64 V128:$Rn))),
+def : Pat<(f64 (AArch64fminv (v2f64 V128:$Rn))),
           (FMINPv2i64p V128:$Rn)>;
 
 //----------------------------------------------------------------------------
 // AdvSIMD INS/DUP instructions
 //----------------------------------------------------------------------------
 
 def DUPv8i8gpr  : SIMDDupFromMain<0, {?,?,?,?,1}, ".8b", v8i8, V64, GPR32>;
 def DUPv16i8gpr : SIMDDupFromMain<1, {?,?,?,?,1}, ".16b", v16i8, V128, GPR32>;

 defm ADDV    : SIMDAcrossLanesBHS<0, 0b11011, "addv">;
 defm SMAXV   : SIMDAcrossLanesBHS<0, 0b01010, "smaxv">;
 defm SMINV   : SIMDAcrossLanesBHS<0, 0b11010, "sminv">;
 defm UMAXV   : SIMDAcrossLanesBHS<1, 0b01010, "umaxv">;
 defm UMINV   : SIMDAcrossLanesBHS<1, 0b11010, "uminv">;
 defm SADDLV  : SIMDAcrossLanesHSD<0, 0b00011, "saddlv">;
 defm UADDLV  : SIMDAcrossLanesHSD<1, 0b00011, "uaddlv">;
 defm FMAXNMV : SIMDFPAcrossLanes<0b01100, 0, "fmaxnmv", AArch64fmaxnmv>;
-defm FMAXV   : SIMDFPAcrossLanes<0b01111, 0, "fmaxv", int_aarch64_neon_fmaxv>;
+defm FMAXV   : SIMDFPAcrossLanes<0b01111, 0, "fmaxv", AArch64fmaxv>;
 defm FMINNMV : SIMDFPAcrossLanes<0b01100, 1, "fminnmv", AArch64fminnmv>;
-defm FMINV   : SIMDFPAcrossLanes<0b01111, 1, "fminv", int_aarch64_neon_fminv>;
+defm FMINV   : SIMDFPAcrossLanes<0b01111, 1, "fminv", AArch64fminv>;
 
 multiclass SIMDAcrossLaneLongPairIntrinsic<string Opc, SDPatternOperator addlp> {
   // Patterns for addv(addlp(x)) ==> addlv
   def : Pat<(i32 (vector_extract (v8i16 (insert_subvector undef,
               (v4i16 (AArch64uaddv (v4i16 (addlp (v8i8 V64:$op))))),
               (i64 0))), (i64 0))),
             (EXTRACT_SUBREG (INSERT_SUBREG (v4i16 (IMPLICIT_DEF)),
               (!cast<Instruction>(Opc#"v8i8v") V64:$op), hsub), ssub)>;

llvm/test/CodeGen/AArch64/vecreduce-fmaximum.ll

 ; CHECK-NOFP-NEXT:    fcvt h1, s1
 ; CHECK-NOFP-NEXT:    fcvt s1, h1
 ; CHECK-NOFP-NEXT:    fmax s0, s1, s0
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    ret
 ;
 ; CHECK-FP-LABEL: test_v4f16:
 ; CHECK-FP:       // %bb.0:
-; CHECK-FP-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-FP-NEXT:    mov h1, v0.h[1]
-; CHECK-FP-NEXT:    mov h2, v0.h[2]
-; CHECK-FP-NEXT:    fmax h1, h0, h1
-; CHECK-FP-NEXT:    mov h0, v0.h[3]
-; CHECK-FP-NEXT:    fmax h1, h1, h2
-; CHECK-FP-NEXT:    fmax h0, h1, h0
+; CHECK-FP-NEXT:    fmaxv h0, v0.4h
 ; CHECK-FP-NEXT:    ret
   %b = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> %a)
   ret half %b
 }
 
 define half @test_v11f16(<11 x half> %a) nounwind {
 ; CHECK-NOFP-LABEL: test_v11f16:
 ; CHECK-NOFP:       // %bb.0:

 ; CHECK-FP-NEXT:    ld1 { v1.h }[1], [x8]
 ; CHECK-FP-NEXT:    add x8, sp, #16
 ; CHECK-FP-NEXT:    mov v0.h[4], v4.h[0]
 ; CHECK-FP-NEXT:    ld1 { v1.h }[2], [x8]
 ; CHECK-FP-NEXT:    mov v0.h[5], v5.h[0]
 ; CHECK-FP-NEXT:    mov v0.h[6], v6.h[0]
 ; CHECK-FP-NEXT:    mov v0.h[7], v7.h[0]
 ; CHECK-FP-NEXT:    fmax v0.8h, v0.8h, v1.8h
-; CHECK-FP-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-FP-NEXT:    fmax v0.4h, v0.4h, v1.4h
-; CHECK-FP-NEXT:    mov h1, v0.h[1]
-; CHECK-FP-NEXT:    mov h2, v0.h[2]
-; CHECK-FP-NEXT:    fmax h1, h0, h1
-; CHECK-FP-NEXT:    mov h0, v0.h[3]
-; CHECK-FP-NEXT:    fmax h1, h1, h2
-; CHECK-FP-NEXT:    fmax h0, h1, h0
+; CHECK-FP-NEXT:    fmaxv h0, v0.8h
 ; CHECK-FP-NEXT:    ret
   %b = call half @llvm.vector.reduce.fmaximum.v11f16(<11 x half> %a)
   ret half %b
 }
 
 ; Neutral element is negative infinity which is chosen for padding the widened
 ; vector.
 define float @test_v3f32(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #-8388608 // =0xff800000
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmax s0, s0, s1
+; CHECK-NEXT:    fmaxv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call float @llvm.vector.reduce.fmaximum.v3f32(<3 x float> %a)
   ret float %b
 }
 
 ; Neutral element chosen for padding the widened vector is not negative infinity.
 define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32_ninf:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #-8388609 // =0xff7fffff
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmax s0, s0, s1
+; CHECK-NEXT:    fmaxv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call ninf float @llvm.vector.reduce.fmaximum.v3f32(<3 x float> %a)
   ret float %b
 }
 
 ; Cannot legalize f128. See PR63267 - The underlying fmaximum has no default
 ; expansion and no libcalls.
 ;define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
 ;  %b = call fp128 @llvm.vector.reduce.fmaximum.v2f128(<2 x fp128> %a)
 ;  ret fp128 %b
 ;}
 
 define float @test_v16f32(<16 x float> %a) nounwind {
 ; CHECK-LABEL: test_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    fmax v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    fmax v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmax s0, s0, s1
+; CHECK-NEXT:    fmaxv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> %a)
   ret float %b
 }
 
 define double @test_v2f64(<2 x double> %a) nounwind {
 ; CHECK-LABEL: test_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    fmax d0, d0, d1
+; CHECK-NEXT:    fmaxp d0, v0.2d
 ; CHECK-NEXT:    ret
   %b = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> %a)
   ret double %b
 }

llvm/test/CodeGen/AArch64/vecreduce-fminimum.ll

 ; CHECK-NOFP-NEXT:    fcvt h1, s1
 ; CHECK-NOFP-NEXT:    fcvt s1, h1
 ; CHECK-NOFP-NEXT:    fmin s0, s1, s0
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    ret
 ;
 ; CHECK-FP-LABEL: test_v4f16:
 ; CHECK-FP:       // %bb.0:
-; CHECK-FP-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-FP-NEXT:    mov h1, v0.h[1]
-; CHECK-FP-NEXT:    mov h2, v0.h[2]
-; CHECK-FP-NEXT:    fmin h1, h0, h1
-; CHECK-FP-NEXT:    mov h0, v0.h[3]
-; CHECK-FP-NEXT:    fmin h1, h1, h2
-; CHECK-FP-NEXT:    fmin h0, h1, h0
+; CHECK-FP-NEXT:    fminv h0, v0.4h
 ; CHECK-FP-NEXT:    ret
   %b = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> %a)
   ret half %b
 }
 
 define half @test_v11f16(<11 x half> %a) nounwind {
 ; CHECK-NOFP-LABEL: test_v11f16:
 ; CHECK-NOFP:       // %bb.0:

 ; CHECK-FP-NEXT:    ld1 { v1.h }[1], [x8]
 ; CHECK-FP-NEXT:    add x8, sp, #16
 ; CHECK-FP-NEXT:    mov v0.h[4], v4.h[0]
 ; CHECK-FP-NEXT:    ld1 { v1.h }[2], [x8]
 ; CHECK-FP-NEXT:    mov v0.h[5], v5.h[0]
 ; CHECK-FP-NEXT:    mov v0.h[6], v6.h[0]
 ; CHECK-FP-NEXT:    mov v0.h[7], v7.h[0]
 ; CHECK-FP-NEXT:    fmin v0.8h, v0.8h, v1.8h
-; CHECK-FP-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-FP-NEXT:    fmin v0.4h, v0.4h, v1.4h
-; CHECK-FP-NEXT:    mov h1, v0.h[1]
-; CHECK-FP-NEXT:    mov h2, v0.h[2]
-; CHECK-FP-NEXT:    fmin h1, h0, h1
-; CHECK-FP-NEXT:    mov h0, v0.h[3]
-; CHECK-FP-NEXT:    fmin h1, h1, h2
-; CHECK-FP-NEXT:    fmin h0, h1, h0
+; CHECK-FP-NEXT:    fminv h0, v0.8h
 ; CHECK-FP-NEXT:    ret
   %b = call half @llvm.vector.reduce.fminimum.v11f16(<11 x half> %a)
   ret half %b
 }
 
 ; Neutral element is negative infinity which is chosen for padding the widened
 ; vector.
 define float @test_v3f32(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2139095040 // =0x7f800000
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    fminv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call float @llvm.vector.reduce.fminimum.v3f32(<3 x float> %a)
   ret float %b
 }
 
 ; Neutral element chosen for padding the widened vector is not negative infinity.
 define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32_ninf:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2139095039 // =0x7f7fffff
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    fminv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call ninf float @llvm.vector.reduce.fminimum.v3f32(<3 x float> %a)
   ret float %b
 }
 
 ; Cannot legalize f128. See PR63267 - The underlying fminimum has no default
 ; expansion and no libcalls.
 ;define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
 ;  %b = call fp128 @llvm.vector.reduce.fminimum.v2f128(<2 x fp128> %a)
 ;  ret fp128 %b
 ;}
 
 define float @test_v16f32(<16 x float> %a) nounwind {
 ; CHECK-LABEL: test_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    fmin v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    fmin v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    fminv s0, v0.4s
 ; CHECK-NEXT:    ret
   %b = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> %a)
   ret float %b
 }
 
 define double @test_v2f64(<2 x double> %a) nounwind {
 ; CHECK-LABEL: test_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    fmin d0, d0, d1
+; CHECK-NEXT:    fminp d0, v0.2d
 ; CHECK-NEXT:    ret
   %b = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> %a)
   ret double %b
 }