[AArch64 - SVE]: Use SVE to lower reduce.fadd.

[hassnaa-arm]

Summary

[AArch64 - SVE]: Use SVE to lower reduce.fadd.
Enable custom-lowering of NEON-like fixed-width vector versions of @llvm.vector.reduce.fadd to use SVE's fadda instruction, if SVE is enabled.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1384–1385

This is quite a limited set of types. I wonder if it actually supports VECREDUCE_SEQ_FADD for e.g. <8 x float> when the vector length is 256, or <8 x double> when the vector length is 512. Can you try this out?

Some other code in this file iterates through all possible FP/integer values and uses useSVEForFixedLengthVectorVT() to determine whether SVE should be used. This function has an option named OverrideNEON, which you can use to make it work for 128bit vectors.

llvm/test/CodeGen/AArch64/sve-fixed-length-fadd-reduce.ll

2 ↗

(On Diff #455261)

Could you also add some RUN lines for other vector lengths?

[dmgreen]

llvm/test/CodeGen/AArch64/sve-fixed-length-fadd-reduce.ll

65 ↗

(On Diff #455261)

This is just an fadd, and is probably simpler to leave as it was before.

[david-arm]

llvm/test/CodeGen/AArch64/sve-fixed-length-fadd-reduce.ll

1 ↗

(On Diff #455261)

Hi @hassnaa-arm, could you just reuse the existing fixed-length tests in CodeGen/AArch64/sve-fixed-length-fp-reduce.ll and just add another RUN line:

; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128

That test file already contains many of the tests added here.

2 ↗

(On Diff #455261)

I think if we reuse sve-fixed-length-fp-reduce.ll, then we get the other lengths for free?

[hassnaa-arm]

Changes of generated test code based on the new changes in codebase.

[david-arm]

LGTM!

llvm/include/llvm/Support/MachineValueType.h

enum SimpleValueType : uint8_t {
       v3f16          =  81,   //    3 x f16
       v4f16          =  82,   //    4 x f16
       v8f16          =  83,   //    8 x f16
       v16f16         =  84,   //   16 x f16
       v32f16         =  85,   //   32 x f16
       v64f16         =  86,   //   64 x f16
       v128f16        =  87,   //  128 x f16
       v256f16        =  88,   //  256 x f16
-      v512f16        =  89,   //  256 x f16
+      v512f16        =  89,   //  512 x f16
 
       v2bf16         =  90,   //    2 x bf16
       v3bf16         =  91,   //    3 x bf16
       v4bf16         =  92,   //    4 x bf16
       v8bf16         =  93,   //    8 x bf16
       v16bf16        =  94,   //   16 x bf16
       v32bf16        =  95,   //   32 x bf16
       v64bf16        =  96,   //   64 x bf16

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
       setOperationAction(ISD::SDIV, VT, Custom);
       setOperationAction(ISD::UDIV, VT, Custom);
     }
 
     // NEON doesn't support 64-bit vector integer muls, but SVE does.
     setOperationAction(ISD::MUL, MVT::v1i64, Custom);
     setOperationAction(ISD::MUL, MVT::v2i64, Custom);
 
+    // NEON doesn't support across-vector reductions, but SVE does.
+    for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v2f64})
+      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
+
     // NOTE: Currently this has to happen after computeRegisterProperties rather
     // than the preferred option of combining it with the addRegisterClass call.
     if (Subtarget->useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
           addTypeForFixedLengthSVE(VT);
       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
if (Subtarget->useSVEForFixedLengthVectors()) {
                       MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
         setOperationAction(ISD::BITREVERSE, VT, Custom);
         setOperationAction(ISD::CTTZ, VT, Custom);
         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
       }
 
-      // FP operations with no NEON support.
-      for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32,
-                      MVT::v1f64, MVT::v2f64})
-        setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
 
       // Use SVE for vectors with more than 2 elements.
       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
     }
 
     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-reduce.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
 ; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ;
 ; FADDA
 ;
 
 ; No single instruction NEON support. Use SVE.
 define half @fadda_v4f16(half %start, <4 x half> %a) vscale_range(1,0) #0 {
 ; CHECK-LABEL: fadda_v4f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mov h2, v1.h[1]
-; CHECK-NEXT:    fadd h0, h0, h1
-; CHECK-NEXT:    mov h3, v1.h[2]
-; CHECK-NEXT:    mov h1, v1.h[3]
-; CHECK-NEXT:    fadd h0, h0, h2
-; CHECK-NEXT:    fadd h0, h0, h3
-; CHECK-NEXT:    fadd h0, h0, h1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    fadda h0, p0, h0, z1.h
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v4f16(half %start, <4 x half> %a)
   ret half %res
 }
 
 ; No single instruction NEON support. Use SVE.
 define half @fadda_v8f16(half %start, <8 x half> %a) vscale_range(1,0) #0 {
 ; CHECK-LABEL: fadda_v8f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov h2, v1.h[1]
-; CHECK-NEXT:    fadd h0, h0, h1
-; CHECK-NEXT:    mov h3, v1.h[2]
-; CHECK-NEXT:    fadd h0, h0, h2
-; CHECK-NEXT:    mov h2, v1.h[3]
-; CHECK-NEXT:    fadd h0, h0, h3
-; CHECK-NEXT:    mov h3, v1.h[4]
-; CHECK-NEXT:    fadd h0, h0, h2
-; CHECK-NEXT:    mov h2, v1.h[5]
-; CHECK-NEXT:    fadd h0, h0, h3
-; CHECK-NEXT:    mov h3, v1.h[6]
-; CHECK-NEXT:    mov h1, v1.h[7]
-; CHECK-NEXT:    fadd h0, h0, h2
-; CHECK-NEXT:    fadd h0, h0, h3
-; CHECK-NEXT:    fadd h0, h0, h1
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    fadda h0, p0, h0, z1.h
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v8f16(half %start, <8 x half> %a)
   ret half %res
 }
 
 define half @fadda_v16f16(half %start, <16 x half>* %a) vscale_range(2,0) #0 {
 ; CHECK-LABEL: fadda_v16f16:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v128f16(half %start, <128 x half> %op)
   ret half %res
 }
 
 ; No single instruction NEON support. Use SVE.
 define float @fadda_v2f32(float %start, <2 x float> %a) vscale_range(1,0) #0 {
 ; CHECK-LABEL: fadda_v2f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    fadd s0, s0, s2
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    fadda s0, p0, s0, z1.s
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call float @llvm.vector.reduce.fadd.v2f32(float %start, <2 x float> %a)
   ret float %res
 }
 
 ; No single instruction NEON support. Use SVE.
 define float @fadda_v4f32(float %start, <4 x float> %a) vscale_range(1,0) #0 {
 ; CHECK-LABEL: fadda_v4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s3, v1.s[2]
-; CHECK-NEXT:    mov s1, v1.s[3]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s0, s3
-; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    fadda s0, p0, s0, z1.s
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %a)
   ret float %res
 }
 
 define float @fadda_v8f32(float %start, <8 x float>* %a) vscale_range(2,0) #0 {
 ; CHECK-LABEL: fadda_v8f32:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   %res = call double @llvm.vector.reduce.fadd.v1f64(double %start, <1 x double> %a)
   ret double %res
 }
 
 ; No single instruction NEON support. Use SVE.
 define double @fadda_v2f64(double %start, <2 x double> %a) vscale_range(1,0) #0 {
 ; CHECK-LABEL: fadda_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d2, v1.d[1]
-; CHECK-NEXT:    fadd d0, d0, d1
-; CHECK-NEXT:    fadd d0, d0, d2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    fadda d0, p0, d0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call double @llvm.vector.reduce.fadd.v2f64(double %start, <2 x double> %a)
   ret double %res
 }
 
 define double @fadda_v4f64(double %start, <4 x double>* %a) vscale_range(2,0) #0 {
 ; CHECK-LABEL: fadda_v4f64:
 ; CHECK:       // %bb.0: