[AArch64] Allow folding FMUL/FADD into FMA for FP16 types

[bsmith]

Summary

isFMAFasterThanFMulAndFAdd should return true for FP16 types when
HasFullFP16 is present, since we have the instructions to handle it for
both SVE and NEON.

[dmgreen]

LGTM. Thanks for the change

[bsmith]

• Update tests in line with changes to D96424

[bsmith]

• Reverse order of dependant patches, moving all SVE tests into D96424

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
     const MachineFunction &MF, EVT VT) const {
   VT = VT.getScalarType();
 
   if (!VT.isSimple())
     return false;
 
   switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::f16:
+    return Subtarget->hasFullFP16();
   case MVT::f32:
   case MVT::f64:
     return true;
   default:
     break;
   }
 
   return false;

llvm/test/CodeGen/AArch64/f16-instructions.ll

 ; CHECK-FP16-NEXT: fmul h0, h0, h1
 ; CHECK-FP16-NEXT: ret
 
 define half @test_fmul(half %a, half %b) #0 {
   %r = fmul half %a, %b
   ret half %r
 }
 
+; CHECK-CVT-LABEL: test_fmadd:
+; CHECK-CVT-NEXT: fcvt s1, h1
+; CHECK-CVT-NEXT: fcvt s0, h0
+; CHECK-CVT-NEXT: fmul s0, s0, s1
+; CHECK-CVT-NEXT: fcvt h0, s0
+; CHECK-CVT-NEXT: fcvt s0, h0
+; CHECK-CVT-NEXT: fcvt s1, h2
+; CHECK-CVT-NEXT: fadd s0, s0, s1
+; CHECK-CVT-NEXT: fcvt h0, s0
+; CHECK-CVT-NEXT: ret
+
+; CHECK-FP16-LABEL: test_fmadd:
+; CHECK-FP16-NEXT: fmadd h0, h0, h1, h2
+; CHECK-FP16-NEXT: ret
+
+define half @test_fmadd(half %a, half %b, half %c) #0 {
+  %mul = fmul fast half %a, %b
+  %r = fadd fast half %mul, %c
+  ret half %r
+}
 ; CHECK-CVT-LABEL: test_fdiv:
 ; CHECK-CVT-NEXT: fcvt s1, h1
 ; CHECK-CVT-NEXT: fcvt s0, h0
 ; CHECK-CVT-NEXT: fdiv s0, s0, s1
 ; CHECK-CVT-NEXT: fcvt h0, s0
 ; CHECK-CVT-NEXT: ret
 
 ; CHECK-FP16-LABEL: test_fdiv:

 ; CHECK-CVT-NEXT: fcvt h0, s0
 ; CHECK-CVT-NEXT: fcvt s0, h0
 ; CHECK-CVT-NEXT: fcvt s1, h2
 ; CHECK-CVT-NEXT: fadd s0, s0, s1
 ; CHECK-CVT-NEXT: fcvt h0, s0
 ; CHECK-CVT-NEXT: ret
 
 ; CHECK-FP16-LABEL: test_fmuladd:
-; CHECK-FP16-NEXT: fmul h0, h0, h1
-; CHECK-FP16-NEXT: fadd h0, h0, h2
+; CHECK-FP16-NEXT: fmadd h0, h0, h1, h2
 ; CHECK-FP16-NEXT: ret
 
 define half @test_fmuladd(half %a, half %b, half %c) #0 {
   %r = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
   ret half %r
 }
 
 ; CHECK-FP16-LABEL: test_vrecpeh_f16:

llvm/test/CodeGen/AArch64/sve-fp-combine.ll

 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
 
+define <vscale x 8 x half> @fmla_h_sel(<vscale x 8 x i1> %pred, <vscale x 8 x half> %acc,
+                               <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fmla_h_sel:
+; CHECK: fmla z0.h, p0/m, z1.h, z2.h
+; CHECK: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %add = fadd fast <vscale x 8 x half> %acc, %mul
+  %res = select <vscale x 8 x i1> %pred, <vscale x 8 x half> %add, <vscale x 8 x half> %acc
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fmla_s_sel(<vscale x 4 x i1> %pred, <vscale x 4 x float> %acc,
                                <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fmla_s_sel:
 ; CHECK: fmla z0.s, p0/m, z1.s, z2.s
 ; CHECK: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %add = fadd fast <vscale x 4 x float> %acc, %mul
   %res = select <vscale x 4 x i1> %pred, <vscale x 4 x float> %add, <vscale x 4 x float> %acc

 ; CHECK: fmla z0.d, p0/m, z1.d, z2.d
 ; CHECK: ret
   %mul = fmul fast <vscale x 2 x double> %m1, %m2
   %add = fadd fast <vscale x 2 x double> %acc, %mul
   %res = select <vscale x 2 x i1> %pred, <vscale x 2 x double> %add, <vscale x 2 x double> %acc
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fmls_h_sel(<vscale x 8 x i1> %pred, <vscale x 8 x half> %acc,
+                               <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fmls_h_sel:
+; CHECK: fmls z0.h, p0/m, z1.h, z2.h
+; CHECK: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %sub = fsub fast <vscale x 8 x half> %acc, %mul
+  %res = select <vscale x 8 x i1> %pred, <vscale x 8 x half> %sub, <vscale x 8 x half> %acc
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fmls_s_sel(<vscale x 4 x i1> %pred, <vscale x 4 x float> %acc,
                                <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fmls_s_sel:
 ; CHECK: fmls z0.s, p0/m, z1.s, z2.s
 ; CHECK: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %sub = fsub fast <vscale x 4 x float> %acc, %mul
   %res = select <vscale x 4 x i1> %pred, <vscale x 4 x float> %sub, <vscale x 4 x float> %acc

 ; CHECK: fmls z0.d, p0/m, z1.d, z2.d
 ; CHECK: ret
   %mul = fmul fast <vscale x 2 x double> %m1, %m2
   %sub = fsub fast <vscale x 2 x double> %acc, %mul
   %res = select <vscale x 2 x i1> %pred, <vscale x 2 x double> %sub, <vscale x 2 x double> %acc
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fmad_h(<vscale x 8 x half> %m1, <vscale x 8 x half> %m2, <vscale x 8 x half> %acc) {
+; CHECK-LABEL: fmad_h:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: fmad z0.h, [[PG]]/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %res = fadd fast <vscale x 8 x half> %acc, %mul
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fmad_s(<vscale x 4 x float> %m1, <vscale x 4 x float> %m2, <vscale x 4 x float> %acc) {
 ; CHECK-LABEL: fmad_s:
 ; CHECK: ptrue [[PG:p[0-9]+]].s
 ; CHECK-NEXT: fmad z0.s, [[PG]]/m, z1.s, z2.s
 ; CHECK-NEXT: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %res = fadd fast <vscale x 4 x float> %acc, %mul
   ret <vscale x 4 x float> %res

 ; CHECK: ptrue [[PG:p[0-9]+]].d
 ; CHECK-NEXT: fmla z0.d, [[PG]]/m, z1.d, z2.d
 ; CHECK-NEXT: ret
   %mul = fmul fast <vscale x 2 x double> %m1, %m2
   %res = fadd fast <vscale x 2 x double> %acc, %mul
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fmls_h(<vscale x 8 x half> %acc, <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fmls_h:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: fmls z0.h, [[PG]]/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %res = fsub fast <vscale x 8 x half> %acc, %mul
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fmls_s(<vscale x 4 x float> %acc, <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fmls_s:
 ; CHECK: ptrue [[PG:p[0-9]+]].s
 ; CHECK-NEXT: fmls z0.s, [[PG]]/m, z1.s, z2.s
 ; CHECK-NEXT: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %res = fsub fast <vscale x 4 x float> %acc, %mul
   ret <vscale x 4 x float> %res
%neg_m1 = fsub fast <vscale x 2 x double>
                  <vscale x 2 x i32> zeroinitializer),
                %m1
 
   %mul = fmul fast <vscale x 2 x double> %neg_m1, %m2
   %res = fsub fast <vscale x 2 x double> %mul, %acc
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fnmla_h(<vscale x 8 x half> %acc, <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fnmla_h:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: fnmla z0.h, [[PG]]/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %neg_m1 = fsub fast <vscale x 8 x half>
+               shufflevector (<vscale x 8 x half>
+                 insertelement (<vscale x 8 x half> undef, half -0.000000e+00, i32 0),
+                 <vscale x 8 x half> undef,
+                 <vscale x 8 x i32> zeroinitializer),
+               %m1
+
+  %mul = fmul fast <vscale x 8 x half> %neg_m1, %m2
+  %res = fsub fast <vscale x 8 x half> %mul, %acc
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fnmla_s(<vscale x 4 x float> %acc, <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fnmla_s:
 ; CHECK: ptrue [[PG:p[0-9]+]].s
 ; CHECK-NEXT: fnmla z0.s, [[PG]]/m, z1.s, z2.s
 ; CHECK-NEXT: ret
   %neg_m1 = fsub fast <vscale x 4 x float>
                shufflevector (<vscale x 4 x float>
                  insertelement (<vscale x 4 x float> undef, float -0.000000e+00, i32 0),
%neg_m1 = fsub fast <vscale x 2 x double>
                  <vscale x 2 x i32> zeroinitializer),
                %m1
 
   %mul = fmul fast <vscale x 2 x double> %neg_m1, %m2
   %res = fsub fast <vscale x 2 x double> %mul, %acc
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fnmla_h_reversed(<vscale x 8 x half> %acc, <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fnmla_h_reversed:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: fnmla z0.h, [[PG]]/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %add = fadd fast <vscale x 8 x half> %mul, %acc
+  %res = fsub fast <vscale x 8 x half>
+               shufflevector (<vscale x 8 x half>
+                 insertelement (<vscale x 8 x half> undef, half -0.000000e+00, i32 0),
+                 <vscale x 8 x half> undef,
+                 <vscale x 8 x i32> zeroinitializer),
+               %add
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fnmla_s_reversed(<vscale x 4 x float> %acc, <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fnmla_s_reversed:
 ; CHECK: ptrue [[PG:p[0-9]+]].s
 ; CHECK-NEXT: fnmla z0.s, [[PG]]/m, z1.s, z2.s
 ; CHECK-NEXT: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %add = fadd fast <vscale x 4 x float> %mul, %acc
   %res = fsub fast <vscale x 4 x float>
%res = fsub fast <vscale x 2 x double>
                shufflevector (<vscale x 2 x double>
                  insertelement (<vscale x 2 x double> undef, double -0.000000e+00, i32 0),
                  <vscale x 2 x double> undef,
                  <vscale x 2 x i32> zeroinitializer),
                %add
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 8 x half> @fnmls_h(<vscale x 8 x half> %acc, <vscale x 8 x half> %m1, <vscale x 8 x half> %m2) {
+; CHECK-LABEL: fnmls_h:
+; CHECK: ptrue [[PG:p[0-9]+]].h
+; CHECK-NEXT: fnmls z0.h, [[PG]]/m, z1.h, z2.h
+; CHECK-NEXT: ret
+  %mul = fmul fast <vscale x 8 x half> %m1, %m2
+  %res = fsub fast <vscale x 8 x half> %mul, %acc
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 4 x float> @fnmls_s(<vscale x 4 x float> %acc, <vscale x 4 x float> %m1, <vscale x 4 x float> %m2) {
 ; CHECK-LABEL: fnmls_s:
 ; CHECK: ptrue [[PG:p[0-9]+]].s
 ; CHECK-NEXT: fnmls z0.s, [[PG]]/m, z1.s, z2.s
 ; CHECK-NEXT: ret
   %mul = fmul fast <vscale x 4 x float> %m1, %m2
   %res = fsub fast <vscale x 4 x float> %mul, %acc
   ret <vscale x 4 x float> %res