[ARM] Fill in FP16 FMA patterns

[dmgreen]

Summary

This adds fp16 variants of all the fma patterns in the ARM backend.

[samparker]

llvm/lib/Target/ARM/ARMInstrVFP.td

2215

Do you think it would be worth doing some canonicalisation somewhere? With two fnegs I'm assuming this pattern is more expensive than the others if not caught.

[dmgreen]

llvm/lib/Target/ARM/ARMInstrVFP.td

2215

I gave this a go, but it appears that the opposite is sometimes true. AMDGPU has instructions that look like v_fma_f32 v0, -v0, v1, -v1, where each of the operands can be inverted for free. If we inverted the whole thing, then we would need to add reverse patterns. RISCV seems to have some cases with fnmadd.s that are made worse too.

[samparker]

Ok, fair enough, thanks for trying.

llvm/lib/Target/ARM/ARMInstrVFP.td

 // Match @llvm.fma.* intrinsics
 // (fma (fneg x), y, z) -> (vfms z, x, y)
 def : Pat<(f64 (fma (fneg DPR:$Dn), DPR:$Dm, DPR:$Ddin)),
           (VFMSD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(f32 (fma (fneg SPR:$Sn), SPR:$Sm, SPR:$Sdin)),
           (VFMSS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(f16 (fma (fneg HPR:$Sn), HPR:$Sm, HPR:$Sdin)),
+          (VFMSH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 // (fma x, (fneg y), z) -> (vfms z, x, y)
 def : Pat<(f64 (fma DPR:$Dn, (fneg DPR:$Dm), DPR:$Ddin)),
           (VFMSD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(f32 (fma SPR:$Sn, (fneg SPR:$Sm), SPR:$Sdin)),
           (VFMSS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(f16 (fma HPR:$Sn, (fneg HPR:$Sm), HPR:$Sdin)),
+          (VFMSH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 
 def VFNMAD : ADbI<0b11101, 0b01, 1, 0,
                   (outs DPR:$Dd), (ins DPR:$Ddin, DPR:$Dn, DPR:$Dm),
                   IIC_fpFMAC64, "vfnma", ".f64\t$Dd, $Dn, $Dm",
                   [(set DPR:$Dd,(fsub_mlx (fneg (fmul_su DPR:$Dn,DPR:$Dm)),
                                           (f64 DPR:$Ddin)))]>,
                 RegConstraint<"$Ddin = $Dd">,
                 Requires<[HasVFP4,HasDPVFP,UseFusedMAC]>,

 // Match @llvm.fma.* intrinsics
 // (fneg (fma x, y, z)) -> (vfnma z, x, y)
 def : Pat<(fneg (fma (f64 DPR:$Dn), (f64 DPR:$Dm), (f64 DPR:$Ddin))),
           (VFNMAD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(fneg (fma (f32 SPR:$Sn), (f32 SPR:$Sm), (f32 SPR:$Sdin))),
           (VFNMAS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(fneg (fma (f16 HPR:$Sn), (f16 HPR:$Sm), (f16 HPR:$Sdin))),
+          (VFNMAH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 // (fma (fneg x), y, (fneg z)) -> (vfnma z, x, y)
 def : Pat<(f64 (fma (fneg DPR:$Dn), DPR:$Dm, (fneg DPR:$Ddin))),
           (VFNMAD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(f32 (fma (fneg SPR:$Sn), SPR:$Sm, (fneg SPR:$Sdin))),
           (VFNMAS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(f16 (fma (fneg HPR:$Sn), HPR:$Sm, (fneg HPR:$Sdin))),
+          (VFNMAH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 
 def VFNMSD : ADbI<0b11101, 0b01, 0, 0,
                   (outs DPR:$Dd), (ins DPR:$Ddin, DPR:$Dn, DPR:$Dm),
                   IIC_fpFMAC64, "vfnms", ".f64\t$Dd, $Dn, $Dm",
                   [(set DPR:$Dd, (fsub_mlx (fmul_su DPR:$Dn, DPR:$Dm),
                                            (f64 DPR:$Ddin)))]>,
                RegConstraint<"$Ddin = $Dd">,
                Requires<[HasVFP4,HasDPVFP,UseFusedMAC]>,

 
 // (fma x, y, (fneg z)) -> (vfnms z, x, y))
 def : Pat<(f64 (fma DPR:$Dn, DPR:$Dm, (fneg DPR:$Ddin))),
           (VFNMSD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(f32 (fma SPR:$Sn, SPR:$Sm, (fneg SPR:$Sdin))),
           (VFNMSS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(f16 (fma HPR:$Sn, HPR:$Sm, (fneg HPR:$Sdin))),
+          (VFNMSH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 // (fneg (fma (fneg x), y, z)) -> (vfnms z, x, y)
 def : Pat<(fneg (f64 (fma (fneg DPR:$Dn), DPR:$Dm, DPR:$Ddin))),
           (VFNMSD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(fneg (f32 (fma (fneg SPR:$Sn), SPR:$Sm, SPR:$Sdin))),
           (VFNMSS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(fneg (f16 (fma (fneg HPR:$Sn), HPR:$Sm, HPR:$Sdin))),
+          (VFNMSH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 // (fneg (fma x, (fneg y), z) -> (vfnms z, x, y)
 def : Pat<(fneg (f64 (fma DPR:$Dn, (fneg DPR:$Dm), DPR:$Ddin))),
           (VFNMSD DPR:$Ddin, DPR:$Dn, DPR:$Dm)>,
       Requires<[HasVFP4,HasDPVFP]>;
 def : Pat<(fneg (f32 (fma SPR:$Sn, (fneg SPR:$Sm), SPR:$Sdin))),
           (VFNMSS SPR:$Sdin, SPR:$Sn, SPR:$Sm)>,
       Requires<[HasVFP4]>;
+def : Pat<(fneg (f16 (fma HPR:$Sn, (fneg HPR:$Sm), HPR:$Sdin))),
+          (VFNMSH HPR:$Sdin, HPR:$Sn, HPR:$Sm)>,
+      Requires<[HasFullFP16]>;
 
 //===----------------------------------------------------------------------===//
 // FP Conditional moves.
 //
 
 let hasSideEffects = 0 in {
 def VMOVDcc  : PseudoInst<(outs DPR:$Dd), (ins DPR:$Dn, DPR:$Dm, cmovpred:$p),
                     IIC_fpUNA64,

llvm/test/CodeGen/ARM/fp16-fusedMAC.ll

; DONT-FUSE-NEXT:    bx lr
   store half %ret, half *%aa, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fms1(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fms1:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vldr.16 s0, [r1]
-; CHECK-NEXT:    vldr.16 s2, [r2]
-; CHECK-NEXT:    vldr.16 s4, [r0]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s2, s4, s0
-; CHECK-NEXT:    vstr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fms1:
 ; DONT-FUSE:       @ %bb.0:
 ; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
-; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
-; DONT-FUSE-NEXT:    vldr.16 s4, [r0]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s2, s4, s0
-; DONT-FUSE-NEXT:    vstr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %s = fsub half -0.0, %f1
   %ret = call half @llvm.fma.f16(half %s, half %f2, half %f3)
   store half %ret, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fms2(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fms2:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr.16 s0, [r1]
-; CHECK-NEXT:    vldr.16 s2, [r2]
-; CHECK-NEXT:    vldr.16 s4, [r0]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s2, s0, s4
-; CHECK-NEXT:    vstr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r1]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fms2:
 ; DONT-FUSE:       @ %bb.0:
-; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
-; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
-; DONT-FUSE-NEXT:    vldr.16 s4, [r0]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s2, s0, s4
-; DONT-FUSE-NEXT:    vstr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %s = fsub half -0.0, %f1
   %ret = call half @llvm.fma.f16(half %f2, half %s, half %f3)
   store half %ret, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fnma1(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fnma1:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:    vfma.f16 s4, s2, s0
-; CHECK-NEXT:    vneg.f16 s0, s4
-; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    vfnma.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fnma1:
 ; DONT-FUSE:       @ %bb.0:
 ; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
 ; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
 ; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
-; DONT-FUSE-NEXT:    vfma.f16 s4, s2, s0
-; DONT-FUSE-NEXT:    vneg.f16 s0, s4
-; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vfnma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %fma = call half @llvm.fma.f16(half %f1, half %f2, half %f3)
   %n1 = fsub half -0.0, %fma
   store half %n1, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fnma2(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fnma2:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-NEXT:    vneg.f16 s2, s2
 ; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s4, s2, s0
+; CHECK-NEXT:    vfnma.f16 s4, s2, s0
 ; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fnma2:
 ; DONT-FUSE:       @ %bb.0:
 ; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
 ; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
-; DONT-FUSE-NEXT:    vneg.f16 s2, s2
 ; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vfnma.f16 s4, s2, s0
 ; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %n1 = fsub half -0.0, %f1
   %n3 = fsub half -0.0, %f3
   %ret = call half @llvm.fma.f16(half %n1, half %f2, half %n3)
   store half %ret, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fnms1(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fnms1:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s4, s2, s0
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
 ; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fnms1:
 ; DONT-FUSE:       @ %bb.0:
 ; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
 ; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
 ; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
 ; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %n3 = fsub half -0.0, %f3
   %ret = call half @llvm.fma.f16(half %f1, half %f2, half %n3)
   store half %ret, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fnms2(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fnms2:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    vldr.16 s0, [r1]
-; CHECK-NEXT:    vldr.16 s2, [r2]
-; CHECK-NEXT:    vldr.16 s4, [r0]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s2, s4, s0
-; CHECK-NEXT:    vneg.f16 s0, s2
-; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fnms2:
 ; DONT-FUSE:       @ %bb.0:
 ; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
-; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
-; DONT-FUSE-NEXT:    vldr.16 s4, [r0]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s2, s4, s0
-; DONT-FUSE-NEXT:    vneg.f16 s0, s2
-; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %n1 = fsub half -0.0, %f1
   %fma = call half @llvm.fma.f16(half %n1, half %f2, half %f3)
   %n = fsub half -0.0, %fma
   store half %n, half *%a1, align 2
   ret void
 }
 
 define arm_aapcs_vfpcc void @fnms3(half *%a1, half *%a2, half *%a3) {
 ; CHECK-LABEL: fnms3:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr.16 s0, [r0]
-; CHECK-NEXT:    vldr.16 s2, [r2]
-; CHECK-NEXT:    vldr.16 s4, [r1]
-; CHECK-NEXT:    vneg.f16 s4, s4
-; CHECK-NEXT:    vfma.f16 s2, s0, s4
-; CHECK-NEXT:    vneg.f16 s0, s2
-; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    vldr.16 s0, [r1]
+; CHECK-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s4, [r2]
+; CHECK-NEXT:    vfnms.f16 s4, s2, s0
+; CHECK-NEXT:    vstr.16 s4, [r0]
 ; CHECK-NEXT:    bx lr
 ;
 ; DONT-FUSE-LABEL: fnms3:
 ; DONT-FUSE:       @ %bb.0:
-; DONT-FUSE-NEXT:    vldr.16 s0, [r0]
-; DONT-FUSE-NEXT:    vldr.16 s2, [r2]
-; DONT-FUSE-NEXT:    vldr.16 s4, [r1]
-; DONT-FUSE-NEXT:    vneg.f16 s4, s4
-; DONT-FUSE-NEXT:    vfma.f16 s2, s0, s4
-; DONT-FUSE-NEXT:    vneg.f16 s0, s2
-; DONT-FUSE-NEXT:    vstr.16 s0, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s0, [r1]
+; DONT-FUSE-NEXT:    vldr.16 s2, [r0]
+; DONT-FUSE-NEXT:    vldr.16 s4, [r2]
+; DONT-FUSE-NEXT:    vfnms.f16 s4, s2, s0
+; DONT-FUSE-NEXT:    vstr.16 s4, [r0]
 ; DONT-FUSE-NEXT:    bx lr
 
   %f1 = load half, half *%a1, align 2
   %f2 = load half, half *%a2, align 2
   %f3 = load half, half *%a3, align 2
   %n2 = fsub half -0.0, %f2
   %fma = call half @llvm.fma.f16(half %f1, half %n2, half %f3)
   %n1 = fsub half -0.0, %fma
   store half %n1, half *%a1, align 2
   ret void
 }
 
 
 declare half @llvm.fma.f16(half, half, half) nounwind readnone