[SVE] Add pattern to commute FMLS operands

[cameron.mcinally]

Summary

AArch64ISD::FMA_PRED nodes with negated operands are matching to suboptimal hardware instructions. We can commute the multiplication operands to generate better instruction sequences. This patch adds a pattern to commute operands for an unmasked FMSB instruction.

I'll try to send matching patches for predicated FMLS and FNMLA patterns soon, assuming this is accepted. Although, if someone is motivated to rework this multiclass in a cohesive way, please feel free.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

562–563

Perhaps you can create a PatFrags called AArch64fmls_p that contains the two compatible forms of AArch64fma_p. You should then be able to update the relevant patterns here instead of needing new ones. See AArch64bic for inspiration.

[cameron.mcinally]

Updated Diff to implement commuting with PatFrags.

@paulwalker-arm, I wasn't sure if the PatFrags live at the head of the file, or close to the pattern that uses them. I chose the head. Keep an eye out for that.

[paulwalker-arm]

In D120570#3349616, @cameron.mcinally wrote:

@paulwalker-arm, I wasn't sure if the PatFrags live at the head of the file, or close to the pattern that uses them. I chose the head. Keep an eye out for that.

PatFrags placement looks perfect to me.

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

296–299

When creating the PatFrags you want the ops operands to be treated consistently. In this instance you always want $op1 to be the negated operand because that is the operand that will be negated by the instruction you'll eventually match it to. So the first pattern is fine but the second should switch the usage of $op1 and $op2.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fp.ll

321–323

Can these tests treat the operands the same as the fma tests. You're currently treating %c as the addend whereas the existing tests have this as %a. The output will likely change to fmls instead of fmsb but that seems find to me.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fp.ll

321–323

I've just checked and these negated patterns are tested within sve-fp-combine.ll so I think that'll be a better home for your new tests.

[cameron.mcinally]

llvm/test/CodeGen/AArch64/sve-fp.ll

321–323

I saw that file when I added the tests. The sve-fp-combine.ll tests all deal with FMUL+[FADD|FSUB] IR. There are no intrinsic tests in that file, so a little bit of a weird fit.

I don't have a preference on where these tests live though, so it's up to you...

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fp.ll

321–323

Oh, I see what you mean. I only focused on the output. In which case feel free to keep the tests here.

[cameron.mcinally]

Addressed Paul's review...

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 def AArch64mul_p_oneuse : PatFrag<(ops node:$pred, node:$src1, node:$src2),
                                   (AArch64mul_p node:$pred, node:$src1, node:$src2), [{
   return N->hasOneUse();
 }]>;
 
 def AArch64fabd_p : PatFrag<(ops node:$pg, node:$op1, node:$op2),
                             (AArch64fabs_mt node:$pg, (AArch64fsub_p node:$pg, node:$op1, node:$op2), undef)>;
 
+// FMAs with a negated multiplication operand can be commuted.
+def AArch64fmls_p : PatFrags<(ops node:$pred, node:$op1, node:$op2, node:$op3),
+                          [(AArch64fma_p node:$pred, (AArch64fneg_mt node:$pred, node:$op1, (undef)), node:$op2, node:$op3),
+                           (AArch64fma_p node:$pred, node:$op2, (AArch64fneg_mt node:$pred, node:$op1, (undef)), node:$op3)]>;
+
 def AArch64fneg_mt_nsz : PatFrag<(ops node:$pred, node:$op, node:$pt),
                                  (AArch64fneg_mt node:$pred, node:$op, node:$pt), [{
   return N->getFlags().hasNoSignedZeros();
 }]>;
 
 def SDT_AArch64Arith_Unpred : SDTypeProfile<1, 2, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>,
   SDTCisSameAs<0,1>, SDTCisSameAs<1,2>
let Predicates = [HasSVEorStreamingSVE] in {
   defm FNMLS_ZPZZZ : sve_fp_3op_p_zds_zx;
 
   multiclass fma<ValueType Ty, ValueType PredTy, string Suffix> {
     // Zd = Za + Zn * Zm
     def : Pat<(Ty (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za)),
               (!cast<Instruction>("FMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zd = Za + -Zn * Zm
-    def : Pat<(Ty (AArch64fma_p PredTy:$P, (AArch64fneg_mt PredTy:$P, Ty:$Zn, (Ty (undef))), Ty:$Zm, Ty:$Za)),
+    def : Pat<(Ty (AArch64fmls_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za)),
               (!cast<Instruction>("FMLS_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zd = -Za + Zn * Zm
     def : Pat<(Ty (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, (AArch64fneg_mt PredTy:$P, Ty:$Za, (Ty (undef))))),
               (!cast<Instruction>("FNMLS_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zd = -Za + -Zn * Zm
     def : Pat<(Ty (AArch64fma_p PredTy:$P, (AArch64fneg_mt PredTy:$P, Ty:$Zn, (Ty (undef))), Ty:$Zm, (AArch64fneg_mt PredTy:$P, Ty:$Za, (Ty (undef))))),
               (!cast<Instruction>("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zd = -(Za + Zn * Zm)

llvm/test/CodeGen/AArch64/sve-fp.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmla z0.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> %c, <vscale x 2 x double> %b, <vscale x 2 x double> %a)
   ret <vscale x 2 x double> %r
 }
 
+define <vscale x 8 x half> @fmls_nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
+; CHECK-LABEL: fmls_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fmls z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 8 x half> %b
+  %r = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> %c, <vscale x 8 x half> %neg, <vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %r
+}
+
+define <vscale x 4 x half> @fmls_nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b, <vscale x 4 x half> %c) {
+; CHECK-LABEL: fmls_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmls z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 4 x half> %b
+  %r = call <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half> %c, <vscale x 4 x half> %neg, <vscale x 4 x half> %a)
+  ret <vscale x 4 x half> %r
+}
+
+define <vscale x 2 x half> @fmls_nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b, <vscale x 2 x half> %c) {
+; CHECK-LABEL: fmls_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmls z0.h, p0/m, z1.h, z2.h
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 2 x half> %b
+  %r = call <vscale x 2 x half> @llvm.fma.nxv2f16(<vscale x 2 x half> %c, <vscale x 2 x half> %neg, <vscale x 2 x half> %a)
+  ret <vscale x 2 x half> %r
+}
+
+define <vscale x 4 x float> @fmls_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
+; CHECK-LABEL: fmls_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmls z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 4 x float> %b
+  %r = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %c, <vscale x 4 x float> %neg, <vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %r
+}
+
+define <vscale x 2 x float> @fmls_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <vscale x 2 x float> %c) {
+; CHECK-LABEL: fmls_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmls z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 2 x float> %b
+  %r = call <vscale x 2 x float> @llvm.fma.nxv2f32(<vscale x 2 x float> %c, <vscale x 2 x float> %neg, <vscale x 2 x float> %a)
+  ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x double> @fmls_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
+; CHECK-LABEL: fmls_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmls z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %neg = fneg <vscale x 2 x double> %b
+  %r = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> %c, <vscale x 2 x double> %neg, <vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %r
+}
+
 define <vscale x 8 x half> @fneg_nxv8f16(<vscale x 8 x half> %a) {
 ; CHECK-LABEL: fneg_nxv8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
 ; CHECK-NEXT:    ret
   %res = fneg <vscale x 8 x half> %a
   ret <vscale x 8 x half> %res