[AArch64] Prefer to fold dup into fmul/fma as opposed to ld1r

[dmgreen]

Summary

There is a fold to create LD1DUPpost from dup(load) that can be postinc. If the dup is used by a "by element" operation such as fmul or fma then it can be slightly better to fold the dup into the fmul instead, which produces slightly fast code.

ld1r { v1.4s }, [x0], #4
fmul v0.4s, v1.4s, v0.4s

vs

ldr s1, [x0], #4
fmul v0.4s, v0.4s, v1.s[0]

This could also be done with integer operations such as smull/umull too, so long as the load/dup gets correctly combined into the mul operation. Currently this just operates on foating point types.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue performPostLD1Combine(SDNode *N,
   for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE;
        ++UI) {
     if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result.
       continue;
     if (*UI != N)
       return SDValue();
   }
 
+  // If there is one use and it can splat the value, prefer that operation.
+  // TODO: This could be expanded to more operations if they reliably use the
+  // index variants.
+  if (N->hasOneUse()) {
+    unsigned UseOpc = N->use_begin()->getOpcode();
+    if (UseOpc == ISD::FMUL || UseOpc == ISD::FMA)
+      return SDValue();
+  }
+
   SDValue Addr = LD->getOperand(1);
   SDValue Vector = N->getOperand(0);
   // Search for a use of the address operand that is an increment.
   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE =
        Addr.getNode()->use_end(); UI != UE; ++UI) {
     SDNode *User = *UI;
     if (User->getOpcode() != ISD::ADD
         || UI.getUse().getResNo() != Addr.getResNo())

llvm/test/CodeGen/AArch64/ld1postmul.ll

 ; CHECK-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
 ; CHECK-NOFP16-NEXT:    fmul v0.4s, v1.4s, v0.4s
 ; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-NOFP16-NEXT:    str d0, [x1]
 ; CHECK-NOFP16-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: fmul_v4f16:
 ; CHECK-FP16:       // %bb.0:
-; CHECK-FP16-NEXT:    ld1r { v1.4h }, [x0], #2
-; CHECK-FP16-NEXT:    fmul v0.4h, v1.4h, v0.4h
+; CHECK-FP16-NEXT:    ldr h1, [x0], #2
+; CHECK-FP16-NEXT:    fmul v0.4h, v0.4h, v1.h[0]
 ; CHECK-FP16-NEXT:    str d0, [x1]
 ; CHECK-FP16-NEXT:    ret
   %l = load half, ptr %p
   %i = insertelement <4 x half> undef, half %l, i32 0
   %s = shufflevector <4 x half> %i, <4 x half> undef, <4 x i32> zeroinitializer
   %m = fmul <4 x half> %s, %t
   store <4 x half> %m, ptr %ps
   %g = getelementptr half, ptr %p, i64 1

 ; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
 ; CHECK-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
 ; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-NOFP16-NEXT:    str d0, [x1]
 ; CHECK-NOFP16-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: fmla_v4f16:
 ; CHECK-FP16:       // %bb.0:
-; CHECK-FP16-NEXT:    ld1r { v2.4h }, [x0], #2
-; CHECK-FP16-NEXT:    fmla v1.4h, v0.4h, v2.4h
+; CHECK-FP16-NEXT:    ldr h2, [x0], #2
+; CHECK-FP16-NEXT:    fmla v1.4h, v0.4h, v2.h[0]
 ; CHECK-FP16-NEXT:    str d1, [x1]
 ; CHECK-FP16-NEXT:    ret
   %l = load half, ptr %p
   %i = insertelement <4 x half> undef, half %l, i32 0
   %s = shufflevector <4 x half> %i, <4 x half> undef, <4 x i32> zeroinitializer
   %m = fmul fast <4 x half> %s, %t
   %a = fadd fast <4 x half> %m, %u
   store <4 x half> %a, ptr %ps
   %g = getelementptr half, ptr %p, i64 1
   ret ptr %g
 }
 
 define ptr @fmul_v4f32(ptr %p, ptr %ps, <4 x float> %t) {
 ; CHECK-LABEL: fmul_v4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1r { v1.4s }, [x0], #4
-; CHECK-NEXT:    fmul v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ldr s1, [x0], #4
+; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
   %l = load float, ptr %p
   %i = insertelement <4 x float> undef, float %l, i32 0
   %s = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %m = fmul <4 x float> %s, %t
   store <4 x float> %m, ptr %ps
   %g = getelementptr float, ptr %p, i64 1
   ret ptr %g
 }
 
 define ptr @fmla_v4f32(ptr %p, ptr %ps, <4 x float> %t, <4 x float> %u) {
 ; CHECK-LABEL: fmla_v4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1r { v2.4s }, [x0], #4
-; CHECK-NEXT:    fmla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldr s2, [x0], #4
+; CHECK-NEXT:    fmla v1.4s, v0.4s, v2.s[0]
 ; CHECK-NEXT:    str q1, [x1]
 ; CHECK-NEXT:    ret
   %l = load float, ptr %p
   %i = insertelement <4 x float> undef, float %l, i32 0
   %s = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %m = fmul fast <4 x float> %s, %t
   %a = fadd fast <4 x float> %m, %u
   store <4 x float> %a, ptr %ps
   %g = getelementptr float, ptr %p, i64 1
   ret ptr %g
 }
 
 define ptr @fmul_v2f64(ptr %p, ptr %ps, <2 x double> %t) {
 ; CHECK-LABEL: fmul_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1r { v1.2d }, [x0], #8
-; CHECK-NEXT:    fmul v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    ldr d1, [x0], #8
+; CHECK-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
   %l = load double, ptr %p
   %i = insertelement <2 x double> undef, double %l, i32 0
   %s = shufflevector <2 x double> %i, <2 x double> undef, <2 x i32> zeroinitializer
   %m = fmul <2 x double> %s, %t
   store <2 x double> %m, ptr %ps
   %g = getelementptr double, ptr %p, i64 1
   ret ptr %g
 }
 
 define ptr @fmla_v2f64(ptr %p, ptr %ps, <2 x double> %t, <2 x double> %u) {
 ; CHECK-LABEL: fmla_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1r { v2.2d }, [x0], #8
-; CHECK-NEXT:    fmla v1.2d, v0.2d, v2.2d
+; CHECK-NEXT:    ldr d2, [x0], #8
+; CHECK-NEXT:    fmla v1.2d, v0.2d, v2.d[0]
 ; CHECK-NEXT:    str q1, [x1]
 ; CHECK-NEXT:    ret
   %l = load double, ptr %p
   %i = insertelement <2 x double> undef, double %l, i32 0
   %s = shufflevector <2 x double> %i, <2 x double> undef, <2 x i32> zeroinitializer
   %m = fmul fast <2 x double> %s, %t
   %a = fadd fast <2 x double> %m, %u
   store <2 x double> %a, ptr %ps
   %g = getelementptr double, ptr %p, i64 1
   ret ptr %g
 }