[AArch64] Avoid more “by element” instructions for Exynos

[az]

Summary

In addition to the SIMD vector instructions already there (vector by element instructions), add the SIMD scalar instructions (scalar by element) to the list of instructions where the “by element” feature should be avoided.

llvm/lib/Target/AArch64/AArch64VectorByElementOpt.cpp

case AArch64::FMULXv2i32_indexed:
     DupMCID = &TII->get(AArch64::DUPv2i32lane);
     MulMCID = &TII->get(AArch64::FMULXv2f32);
     break;
   case AArch64::FMULv2i32_indexed:
     RC = &AArch64::FPR64RegClass;
     DupMCID = &TII->get(AArch64::DUPv2i32lane);
     MulMCID = &TII->get(AArch64::FMULv2f32);
     break;
+
+  // 1X64 instructions
+  case AArch64::FMLAv1i64_indexed:
+    RC = &AArch64::FPR64RegClass;
+    DupMCID = &TII->get(AArch64::CPYi64);
+    MulMCID = &TII->get(AArch64::FMADDDrrr);
+    break;
+  case AArch64::FMLSv1i64_indexed:
+    RC = &AArch64::FPR64RegClass;
+    DupMCID = &TII->get(AArch64::CPYi64);
+    MulMCID = &TII->get(AArch64::FMSUBDrrr);
+    break;
+  case AArch64::FMULXv1i64_indexed:
+    RC = &AArch64::FPR64RegClass;
+    DupMCID = &TII->get(AArch64::CPYi64);
+    MulMCID = &TII->get(AArch64::FMULX64);
+    break;
+  case AArch64::FMULv1i64_indexed:
+    RC = &AArch64::FPR64RegClass;
+    DupMCID = &TII->get(AArch64::CPYi64);
+    MulMCID = &TII->get(AArch64::FMULDrr);
+    break;
+
+  // 1X32 instructions
+  case AArch64::FMLAv1i32_indexed:
+    RC = &AArch64::FPR32RegClass;
+    DupMCID = &TII->get(AArch64::CPYi32);
+    MulMCID = &TII->get(AArch64::FMADDSrrr);
+    break;
+  case AArch64::FMLSv1i32_indexed:
+    RC = &AArch64::FPR32RegClass;
+    DupMCID = &TII->get(AArch64::CPYi32);
+    MulMCID = &TII->get(AArch64::FMSUBSrrr);
+    break;
+  case AArch64::FMULXv1i32_indexed:
+    RC = &AArch64::FPR32RegClass;
+    DupMCID = &TII->get(AArch64::CPYi32);
+    MulMCID = &TII->get(AArch64::FMULX32);
+    break;
+  case AArch64::FMULv1i32_indexed:
+    RC = &AArch64::FPR32RegClass;
+    DupMCID = &TII->get(AArch64::CPYi32);
+    MulMCID = &TII->get(AArch64::FMULSrr);
+    break;
   }
 
   if (!shouldReplaceInstruction(MI.getParent()->getParent(),
                                 &TII->get(MI.getOpcode()), DupMCID, MulMCID,
                                 *VecInstElemTable))
     return false;
 
   const DebugLoc &DL = MI.getDebugLoc();
if (MI.getNumOperands() == 5) {
     // has already been created before, then use that one instread of creating
     // a new one.
     if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) {
       DupDest = MRI.createVirtualRegister(RC);
       BuildMI(MBB, MI, DL, *DupMCID, DupDest)
           .addReg(SrcReg2, Src2IsKill)
           .addImm(LaneNumber);
     }
-    BuildMI(MBB, MI, DL, *MulMCID, MulDest)
-        .addReg(SrcReg0, Src0IsKill)
-        .addReg(SrcReg1, Src1IsKill)
-        .addReg(DupDest, Src2IsKill);
+    // Create the new non-indexed instruction
+    switch (MI.getOpcode()) {
+      default:
+        // for most SIMD instructions, use this to create instruction 
+        BuildMI(MBB, MI, DL, *MulMCID, MulDest)
+          .addReg(SrcReg0, Src0IsKill)
+          .addReg(SrcReg1, Src1IsKill)
+          .addReg(DupDest, Src2IsKill);
+        break;
+      case AArch64::FMLAv1i32_indexed:
+      case AArch64::FMLSv1i32_indexed:
+      case AArch64::FMLAv1i64_indexed:
+      case AArch64::FMLSv1i64_indexed:
+        // for floating point fmadd/fmsub instructions, use this
+        BuildMI(MBB, MI, DL, *MulMCID, MulDest)
+          .addReg(SrcReg1, Src1IsKill)
+          .addReg(DupDest, Src2IsKill)
+          .addReg(SrcReg0, Src0IsKill);
+        break;
+    }
   } else if (MI.getNumOperands() == 4) {
     unsigned LaneNumber = MI.getOperand(3).getImm();
     if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) {
       DupDest = MRI.createVirtualRegister(RC);
       BuildMI(MBB, MI, DL, *DupMCID, DupDest)
           .addReg(SrcReg1, Src1IsKill)
           .addImm(LaneNumber);
     }

llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll

 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast | FileCheck %s
 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -mattr=+neon -fp-contract=fast -mcpu=exynos-m1 | FileCheck --check-prefix=EXYNOS %s
 ; The instruction latencies of Exynos-M1 trigger the transform we see under the Exynos check.
 
 declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>)
 
 declare <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float>, <4 x float>)
 
 declare <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float>, <2 x float>)
 
+declare float @llvm.aarch64.neon.fmulx.f32(float, float)
+
+declare double @llvm.aarch64.neon.fmulx.f64(double, double)
+
 declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>)
 
 declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32>, <2 x i32>)
 
 declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>)
 
 declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>)
 

 ; EXYNOS: fmla {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, [[x]].2d
 ; EXYNOS-NEXT: ret
 entry:
   %lane = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>
   %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)
   ret <2 x double> %0
 }
 
+define double @test_vfmla_lane_f64(double %a, double %b, <2 x double> %v) {
+; CHECK-LABEL: test_vfmla_lane_f64:
+; CHECK: fmla {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
+; CHECK-NEXT: ret
+; EXYNOS-LABEL: test_vfmla_lane_f64
+; EXYNOS: fmadd {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
+; EXYNOS-NEXT: ret
+entry:
+  %extract = extractelement <2 x double> %v, i32 1
+  %0 = tail call double @llvm.fma.f64(double %b, double %extract, double %a)
+  ret double %0
+}
+
 define <2 x double> @test_vfmsq_lane_f64(<2 x double> %a, <2 x double> %b, <1 x double> %v) {
 ; CHECK-LABEL: test_vfmsq_lane_f64:
 ; CHECK: fmls {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, {{v[0-9]+}}.d[0]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vfmsq_lane_f64:
 ; EXYNOS: dup  [[x:v[0-9]+]].2d, {{v[0-9]+}}.d[0]
 ; EXYNOS: fmls {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, [[x]].2d
 ; EXYNOS-NEXT: ret

 
 declare double @llvm.fma.f64(double, double, double)
 
 define float @test_vfmss_lane_f32(float %a, float %b, <2 x float> %v) {
 ; CHECK-LABEL: test_vfmss_lane_f32
 ; CHECK: fmls {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vfmss_lane_f32
-; EXYNOS: fmls {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+; EXYNOS: fmsub {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; EXYNOS-NEXT: ret
 entry:
   %extract.rhs = extractelement <2 x float> %v, i32 1
   %extract = fsub float -0.000000e+00, %extract.rhs
   %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)
   ret float %0
 }
 
 define float @test_vfmss_laneq_f32(float %a, float %b, <4 x float> %v) {
 ; CHECK-LABEL: test_vfmss_laneq_f32
 ; CHECK: fmls {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[3]
 ; CHECK-NEXT: ret
 entry:
   %extract.rhs = extractelement <4 x float> %v, i32 3
   %extract = fsub float -0.000000e+00, %extract.rhs
   %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)
   ret float %0
 }
 
 define double @test_vfmsd_laneq_f64(double %a, double %b, <2 x double> %v) {
 ; CHECK-LABEL: test_vfmsd_laneq_f64
 ; CHECK: fmls {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vfmsd_laneq_f64
-; EXYNOS: fmls {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
+; EXYNOS: fmsub {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
 ; EXYNOS-NEXT: ret
 entry:
   %extract.rhs = extractelement <2 x double> %v, i32 1
   %extract = fsub double -0.000000e+00, %extract.rhs
   %0 = tail call double @llvm.fma.f64(double %b, double %extract, double %a)
   ret double %0
 }
 
 define double @test_vfmsd_lane_f64_0(double %a, double %b, <1 x double> %v) {
 ; CHCK-LABEL: test_vfmsd_lane_f64_0
 ; CHCK: fmsub {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
 ; CHCK-NEXT: ret
 entry:
   %tmp0 = fsub <1 x double> <double -0.000000e+00>, %v
   %tmp1 = extractelement <1 x double> %tmp0, i32 0
   %0 = tail call double @llvm.fma.f64(double %b, double %tmp1, double %a)
   ret double %0
 }
 
 define float @test_vfmss_lane_f32_0(float %a, float %b, <2 x float> %v) {
 ; CHECK-LABEL: test_vfmss_lane_f32_0
 ; CHECK: fmls {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vfmss_lane_f32_0
-; EXYNOS: fmls {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+; EXYNOS: fmsub {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; EXYNOS-NEXT: ret
 entry:
   %tmp0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v
   %tmp1 = extractelement <2 x float> %tmp0, i32 1
   %0 = tail call float @llvm.fma.f32(float %b, float %tmp1, float %a)
   ret float %0
 }
 
entry:
   ret <2 x float> %mul
 }
 
 define <1 x double> @test_vmul_laneq_f64(<1 x double> %a, <2 x double> %v) {
 ; CHECK-LABEL: test_vmul_laneq_f64:
 ; CHECK: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vmul_laneq_f64:
-; EXYNOS: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
+; EXYNOS: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
 ; EXYNOS-NEXT: ret
 entry:
   %0 = bitcast <1 x double> %a to <8 x i8>
   %1 = bitcast <8 x i8> %0 to double
   %extract = extractelement <2 x double> %v, i32 1
   %2 = fmul double %1, %extract
   %3 = insertelement <1 x double> undef, double %2, i32 0
   ret <1 x double> %3

 ; EXYNOS: fmla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[x]].4s
 ; EXYNOS-NEXT: ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
   %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)
   ret <4 x float> %0
 }
 
+define float @test_vfmla_lane_f32(float %a, float%b, <2 x float> %v) {
+; CHECK-LABEL: test_vfmla_lane_f32:
+; CHECK: fmla {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+; CHECK-NEXT: ret
+; EXYNOS-LABEL: test_vfmla_lane_f32
+; EXYNOS: fmadd {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
+; EXYNOS-NEXT: ret
+entry:
+  %extract = extractelement <2 x float> %v, i32 1
+  %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)
+  ret float %0
+}
+
 define <2 x float> @test_vfms_lane_f32_0(<2 x float> %a, <2 x float> %b, <2 x float> %v) {
 ; CHECK-LABEL: test_vfms_lane_f32_0:
 ; CHECK: fmls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vfms_lane_f32_0:
 ; EXYNOS: dup  [[x:v[0-9]+]].2s, {{v[0-9]+}}.s[0]
 ; EXYNOS: fmls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[x]].2s
 ; EXYNOS-NEXT: ret
entry:
   ret <2 x float> %mul
 }
 
 define <1 x double> @test_vmul_laneq_f64_0(<1 x double> %a, <2 x double> %v) {
 ; CHECK-LABEL: test_vmul_laneq_f64_0:
 ; CHECK: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[0]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: test_vmul_laneq_f64_0:
-; EXYNOS: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[0]
+; EXYNOS: fmul {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
 ; EXYNOS-NEXT: ret
 entry:
   %0 = bitcast <1 x double> %a to <8 x i8>
   %1 = bitcast <8 x i8> %0 to double
   %extract = extractelement <2 x double> %v, i32 0
   %2 = fmul double %1, %extract
   %3 = insertelement <1 x double> undef, double %2, i32 0
   ret <1 x double> %3

 ; EXYNOS: mulx {{v[0-9]+}}.2d, {{v[0-9]+}}.2d, [[x]].2d
 ; EXYNOS-NEXT: ret
 entry:
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)
   ret <2 x double> %vmulx2.i
 }
 
+define double @test_vmulx_lane_f64(double %a, <2 x double> %v) {
+; CHECK-LABEL: test_vmulx_lane_f64:
+; CHECK: mulx {{d[0-9]+}}, {{d[0-9]+}}, {{v[0-9]+}}.d[1]
+; CHECK-NEXT: ret
+; EXYNOS-LABEL: test_vmulx_lane_f64:
+; EXYNOS: mulx {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
+; EXYNOS-NEXT: ret
+entry:
+  %extract = extractelement <2 x double> %v, i32 1
+  %0 = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %extract)
+  ret double %0
+}
+
+define float @test_vmulx_lane_f32_1(float %a, <2 x float> %v) {
+; CHECK-LABEL: test_vmulx_lane_f32_1:
+; CHECK: mulx {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+; CHECK-NEXT: ret
+; EXYNOS-LABEL: test_vmulx_lane_f32_1:
+; EXYNOS: mulx {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
+; EXYNOS-NEXT: ret
+entry:
+  %extract = extractelement <2 x float> %v, i32 1
+  %0 = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %extract)
+  ret float %0
+}
+
 define <4 x float> @optimize_dup(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %v) {
 ; CHECK-LABEL: optimize_dup:
 ; CHECK: fmla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[3]
 ; CHECK: fmls {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[3]
 ; CHECK-NEXT: ret
 ; EXYNOS-LABEL: optimize_dup:
 ; EXYNOS: dup  [[x:v[0-9]+]].4s, {{v[0-9]+}}.s[3]
 ; EXYNOS: fmla {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[x]].4s