[Clang] Add vcmla and rotated variants for Arm ACLE.

[fhahn]

Summary

This patch adds vcmla and the rotated variants as defined in
"Arm Neon Intrinsics Reference for ACLE Q3 2020" [1]

The *_lane_* are still missing, but they can be added separately.

This patch only adds the builtin mapping for AArch64.

[1] https://developer.arm.com/documentation/ihi0073/latest

[t.p.northover]

Looks fine to me except for one weird quirk that I don't think should be there.

clang/lib/CodeGen/CGBuiltin.cpp

5237

This line looks identical to the one above.

[fhahn]

Updated to use new VCMLA_ROTS multiclass to define rotated variants.

[fhahn]

clang/lib/CodeGen/CGBuiltin.cpp

5237

Yes I accidentally copied this line. It's gone now.

[fhahn]

Loop over rotation variants.

[t.p.northover]

That's a nice improvement. Still LGTM.

clang/include/clang/Basic/arm_neon.td

let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
 
   def VFMLALB_LANE_BF : SOpInst<"vbfmlalb_lane", "..B(Bq)I", "Qf", OP_BFMLALB_LN>;
   def VFMLALB_LANEQ_BF : SOpInst<"vbfmlalb_laneq", "..B(BQ)I", "Qf", OP_BFMLALB_LN>;
 
   def VFMLALT_LANE_BF : SOpInst<"vbfmlalt_lane", "..B(Bq)I", "Qf", OP_BFMLALT_LN>;
   def VFMLALT_LANEQ_BF : SOpInst<"vbfmlalt_laneq", "..B(BQ)I", "Qf", OP_BFMLALT_LN>;
 }
 
+multiclass VCMLA_ROTS<string args, string type, string qstr> {
+  def ROT0 : SInst<"vcmla" # qstr, args, type>;
+  def ROT90 : SInst<"vcmla" # qstr # "_rot90", args, type>;
+  def ROT180 : SInst<"vcmla" # qstr # "_rot180", args, type>;
+  def ROT3270 : SInst<"vcmla" # qstr # "_rot270", args, type>;
+}
+
 // v8.3-A Vector complex addition intrinsics
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
   def VCADD_ROT90_FP16   : SInst<"vcadd_rot90", "...", "h">;
   def VCADD_ROT270_FP16  : SInst<"vcadd_rot270", "...", "h">;
   def VCADDQ_ROT90_FP16  : SInst<"vcaddq_rot90", "QQQ", "h">;
   def VCADDQ_ROT270_FP16 : SInst<"vcaddq_rot270", "QQQ", "h">;
+
+
+  defm VCMLA_FP16  : VCMLA_ROTS<"....", "h", "">;
+  defm VCMLAQ_FP16 : VCMLA_ROTS<"QQQQ", "h", "q">;
 }
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX)" in {
   def VCADD_ROT90   : SInst<"vcadd_rot90", "...", "f">;
   def VCADD_ROT270  : SInst<"vcadd_rot270", "...", "f">;
   def VCADDQ_ROT90  : SInst<"vcaddq_rot90", "QQQ", "f">;
   def VCADDQ_ROT270 : SInst<"vcaddq_rot270", "QQQ", "f">;
+
+  defm VCMLA_F32        : VCMLA_ROTS< "....", "f", "">;
+  defm VCMLAQ_F32 : VCMLA_ROTS<"QQQQ", "f", "q">;
 }
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)" in {
   def VCADDQ_ROT90_FP64  : SInst<"vcaddq_rot90", "QQQ", "d">;
   def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
+
+  defm VCMLAQ_FP64 : VCMLA_ROTS<"QQQQ", "d", "q">;
 }
 
 // V8.2-A BFloat intrinsics
 let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
   def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> {
     let BigEndianSafe = 1;
   }
 

clang/lib/CodeGen/CGBuiltin.cpp

static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vbfmlaltq_v, arm_neon_bfmlalt, 0),
   NEONMAP1(vbfmmlaq_v, arm_neon_bfmmla, 0),
   NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
   NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
   NEONMAP1(vcadd_rot270_v, arm_neon_vcadd_rot270, Add1ArgType),
   NEONMAP1(vcadd_rot90_v, arm_neon_vcadd_rot90, Add1ArgType),
   NEONMAP1(vcaddq_rot270_v, arm_neon_vcadd_rot270, Add1ArgType),
   NEONMAP1(vcaddq_rot90_v, arm_neon_vcadd_rot90, Add1ArgType),
   NEONMAP1(vcage_v, arm_neon_vacge, 0),
   NEONMAP1(vcageq_v, arm_neon_vacge, 0),
   NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
   NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
   NEONMAP1(vcale_v, arm_neon_vacge, 0),
   NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
   NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
   NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
   NEONMAP0(vceqz_v),
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vusmmlaq_v, arm_neon_usmmla, 0),
   NEONMAP0(vuzp_v),
   NEONMAP0(vuzpq_v),
   NEONMAP0(vzip_v),
   NEONMAP0(vzipq_v)
 };
 
 static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
   NEONMAP1(__a64_vcvtq_low_bf16_v, aarch64_neon_bfcvtn, 0),
   NEONMAP0(splat_lane_v),
   NEONMAP0(splat_laneq_v),
   NEONMAP0(splatq_lane_v),
   NEONMAP0(splatq_laneq_v),
   NEONMAP1(vabs_v, aarch64_neon_abs, 0),
   NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
   NEONMAP0(vaddhn_v),
   NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
   NEONMAP0(vclez_v),
   NEONMAP0(vclezq_v),
   NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
   NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
   NEONMAP0(vcltz_v),
   NEONMAP0(vcltzq_v),
   NEONMAP1(vclz_v, ctlz, Add1ArgType),
   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
+  NEONMAP1(vcmla_rot180_v, aarch64_neon_vcmla_rot180, Add1ArgType),
+  NEONMAP1(vcmla_rot270_v, aarch64_neon_vcmla_rot270, Add1ArgType),
+  NEONMAP1(vcmla_rot90_v, aarch64_neon_vcmla_rot90, Add1ArgType),
+  NEONMAP1(vcmla_v, aarch64_neon_vcmla_rot0, Add1ArgType),
+  NEONMAP1(vcmlaq_rot180_v, aarch64_neon_vcmla_rot180, Add1ArgType),
+  NEONMAP1(vcmlaq_rot270_v, aarch64_neon_vcmla_rot270, Add1ArgType),
+  NEONMAP1(vcmlaq_rot90_v, aarch64_neon_vcmla_rot90, Add1ArgType),
+  NEONMAP1(vcmlaq_v, aarch64_neon_vcmla_rot0, Add1ArgType),
   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
   NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
   NEONMAP0(vcvt_f16_v),
   NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
   NEONMAP0(vcvt_f32_v),
   NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
   NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),

clang/test/CodeGen/aarch64-neon-vcmla.c

+// RUN: %clang_cc1 -triple aarch64-linux-gnu -target-feature +neon \
+// RUN:  -target-feature +v8.3a -target-feature +fullfp16 -S -emit-llvm -o - %s \
+// RUN:  | FileCheck %s
+
+#include <arm_neon.h>
+
+void foo_16x4(float16x4_t a, float16x4_t b, float16x4_t c) {
+ // CHECK: call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16
+  float16x4_t result = vcmla_f16(a, b, c);
+}
+
+void foo_rot90_16x4(float16x4_t a, float16x4_t b, float16x4_t c) {
+ // CHECK: call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16
+  float16x4_t result = vcmla_rot90_f16(a, b, c);
+}
+
+void foo_rot180_16x4(float16x4_t a, float16x4_t b, float16x4_t c) {
+ // CHECK: call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16
+  float16x4_t result = vcmla_rot180_f16(a, b, c);
+}
+
+void foo_rot270_16x4(float16x4_t a, float16x4_t b, float16x4_t c) {
+ // CHECK: call <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16
+  float16x4_t result = vcmla_rot270_f16(a, b, c);
+}
+
+void foo_16x8(float16x8_t a, float16x8_t b, float16x8_t c) {
+ // CHECK: call <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16
+  float16x8_t result = vcmlaq_f16(a, b, c);
+}
+
+void foo_rot90_16x8(float16x8_t a, float16x8_t b, float16x8_t c) {
+ // CHECK: call <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16
+  float16x8_t result = vcmlaq_rot90_f16(a, b, c);
+}
+
+void foo_rot180_16x8(float16x8_t a, float16x8_t b, float16x8_t c) {
+ // CHECK: call <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16
+  float16x8_t result = vcmlaq_rot180_f16(a, b, c);
+}
+
+void foo_rot270_16x8(float16x8_t a, float16x8_t b, float16x8_t c) {
+ // CHECK: call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16
+  float16x8_t result = vcmlaq_rot270_f16(a, b, c);
+}
+
+void foo_32x2(float32x2_t a, float32x2_t b, float32x2_t c) {
+ // CHECK: call <2 x float> @llvm.aarch64.neon.vcmla.rot0.v2f32
+  float32x2_t result = vcmla_f32(a, b, c);
+}
+
+void foo_rot90_32x2(float32x2_t a, float32x2_t b, float32x2_t c) {
+ // CHECK: call <2 x float> @llvm.aarch64.neon.vcmla.rot90.v2f32
+  float32x2_t result = vcmla_rot90_f32(a, b, c);
+}
+
+void foo_rot180_32x2(float32x2_t a, float32x2_t b, float32x2_t c) {
+ // CHECK: call <2 x float> @llvm.aarch64.neon.vcmla.rot180.v2f32
+  float32x2_t result = vcmla_rot180_f32(a, b, c);
+}
+
+void foo_rot270_32x2(float32x2_t a, float32x2_t b, float32x2_t c) {
+ // CHECK: call <2 x float> @llvm.aarch64.neon.vcmla.rot270.v2f32
+  float32x2_t result = vcmla_rot270_f32(a, b, c);
+}
+
+void foo_32x4(float32x4_t a, float32x4_t b, float32x4_t c) {
+ // CHECK: call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32
+  float32x4_t result = vcmlaq_f32(a, b, c);
+}
+
+void foo_rot90_32x4(float32x4_t a, float32x4_t b, float32x4_t c) {
+ // CHECK: call <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32
+  float32x4_t result = vcmlaq_rot90_f32(a, b, c);
+}
+
+void foo_rot180_32x4(float32x4_t a, float32x4_t b, float32x4_t c) {
+ // CHECK: call <4 x float> @llvm.aarch64.neon.vcmla.rot180.v4f32
+  float32x4_t result = vcmlaq_rot180_f32(a, b, c);
+}
+
+void foo_rot270_32x4(float32x4_t a, float32x4_t b, float32x4_t c) {
+ // CHECK: call <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32
+  float32x4_t result = vcmlaq_rot270_f32(a, b, c);
+}
+
+void foo_64x2(float64x2_t a, float64x2_t b, float64x2_t c) {
+ // CHECK: call <2 x double> @llvm.aarch64.neon.vcmla.rot0.v2f64
+  float64x2_t result = vcmlaq_f64(a, b, c);
+}
+
+void foo_rot90_64x2(float64x2_t a, float64x2_t b, float64x2_t c) {
+ // CHECK: call <2 x double> @llvm.aarch64.neon.vcmla.rot90.v2f64
+  float64x2_t result = vcmlaq_rot90_f64(a, b, c);
+}
+
+void foo_rot180_64x2(float64x2_t a, float64x2_t b, float64x2_t c) {
+ // CHECK: call <2 x double> @llvm.aarch64.neon.vcmla.rot180.v2f64
+  float64x2_t result = vcmlaq_rot180_f64(a, b, c);
+}
+
+void foo_rot270_64x2(float64x2_t a, float64x2_t b, float64x2_t c) {
+ // CHECK: call <2 x double> @llvm.aarch64.neon.vcmla.rot270.v2f64
+  float64x2_t result = vcmlaq_rot270_f64(a, b, c);
+}