[AArch64] Add nvcast patterns for v4f16 and v8f16

[pirama]

Summary

Constant stores of f16 vectors can create NvCast nodes from various
operand types to v4f16 or v8f16 depending on patterns in the stored
constants. This patch adds nvcast rules with v4f16 and v8f16 values.

AArchISelLowering::LowerBUILD_VECTOR has the details on which constant
patterns generate the nvcast nodes.

[pirama]

According to AArchISelLowering::LowerBUILD_VECTOR, nvcast from v2f32, v4f32, v2f64 can also be generated. However, I couldn't create the correct constant pattern that creates these ops.

For completeness, I can add patterns for v2f32 to v4f16 and also v4f32 and v2f64 to v8f16, but I might need some assistance adding tests for them, though.

[jmolloy]

LGTM

[ab]

test/CodeGen/AArch64/fp16-vector-nvcast.ll

6

Note that you can add the nounwind attribute to the function (in a group) to avoid matching those pesky CFI directives.

[pirama]

Clean up test by setting nounwind attribute to functions.

lib/Target/AArch64/AArch64InstrInfo.td

 // Most bitconverts require some sort of conversion. The only exceptions are:
 //   a) Identity conversions -  vNfX <-> vNiX
 //   b) Single-lane-to-scalar - v1fX <-> fX or v1iX <-> iX
 //
 
 // Natural vector casts (64 bit)
 def : Pat<(v8i8 (AArch64NvCast (v2i32 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v2i32 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v2i32 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v2i32 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v2f32 (AArch64NvCast (v2i32 FPR64:$src))), (v2f32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v2i32 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (v4i16 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v4i16 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v4i16 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v4i16 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v4i16 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (v8i8 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v8i8 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (v8i8 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v8i8 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v8i8 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (f64 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (f64 FPR64:$src))), (v4i16 FPR64:$src)>;
+def : Pat<(v4f16 (AArch64NvCast (f64 FPR64:$src))), (v4f16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (f64 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v2f32 (AArch64NvCast (f64 FPR64:$src))), (v2f32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (f64 FPR64:$src))), (v1i64 FPR64:$src)>;
 def : Pat<(v1f64 (AArch64NvCast (f64 FPR64:$src))), (v1f64 FPR64:$src)>;
 
 def : Pat<(v8i8 (AArch64NvCast (v2f32 FPR64:$src))), (v8i8 FPR64:$src)>;
 def : Pat<(v4i16 (AArch64NvCast (v2f32 FPR64:$src))), (v4i16 FPR64:$src)>;
 def : Pat<(v2i32 (AArch64NvCast (v2f32 FPR64:$src))), (v2i32 FPR64:$src)>;
 def : Pat<(v2f32 (AArch64NvCast (v2f32 FPR64:$src))), (v2f32 FPR64:$src)>;
 def : Pat<(v1i64 (AArch64NvCast (v2f32 FPR64:$src))), (v1i64 FPR64:$src)>;
 
 // Natural vector casts (128 bit)
 def : Pat<(v16i8 (AArch64NvCast (v4i32 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v4i32 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v4i32 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v4i32 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v4f32 (AArch64NvCast (v4i32 FPR128:$src))), (v4f32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v4i32 FPR128:$src))), (v2i64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v8i16 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v8i16 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v8i16 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v8i16 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v8i16 FPR128:$src))), (v2i64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v16i8 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v16i8 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v16i8 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v16i8 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v16i8 FPR128:$src))), (v2i64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v2i64 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v2i64 FPR128:$src))), (v8i16 FPR128:$src)>;
+def : Pat<(v8f16 (AArch64NvCast (v2i64 FPR128:$src))), (v8f16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v2i64 FPR128:$src))), (v4i32 FPR128:$src)>;
 def : Pat<(v2i64 (AArch64NvCast (v2i64 FPR128:$src))), (v2i64 FPR128:$src)>;
 def : Pat<(v4f32 (AArch64NvCast (v2i64 FPR128:$src))), (v4f32 FPR128:$src)>;
 def : Pat<(v2f64 (AArch64NvCast (v2i64 FPR128:$src))), (v2f64 FPR128:$src)>;
 
 def : Pat<(v16i8 (AArch64NvCast (v4f32 FPR128:$src))), (v16i8 FPR128:$src)>;
 def : Pat<(v8i16 (AArch64NvCast (v4f32 FPR128:$src))), (v8i16 FPR128:$src)>;
 def : Pat<(v4i32 (AArch64NvCast (v4f32 FPR128:$src))), (v4i32 FPR128:$src)>;

test/CodeGen/AArch64/fp16-vector-nvcast.ll

+; RUN: llc < %s -asm-verbose=false -mtriple=aarch64-none-eabi | FileCheck %s
+
+; Test pattern (v4f16 (AArch64NvCast (v2i32 FPR64:$src)))
+define void @nvcast_v2i32(<4 x half>* %a) {
+; CHECK-LABEL: nvcast_v2i32:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].2s, #0xab, lsl #16
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (v4i16 FPR64:$src)))
+define void @nvcast_v4i16(<4 x half>* %a) {
+; CHECK-LABEL: nvcast_v4i16:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].4h, #0xab
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (v8i8 FPR64:$src)))
+define void @nvcast_v8i8(<4 x half>* %a) {
+; CHECK-LABEL: nvcast_v8i8:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].8b, #0xab
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> <half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB>, <4 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v4f16 (AArch64NvCast (f64 FPR64:$src)))
+define void @nvcast_f64(<4 x half>* %a) {
+; CHECK-LABEL: nvcast_f64:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi d[[REG:[0-9]+]], #0000000000000000
+; CHECK-NEXT: str d[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <4 x half> zeroinitializer, <4 x half>* %a
+  ret void
+}
+
+; Test pattern (v8f16 (AArch64NvCast (v4i32 FPR128:$src)))
+define void @nvcast_v4i32(<8 x half>* %a) {
+; CHECK-LABEL: nvcast_v4i32:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].4s, #0xab, lsl #16
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB, half 0xH0000, half 0xH00AB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v8i16 FPR128:$src)))
+define void @nvcast_v8i16(<8 x half>* %a) {
+; CHECK-LABEL: nvcast_v8i16:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].8h, #0xab
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB, half 0xH00AB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v16i8 FPR128:$src)))
+define void @nvcast_v16i8(<8 x half>* %a) {
+; CHECK-LABEL: nvcast_v16i8:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].16b, #0xab
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> <half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB, half 0xHABAB>, <8 x half>* %a
+  ret void
+}
+
+
+; Test pattern (v8f16 (AArch64NvCast (v2i64 FPR128:$src)))
+define void @nvcast_v2i64(<8 x half>* %a) {
+; CHECK-LABEL: nvcast_v2i64:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: movi v[[REG:[0-9]+]].2d, #0000000000000000
+; CHECK-NEXT: str q[[REG]], [x0]
+; CHECK-NEXT: ret
+  store volatile <8 x half> zeroinitializer, <8 x half>* %a
+  ret void
+}