[AArch64][SME] Support NEON vector to GPR integer moves in streaming mode

[c-rhodes]

Summary

A small subset of the NEON instruction set is legal in streaming mode.
This patch adds support for the following vector to integer move
instructions:

0x00 1110 0000 0001 0010 11xx xxxx xxxx # SMOV W|Xd,Vn.B[0]
0x00 1110 0000 0010 0010 11xx xxxx xxxx # SMOV W|Xd,Vn.H[0]
0100 1110 0000 0100 0010 11xx xxxx xxxx # SMOV Xd,Vn.S[0]
0000 1110 0000 0001 0011 11xx xxxx xxxx # UMOV Wd,Vn.B[0]
0000 1110 0000 0010 0011 11xx xxxx xxxx # UMOV Wd,Vn.H[0]
0000 1110 0000 0100 0011 11xx xxxx xxxx # UMOV Wd,Vn.S[0]
0100 1110 0000 1000 0011 11xx xxxx xxxx # UMOV Xd,Vn.D[0]

Only the zero index variants are legal, all others indexes are illegal.
To support this, new instructions are defined specifically for zero
index which is hardcoded, along an implicit 'VectorIndex0' operand.
Since the index operand is implicit and takes no bits in the encoding,
custom decoding is required to add the operand.

I'm not sure if this is the best approach but the predicate constraint
on a subset of an operand is unusual. Would be interested to hear some
alternatives.

The instructions are predicated on 'HasNEONorStreamingSVE', i.e. they're
enabled by either +neon or +streaming-sve. This follows on from the work
in D106272 to support the subset of SVE(2) instructions that are legal
in streaming mode.

Depends on D107902.

[c-rhodes]

Fix failing tests, UMOV aliases were using wrong index type, should have been VectorIndex0.

[c-rhodes]

ping

[sdesmalen]

llvm/test/MC/AArch64/SME/streaming-mode-neon.s

16

I was expecting streaming-sve or neon, or is the diagnostic arbitrary? (i.e. both are valid)

[c-rhodes]

llvm/test/MC/AArch64/SME/streaming-mode-neon.s

16

I was expecting streaming-sve or neon, or is the diagnostic arbitrary? (i.e. both are valid)

I expected the same but I think it's arbitrary, for the other NEON instructions that are legal in streaming mode but not for a specific variant like this the diagnostic is streaming-sve or neon.

llvm/lib/Target/AArch64/AArch64InstrFormats.td

class AsmVectorIndexOpnd<ValueType ty, AsmOperandClass mc>
   let PrintMethod = "printVectorIndex";
 }
 
 multiclass VectorIndex<ValueType ty, AsmOperandClass mc, code pred> {
   def "" : AsmVectorIndexOpnd<ty, mc>, ImmLeaf<ty, pred>;
   def _timm : AsmVectorIndexOpnd<ty, mc>, TImmLeaf<ty, pred>;
 }
 
+def VectorIndex0Operand : AsmVectorIndex<0, 0>;
 def VectorIndex1Operand : AsmVectorIndex<1, 1>;
 def VectorIndexBOperand : AsmVectorIndex<0, 15>;
 def VectorIndexHOperand : AsmVectorIndex<0, 7>;
 def VectorIndexSOperand : AsmVectorIndex<0, 3>;
 def VectorIndexDOperand : AsmVectorIndex<0, 1>;
 
+defm VectorIndex0 : VectorIndex<i64, VectorIndex0Operand,
+                                [{ return ((uint64_t)Imm) == 0; }]>;
 defm VectorIndex1 : VectorIndex<i64, VectorIndex1Operand,
                                 [{ return ((uint64_t)Imm) == 1; }]>;
 defm VectorIndexB : VectorIndex<i64, VectorIndexBOperand,
                                 [{ return ((uint64_t)Imm) < 16; }]>;
 defm VectorIndexH : VectorIndex<i64, VectorIndexHOperand,
                                 [{ return ((uint64_t)Imm) < 8; }]>;
 defm VectorIndexS : VectorIndex<i64, VectorIndexSOperand,
                                 [{ return ((uint64_t)Imm) < 4; }]>;

 
 class SIMDMovAlias<string asm, string size, Instruction inst,
                    RegisterClass regtype, Operand idxtype>
     : InstAlias<asm#"{\t$dst, $src"#size#"$idx" #
                     "|" # size # "\t$dst, $src$idx}",
                 (inst regtype:$dst, V128:$src, idxtype:$idx)>;
 
 multiclass SMov {
+  // SMOV with vector index of 0 are legal in Scalable Matrix Extension (SME)
+  // streaming mode.
+  let Predicates = [HasNEONorStreamingSVE] in {
+    def vi8to32_idx0 : SIMDSMov<0, ".b", GPR32, VectorIndex0> {
+      let Inst{20-16} = 0b00001;
+    }
+    def vi8to64_idx0 : SIMDSMov<1, ".b", GPR64, VectorIndex0> {
+      let Inst{20-16} = 0b00001;
+    }
+    def vi16to32_idx0 : SIMDSMov<0, ".h", GPR32, VectorIndex0> {
+      let Inst{20-16} = 0b00010;
+    }
+    def vi16to64_idx0 : SIMDSMov<1, ".h", GPR64, VectorIndex0> {
+      let Inst{20-16} = 0b00010;
+    }
+    def vi32to64_idx0 : SIMDSMov<1, ".s", GPR64, VectorIndex0> {
+      let Inst{20-16} = 0b00100;
+    }
+  }
   def vi8to32 : SIMDSMov<0, ".b", GPR32, VectorIndexB> {
     bits<4> idx;
     let Inst{20-17} = idx;
     let Inst{16} = 1;
   }
   def vi8to64 : SIMDSMov<1, ".b", GPR64, VectorIndexB> {
     bits<4> idx;
     let Inst{20-17} = idx;
multiclass SMov {
   def vi32to64 : SIMDSMov<1, ".s", GPR64, VectorIndexS> {
     bits<2> idx;
     let Inst{20-19} = idx;
     let Inst{18-16} = 0b100;
   }
 }
 
 multiclass UMov {
+  // UMOV with vector index of 0 are legal in Scalable Matrix Extension (SME)
+  // streaming mode.
+  let Predicates = [HasNEONorStreamingSVE] in {
+    def vi8_idx0 : SIMDUMov<0, ".b", v16i8, GPR32, VectorIndex0> {
+      let Inst{20-16} = 0b00001;
+    }
+    def vi16_idx0 : SIMDUMov<0, ".h", v8i16, GPR32, VectorIndex0> {
+      let Inst{20-16} = 0b00010;
+    }
+    def vi32_idx0 : SIMDUMov<0, ".s", v4i32, GPR32, VectorIndex0> {
+      let Inst{20-16} = 0b00100;
+    }
+    def vi64_idx0 : SIMDUMov<1, ".d", v2i64, GPR64, VectorIndex0> {
+      let Inst{20-16} = 0b01000;
+    }
+    def : SIMDMovAlias<"mov", ".s",
+                       !cast<Instruction>(NAME # vi32_idx0),
+                       GPR32, VectorIndex0>;
+    def : SIMDMovAlias<"mov", ".d",
+                       !cast<Instruction>(NAME # vi64_idx0),
+                       GPR64, VectorIndex0>;
+  }
   def vi8 : SIMDUMov<0, ".b", v16i8, GPR32, VectorIndexB> {
     bits<4> idx;
     let Inst{20-17} = idx;
     let Inst{16} = 1;
   }
   def vi16 : SIMDUMov<0, ".h", v8i16, GPR32, VectorIndexH> {
     bits<3> idx;
     let Inst{20-18} = idx;

llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp

return Error(Loc, "immediate must be an integer in range [-128, 255]"
                       " with a shift amount of 0");
   case Match_InvalidSVECpyImm16:
     return Error(Loc, "immediate must be an integer in range [-128, 127] or a "
                       "multiple of 256 in range [-32768, 65280]");
   case Match_InvalidSVECpyImm32:
   case Match_InvalidSVECpyImm64:
     return Error(Loc, "immediate must be an integer in range [-128, 127] or a "
                       "multiple of 256 in range [-32768, 32512]");
+  case Match_InvalidIndexRange0_0:
+    return Error(Loc, "expected lane specifier '[0]'");
   case Match_InvalidIndexRange1_1:
     return Error(Loc, "expected lane specifier '[1]'");
   case Match_InvalidIndexRange0_15:
     return Error(Loc, "vector lane must be an integer in range [0, 15].");
   case Match_InvalidIndexRange0_7:
     return Error(Loc, "vector lane must be an integer in range [0, 7].");
   case Match_InvalidIndexRange0_3:
     return Error(Loc, "vector lane must be an integer in range [0, 3].");
bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSVEAddSubImm8:
   case Match_InvalidSVEAddSubImm16:
   case Match_InvalidSVEAddSubImm32:
   case Match_InvalidSVEAddSubImm64:
   case Match_InvalidSVECpyImm8:
   case Match_InvalidSVECpyImm16:
   case Match_InvalidSVECpyImm32:
   case Match_InvalidSVECpyImm64:
+  case Match_InvalidIndexRange0_0:
   case Match_InvalidIndexRange1_1:
   case Match_InvalidIndexRange0_15:
   case Match_InvalidIndexRange0_7:
   case Match_InvalidIndexRange0_3:
   case Match_InvalidIndexRange0_1:
   case Match_InvalidSVEIndexRange0_63:
   case Match_InvalidSVEIndexRange0_31:
   case Match_InvalidSVEIndexRange0_15:

llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp

case AArch64::INSERT_MXIPZ_V_B:
       MI.insert(MI.begin(), MCOperand::createReg(AArch64::ZAB0));
       break;
     case AArch64::EXTRACT_ZPMXI_H_B:
     case AArch64::EXTRACT_ZPMXI_V_B:
       // MOVA <Zd>.B, <Pg>/M, ZA0<HV>.B[<Ws>, <imm>]
       //                      ^ insert implicit 8-bit element tile
       MI.insert(MI.begin()+2, MCOperand::createReg(AArch64::ZAB0));
       break;
+    case AArch64::SMOVvi8to32_idx0:
+    case AArch64::SMOVvi8to64_idx0:
+    case AArch64::SMOVvi16to32_idx0:
+    case AArch64::SMOVvi16to64_idx0:
+    case AArch64::SMOVvi32to64_idx0:
+    case AArch64::UMOVvi8_idx0:
+    case AArch64::UMOVvi16_idx0:
+    case AArch64::UMOVvi32_idx0:
+    case AArch64::UMOVvi64_idx0:
+      MI.addOperand(MCOperand::createImm(0));
+      break;
     }
 
     if (Result != MCDisassembler::Fail)
       return Result;
   }
 
   return MCDisassembler::Fail;
 }

llvm/test/MC/AArch64/SME/streaming-mode-neon-negative.s

 
 // ------------------------------------------------------------------------- //
 // Check non-scalar v8.6a BFloat16 instructions are illegal in streaming mode
 
 bfcvtn v5.4h, v5.4s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
 // CHECK-NEXT: bfcvtn v5.4h, v5.4s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// Check non-zero index is illegal in streaming mode
+// ------------------------------------------------------------------------- //
+// SMOV 8-bit to 32-bit
+
+smov w0, v0.b[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.b[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov w0, v0.b[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.b[7]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov w0, v0.b[15]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.b[15]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// SMOV 8-bit to 64-bit
+
+smov x0, v0.b[2]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.b[2]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.b[6]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.b[6]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.b[12]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.b[12]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// SMOV 16-bit to 32-bit
+
+smov w0, v0.h[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.h[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov w0, v0.h[3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.h[3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov w0, v0.h[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov w0, v0.h[7]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// SMOV 16-bit to 64-bit
+
+smov x0, v0.h[2]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.h[2]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.h[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.h[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.h[6]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.h[6]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// SMOV 32-bit to 64-bit
+
+smov x0, v0.s[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.s[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.s[2]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.s[2]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+smov x0, v0.s[3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: smov x0, v0.s[3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// UMOV 8-bit to 32-bit
+
+umov w0, v0.b[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.b[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.b[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.b[7]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.b[15]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.b[15]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// UMOV 16-bit to 32-bit
+
+umov w0, v0.h[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.h[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.h[3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.h[3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.h[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.h[7]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// UMOV 32-bit to 32-bit
+
+umov w0, v0.s[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.s[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.s[2]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.s[2]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+umov w0, v0.s[3]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov w0, v0.s[3]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+// ------------------------------------------------------------------------- //
+// UMOV 64-bit to 64-bit
+
+umov x0, v0.d[1]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction requires: neon
+// CHECK-NEXT: umov x0, v0.d[1]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:

llvm/test/MC/AArch64/SME/streaming-mode-neon.s

 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+streaming-sve < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=-neon < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+streaming-sve < %s \
 // RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-INST
 // Disassemble encoding and check the re-encoding (-show-encoding) matches.
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+streaming-sve < %s \
 // RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
 // RUN:        | llvm-mc -triple=aarch64 -mattr=+streaming-sve -disassemble -show-encoding \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // Scalar FP instructions
 
 fmulx s0, s1, s2
 // CHECK-INST: fmulx s0, s1, s2
 // CHECK-ENCODING: [0x20,0xdc,0x22,0x5e]
 // CHECK-ERROR: instruction requires: streaming-sve or neon
 
 fmulx d0, d1, d2
 // CHECK-INST: fmulx d0, d1, d2
 // CHECK-ENCODING: [0x20,0xdc,0x62,0x5e]
 // CHECK-ERROR: instruction requires: streaming-sve or neon
 

 // CHECK-INST: frsqrte s0, s1
 // CHECK-ENCODING: [0x20,0xd8,0xa1,0x7e]
 // CHECK-ERROR: instruction requires: streaming-sve or neon
 
 frsqrte d0, d1
 // CHECK-INST: frsqrte d0, d1
 // CHECK-ENCODING: [0x20,0xd8,0xe1,0x7e]
 // CHECK-ERROR: instruction requires: streaming-sve or neon
+
+// Vector to GPR integer move instructions
+
+smov w0, v0.b[0]
+// CHECK-INST: smov w0, v0.b[0]
+// CHECK-ENCODING: [0x00,0x2c,0x01,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+smov x0, v0.b[0]
+// CHECK-INST: smov x0, v0.b[0]
+// CHECK-ENCODING: [0x00,0x2c,0x01,0x4e]
+// CHECK-ERROR: instruction requires: neon
+
+smov w0, v0.h[0]
+// CHECK-INST: smov w0, v0.h[0]
+// CHECK-ENCODING: [0x00,0x2c,0x02,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+smov x0, v0.h[0]
+// CHECK-INST: smov x0, v0.h[0]
+// CHECK-ENCODING: [0x00,0x2c,0x02,0x4e]
+// CHECK-ERROR: instruction requires: neon
+
+smov x0, v0.s[0]
+// CHECK-INST: smov x0, v0.s[0]
+// CHECK-ENCODING: [0x00,0x2c,0x04,0x4e]
+// CHECK-ERROR: instruction requires: neon
+
+umov w0, v0.b[0]
+// CHECK-INST: umov w0, v0.b[0]
+// CHECK-ENCODING: [0x00,0x3c,0x01,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+umov w0, v0.h[0]
+// CHECK-INST: umov w0, v0.h[0]
+// CHECK-ENCODING: [0x00,0x3c,0x02,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+umov w0, v0.s[0]
+// CHECK-INST: mov w0, v0.s[0]
+// CHECK-ENCODING: [0x00,0x3c,0x04,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+umov x0, v0.d[0]
+// CHECK-INST: mov x0, v0.d[0]
+// CHECK-ENCODING: [0x00,0x3c,0x08,0x4e]
+// CHECK-ERROR: instruction requires: neon
+
+// Aliases
+
+mov w0, v0.s[0]
+// CHECK-INST: mov w0, v0.s[0]
+// CHECK-ENCODING: [0x00,0x3c,0x04,0x0e]
+// CHECK-ERROR: instruction requires: neon
+
+mov x0, v0.d[0]
+// CHECK-INST: mov x0, v0.d[0]
+// CHECK-ENCODING: [0x00,0x3c,0x08,0x4e]
+// CHECK-ERROR: instruction requires: neon