[AArch64] Adding the v8.7-A LD64B/ST64B Accelerator extension

[pratlucas]

Summary

This adds support for the v8.7-A LD64B/ST64B Accelerator extension
through a subtarget feature called "ls64". It adds four 64-byte
load/store instructions with an operand in the new GPR64x8 register
class, and one system register that's part of the same extension.

Based on patches written by Simon Tatham.

[ostannard]

LGTM with one minor comment.

llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp

476

This looks like it belongs in D91774.

[pratlucas]

Rebasing.

llvm/lib/Target/AArch64/AArch64.td

def FeatureXS : SubtargetFeature<"xs", "HasXS",
     "true", "Enable Armv8.7-A limited-TLB-maintenance instruction">;
 
 def FeatureWFxT : SubtargetFeature<"wfxt", "HasWFxT",
     "true", "Enable Armv8.7-A WFET and WFIT instruction">;
 
 def FeatureHCX : SubtargetFeature<
     "hcx", "HasHCX", "true", "Enable Armv8.7-A HCRX_EL2 system register">;
 
+def FeatureLS64 : SubtargetFeature<"ls64", "HasLS64",
+    "true", "Enable Armv8.7-A LD64B/ST64B Accelerator Extension">;
+
 def FeatureFineGrainedTraps : SubtargetFeature<"fgt", "HasFineGrainedTraps",
     "true", "Enable fine grained virtualization traps extension">;
 
 def FeatureEnhancedCounterVirtualization :
       SubtargetFeature<"ecv", "HasEnhancedCounterVirtualization",
       "true", "Enable enhanced counter virtualization extension">;
 
 //===----------------------------------------------------------------------===//

llvm/lib/Target/AArch64/AArch64InstrFormats.td

multiclass STOPregister<string asm, string instr> {
   def : BaseSTOPregister<asm # "h",  GPR32, WZR,
                     !cast<Instruction>(instr # "H")>;
   def : BaseSTOPregister<asm,        GPR32, WZR,
                     !cast<Instruction>(instr # "W")>;
   def : BaseSTOPregister<asm,        GPR64, XZR,
                     !cast<Instruction>(instr # "X")>;
 }
 
+class LoadStore64B_base<bits<3> opc, string asm_inst, string asm_ops,
+                        dag iops, dag oops, list<dag> pat>
+    : I<oops, iops, asm_inst, asm_ops, "", pat>,
+      Sched<[]> /* FIXME: fill in scheduling details once known */ {
+  bits<5> Rt;
+  bits<5> Rn;
+  let Inst{31-21} = 0b11111000001;
+  let Inst{15}    = 1;
+  let Inst{14-12} = opc;
+  let Inst{11-10} = 0b00;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Rt;
+
+  let Predicates = [HasV8_7a];
+}
+
+class LoadStore64B<bits<3> opc, string asm_inst, dag iops, dag oops,
+                      list<dag> pat = []>
+    : LoadStore64B_base<opc, asm_inst, "\t$Rt, [$Rn]", iops, oops, pat> {
+  let Inst{20-16} = 0b11111;
+}
+
+class Store64BV<bits<3> opc, string asm_inst, list<dag> pat = []>
+    : LoadStore64B_base<opc, asm_inst, "\t$Rs, $Rt, [$Rn]",
+                       (ins GPR64x8:$Rt, GPR64sp:$Rn), (outs GPR64:$Rs), pat> {
+  bits<5> Rs;
+  let Inst{20-16} = Rs;
+}
+
 //----------------------------------------------------------------------------
 // Allow the size specifier tokens to be upper case, not just lower.
 def : TokenAlias<".4B", ".4b">;  // Add dot product
 def : TokenAlias<".8B", ".8b">;
 def : TokenAlias<".4H", ".4h">;
 def : TokenAlias<".2S", ".2s">;
 def : TokenAlias<".1D", ".1d">;
 def : TokenAlias<".16B", ".16b">;

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 def HasMatMulFP32    : Predicate<"Subtarget->hasMatMulFP32()">,
                        AssemblerPredicate<(all_of FeatureMatMulFP32), "f32mm">;
 def HasMatMulFP64    : Predicate<"Subtarget->hasMatMulFP64()">,
                        AssemblerPredicate<(all_of FeatureMatMulFP64), "f64mm">;
 def HasXS            : Predicate<"Subtarget->hasXS()">,
                        AssemblerPredicate<(all_of FeatureXS), "xs">;
 def HasWFxT          : Predicate<"Subtarget->hasWFxT()">,
                        AssemblerPredicate<(all_of FeatureWFxT), "wfxt">;
+def HasLS64          : Predicate<"Subtarget->hasLS64()">,
+                       AssemblerPredicate<(all_of FeatureLS64), "ls64">;
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;
 def IsWindows        : Predicate<"Subtarget->isTargetWindows()">;
 def UseExperimentalZeroingPseudos
     : Predicate<"Subtarget->useExperimentalZeroingPseudos()">;
 def UseAlternateSExtLoadCVTF32
     : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">;
 
let AddedComplexity = 10 in {
   def : dot_v8i8<UDOTv8i8, AArch64umull, zext>;
   def : dot_v16i8<SDOTv16i8, AArch64smull, sext>;
   def : dot_v16i8<UDOTv16i8, AArch64umull, zext>;
 
   // FIXME: add patterns to generate vector by element dot product.
   // FIXME: add SVE dot-product patterns.
 }
 
+let Predicates = [HasLS64] in {
+  def LD64B: LoadStore64B<0b101, "ld64b", (ins GPR64sp:$Rn),
+                                          (outs GPR64x8:$Rt)>;
+  def ST64B: LoadStore64B<0b001, "st64b", (ins GPR64sp:$Rn, GPR64x8:$Rt),
+                                          (outs)>;
+  def ST64BV:   Store64BV<0b011, "st64bv">;
+  def ST64BV0:  Store64BV<0b010, "st64bv0">;
+}
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
 
 include "AArch64InstrGISel.td"

llvm/lib/Target/AArch64/AArch64Subtarget.h

protected:
   bool HasAMVS = false;
   bool HasFineGrainedTraps = false;
   bool HasEnhancedCounterVirtualization = false;
 
   // Armv8.7-A Extensions
   bool HasXS = false;
   bool HasWFxT = false;
   bool HasHCX = false;
+  bool HasLS64 = false;
 
   // Arm SVE2 extensions
   bool HasSVE2 = false;
   bool HasSVE2AES = false;
   bool HasSVE2SM4 = false;
   bool HasSVE2SHA3 = false;
   bool HasSVE2BitPerm = false;
 
public:
   bool hasMPAM() const { return HasMPAM; }
   bool hasDIT() const { return HasDIT; }
   bool hasTRACEV8_4() const { return HasTRACEV8_4; }
   bool hasAM() const { return HasAM; }
   bool hasAMVS() const { return HasAMVS; }
   bool hasXS() const { return HasXS; }
   bool hasWFxT() const { return HasWFxT; }
   bool hasHCX() const { return HasHCX; }
+  bool hasLS64() const { return HasLS64; }
   bool hasSEL2() const { return HasSEL2; }
   bool hasPMU() const { return HasPMU; }
   bool hasTLB_RMI() const { return HasTLB_RMI; }
   bool hasFMI() const { return HasFMI; }
   bool hasRCPC_IMMO() const { return HasRCPC_IMMO; }
 
   bool addrSinkUsingGEPs() const override {
     // Keeping GEPs inbounds is important for exploiting AArch64

llvm/lib/Target/AArch64/AArch64SystemOperands.td

 def : RWSysReg<"CNTSCALE_EL2",     0b11, 0b100, 0b1110, 0b0000, 0b100>;
 def : RWSysReg<"CNTISCALE_EL2",    0b11, 0b100, 0b1110, 0b0000, 0b101>;
 def : RWSysReg<"CNTPOFF_EL2",      0b11, 0b100, 0b1110, 0b0000, 0b110>;
 def : RWSysReg<"CNTVFRQ_EL2",      0b11, 0b100, 0b1110, 0b0000, 0b111>;
 def : RWSysReg<"CNTPCTSS_EL0",     0b11, 0b011, 0b1110, 0b0000, 0b101>;
 def : RWSysReg<"CNTVCTSS_EL0",     0b11, 0b011, 0b1110, 0b0000, 0b110>;
 }
 
+// v8.7a LD64B/ST64B Accelerator Extension system register
+let Requires = [{ {AArch64::FeatureLS64} }] in
+def : RWSysReg<"ACCDATA_EL1",       0b11, 0b000, 0b1101, 0b0000, 0b101>;
+
 // Cyclone specific system registers
 //                                 Op0    Op1     CRn     CRm    Op2
 let Requires = [{ {AArch64::ProcAppleA7} }] in
 def : RWSysReg<"CPM_IOACC_CTL_EL3", 0b11, 0b111, 0b1111, 0b0010, 0b000>;

llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp

static const unsigned GPR64x8DecoderTable[] = {
   AArch64::X6_X7_X8_X9_X10_X11_X12_X13,
   AArch64::X8_X9_X10_X11_X12_X13_X14_X15,
   AArch64::X10_X11_X12_X13_X14_X15_X16_X17,
   AArch64::X12_X13_X14_X15_X16_X17_X18_X19,
   AArch64::X14_X15_X16_X17_X18_X19_X20_X21,
   AArch64::X16_X17_X18_X19_X20_X21_X22_X23,
   AArch64::X18_X19_X20_X21_X22_X23_X24_X25,
   AArch64::X20_X21_X22_X23_X24_X25_X26_X27,
+  AArch64::X22_X23_X24_X25_X26_X27_X28_FP,
 };
 
 static DecodeStatus DecodeGPR64x8ClassRegisterClass(MCInst &Inst,
                                                     unsigned RegNo,
                                                     uint64_t Address,
                                                     const void *Decoder) {
   if (RegNo > 22)
     return Fail;

llvm/test/MC/AArch64/armv8.7a-ls64.s

+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+ls64 < %s 2>%t | FileCheck %s
+// RUN: FileCheck --check-prefix=CHECK-ERR --check-prefix=CHECK-LS64-ERR %s < %t
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-ERR --check-prefix=CHECK-NO-LS64-ERR %s < %t
+
+  ld64b x0, [x13]
+  st64b x14, [x13]
+  st64bv x1, x20, [x13]
+  st64bv0 x1, x22, [x13]
+// CHECK: ld64b x0, [x13]        // encoding: [0xa0,0xd1,0x3f,0xf8]
+// CHECK: st64b x14, [x13]       // encoding: [0xae,0x91,0x3f,0xf8]
+// CHECK: st64bv x1, x20, [x13]  // encoding: [0xb4,0xb1,0x21,0xf8]
+// CHECK: st64bv0 x1, x22, [x13] // encoding: [0xb6,0xa1,0x21,0xf8]
+// CHECK-NO-LS64-ERR: [[@LINE-8]]:3: error: instruction requires: ls64
+// CHECK-NO-LS64-ERR: [[@LINE-8]]:3: error: instruction requires: ls64
+// CHECK-NO-LS64-ERR: [[@LINE-8]]:3: error: instruction requires: ls64
+// CHECK-NO-LS64-ERR: [[@LINE-8]]:3: error: instruction requires: ls64
+
+  ld64b x0, [sp]
+  st64b x14, [sp]
+  st64bv x1, x20, [sp]
+  st64bv0 x1, x22, [sp]
+// CHECK: ld64b x0, [sp]         // encoding: [0xe0,0xd3,0x3f,0xf8]
+// CHECK: st64b x14, [sp]        // encoding: [0xee,0x93,0x3f,0xf8]
+// CHECK: st64bv x1, x20, [sp]   // encoding: [0xf4,0xb3,0x21,0xf8]
+// CHECK: st64bv0 x1, x22, [sp]  // encoding: [0xf6,0xa3,0x21,0xf8]
+
+  ld64b x1, [x13]
+  ld64b x24, [x13]
+// CHECK-ERR: [[@LINE-2]]:9: error: expected an even-numbered x-register in the range [x0,x22]
+// CHECK-ERR: [[@LINE-2]]:9: error: expected an even-numbered x-register in the range [x0,x22]
+
+  mrs x0, accdata_el1
+  msr accdata_el1, x0
+// CHECK: mrs x0, ACCDATA_EL1    // encoding: [0xa0,0xd0,0x38,0xd5]
+// CHECK: msr ACCDATA_EL1, x0    // encoding: [0xa0,0xd0,0x18,0xd5]
+// CHECK-NO-LS64-ERR: [[@LINE-4]]:11: error: expected readable system register
+// CHECK-NO-LS64-ERR: [[@LINE-4]]:7: error: expected writable system register

llvm/test/MC/Disassembler/AArch64/armv8.7a-ls64.txt

+# RUN: not llvm-mc -triple=aarch64 -mattr=+ls64 -disassemble %s 2> %t | FileCheck %s
+# RUN: FileCheck --check-prefix=CHECK-ERR %s < %t
+# RUN: not llvm-mc -triple=aarch64 -disassemble %s 2> %t | FileCheck --check-prefix=CHECK-NO-LS64 %s
+# RUN: FileCheck --check-prefix=CHECK-ERR --check-prefix=CHECK-NO-LS64-ERR %s < %t
+
+[0xa0,0xd1,0x3f,0xf8]
+[0xae,0x91,0x3f,0xf8]
+[0xb4,0xb1,0x21,0xf8]
+[0xb6,0xa1,0x21,0xf8]
+# CHECK: ld64b x0, [x13]
+# CHECK: st64b x14, [x13]
+# CHECK: st64bv x1, x20, [x13]
+# CHECK: st64bv0 x1, x22, [x13]
+# CHECK-NO-LS64-ERR: [[@LINE-8]]:2: warning: invalid instruction encoding
+# CHECK-NO-LS64-ERR: [[@LINE-8]]:2: warning: invalid instruction encoding
+# CHECK-NO-LS64-ERR: [[@LINE-8]]:2: warning: invalid instruction encoding
+# CHECK-NO-LS64-ERR: [[@LINE-8]]:2: warning: invalid instruction encoding
+
+[0xe0,0xd3,0x3f,0xf8]
+[0xee,0x93,0x3f,0xf8]
+[0xf4,0xb3,0x21,0xf8]
+[0xf6,0xa3,0x21,0xf8]
+# CHECK: ld64b x0, [sp]
+# CHECK: st64b x14, [sp]
+# CHECK: st64bv x1, x20, [sp]
+# CHECK: st64bv0 x1, x22, [sp]
+
+[0xb3,0xd1,0x3f,0xf8]
+[0xb8,0xd1,0x3f,0xf8]
+# CHECK-ERR: [[@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-ERR: [[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xa0,0xd0,0x38,0xd5]
+[0xa0,0xd0,0x18,0xd5]
+# CHECK: mrs x0, ACCDATA_EL1
+# CHECK: msr ACCDATA_EL1, x0
+# CHECK-NO-LS64: mrs x0, S3_0_C13_C0_5
+# CHECK-NO-LS64: msr S3_0_C13_C0_5, x0