[AArch64][v8.5A] Add prediction invalidation instructions to AArch64

[olista01]

Summary

This adds new system instructions which act as barriers to speculative
execution based on earlier execution within a particular execution
context.

Patch by Pablo Barrio!

[t.p.northover]

This looks fine.

lib/Target/AArch64/AArch64.td

 
 def FeatureFRIntNNT : SubtargetFeature<"fptoint", "HasFRIntNNT", "true",
   "Enable FRInt[32|64][Z|X] instructions that round a floating-point number to "
   "an integer (in FP format) forcing it to fit into a 32- or 64-bit int" >;
 
 def FeatureSpecCtrl : SubtargetFeature<"specctrl", "HasSpecCtrl", "true",
   "Enable speculation control barrier" >;
 
+def FeaturePredCtrl : SubtargetFeature<"predctrl", "HasPredCtrl", "true",
+  "Enable execution and data prediction invalidation instructions" >;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
 
 def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true",
   "Support ARM v8.1a instructions", [FeatureCRC, FeatureLSE, FeatureRDM]>;
 
 def HasV8_2aOps : SubtargetFeature<"v8.2a", "HasV8_2aOps", "true",
   "Support ARM v8.2a instructions", [HasV8_1aOps, FeatureRAS]>;
 
 def HasV8_3aOps : SubtargetFeature<"v8.3a", "HasV8_3aOps", "true",
   "Support ARM v8.3a instructions", [HasV8_2aOps, FeatureRCPC]>;
 
 def HasV8_4aOps : SubtargetFeature<"v8.4a", "HasV8_4aOps", "true",
   "Support ARM v8.4a instructions", [HasV8_3aOps, FeatureDotProd]>;
 
 def HasV8_5aOps : SubtargetFeature<
   "v8.5a", "HasV8_5aOps", "true", "Support ARM v8.5a instructions",
-  [HasV8_4aOps, FeatureAltFPCmp, FeatureFRIntNNT, FeatureSpecCtrl]
+  [HasV8_4aOps, FeatureAltFPCmp, FeatureFRIntNNT, FeatureSpecCtrl,
+   FeaturePredCtrl]
 >;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
 //===----------------------------------------------------------------------===//
 
 include "AArch64RegisterInfo.td"
 include "AArch64RegisterBanks.td"

lib/Target/AArch64/AArch64InstrInfo.td

 def HasRCPC          : Predicate<"Subtarget->hasRCPC()">,
                                  AssemblerPredicate<"FeatureRCPC", "rcpc">;
 def HasAltNZCV       : Predicate<"Subtarget->hasAlternativeNZCV()">,
                        AssemblerPredicate<"FeatureAltFPCmp", "altnzcv">;
 def HasFRIntNNT      : Predicate<"Subtarget->hasFRIntNNT()">,
                        AssemblerPredicate<"FeatureFRIntNNT", "fptoint">;
 def HasSpecCtrl      : Predicate<"Subtarget->hasSpecCtrl()">,
                        AssemblerPredicate<"FeatureSpecCtrl", "specctrl">;
+def HasPredCtrl      : Predicate<"Subtarget->hasPredCtrl()">,
+                       AssemblerPredicate<"FeaturePredCtrl", "predctrl">;
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;
 def UseAlternateSExtLoadCVTF32
     : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">;
 
 def UseNegativeImmediates
     : Predicate<"false">, AssemblerPredicate<"!FeatureNoNegativeImmediates",
                                              "NegativeImmediates">;

lib/Target/AArch64/AArch64Subtarget.h

protected:
   bool HasSVE = false;
   bool HasRCPC = false;
   bool HasAggressiveFMA = false;
 
   // Armv8.5-A Extensions
   bool HasAlternativeNZCV = false;
   bool HasFRIntNNT = false;
   bool HasSpecCtrl = false;
+  bool HasPredCtrl = false;
 
   // HasZeroCycleRegMove - Has zero-cycle register mov instructions.
   bool HasZeroCycleRegMove = false;
 
   // HasZeroCycleZeroing - Has zero-cycle zeroing instructions.
   bool HasZeroCycleZeroing = false;
   bool HasZeroCycleZeroingFPWorkaround = false;
 
public:
   bool hasSPE() const { return HasSPE; }
   bool hasLSLFast() const { return HasLSLFast; }
   bool hasSVE() const { return HasSVE; }
   bool hasRCPC() const { return HasRCPC; }
   bool hasAggressiveFMA() const { return HasAggressiveFMA; }
   bool hasAlternativeNZCV() const { return HasAlternativeNZCV; }
   bool hasFRIntNNT() { return HasFRIntNNT; }
   bool hasSpecCtrl() { return HasSpecCtrl; }
+  bool hasPredCtrl() { return HasPredCtrl; }
 
   bool isLittleEndian() const { return IsLittle; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
   bool isTargetIOS() const { return TargetTriple.isiOS(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
   bool isTargetWindows() const { return TargetTriple.isOSWindows(); }
   bool isTargetAndroid() const { return TargetTriple.isAndroid(); }

lib/Target/AArch64/AArch64SystemOperands.td

 def : TLBI<"RVAE3",        0b110, 0b1000, 0b0110, 0b001>;
 def : TLBI<"RVALE3",       0b110, 0b1000, 0b0110, 0b101>;
 def : TLBI<"RVAE3IS",      0b110, 0b1000, 0b0010, 0b001>;
 def : TLBI<"RVALE3IS",     0b110, 0b1000, 0b0010, 0b101>;
 def : TLBI<"RVAE3OS",      0b110, 0b1000, 0b0101, 0b001>;
 def : TLBI<"RVALE3OS",     0b110, 0b1000, 0b0101, 0b101>;
 }
 
+// Armv8.5-A Prediction Restriction by Context instruction options:
+class PRCTX<string name, bits<4> crm> : SearchableTable {
+  let SearchableFields = ["Name", "Encoding"];
+  let EnumValueField = "Encoding";
+
+  string Name = name;
+  bits<11> Encoding;
+  let Encoding{10-4} = 0b0110111;
+  let Encoding{3-0} = crm;
+  bit NeedsReg = 1;
+  code Requires = [{ {} }];
+}
+
+let Requires = [{ {AArch64::FeaturePredCtrl} }] in {
+def : PRCTX<"RCTX", 0b0011>;
+}
+
 //===----------------------------------------------------------------------===//
 // MRS/MSR (system register read/write) instruction options.
 //===----------------------------------------------------------------------===//
 
 class SysReg<string name, bits<2> op0, bits<3> op1, bits<4> crn, bits<4> crm,
              bits<3> op2> : SearchableTable {
   let SearchableFields = ["Name", "Encoding"];
   let EnumValueField = "Encoding";

lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp

static const struct Extension {
   { "sha3", {AArch64::FeatureSHA3} },
   { "sha2", {AArch64::FeatureSHA2} },
   { "aes",  {AArch64::FeatureAES} },
   { "crypto", {AArch64::FeatureCrypto} },
   { "fp", {AArch64::FeatureFPARMv8} },
   { "simd", {AArch64::FeatureNEON} },
   { "ras", {AArch64::FeatureRAS} },
   { "lse", {AArch64::FeatureLSE} },
+  { "predctrl", {AArch64::FeaturePredCtrl} },
 
   // FIXME: Unsupported extensions
   { "pan", {} },
   { "lor", {} },
   { "rdma", {} },
   { "profile", {} },
 };
 
if (Mnemonic == "ic") {
     if (!TLBI)
       return TokError("invalid operand for TLBI instruction");
     else if (!TLBI->haveFeatures(getSTI().getFeatureBits())) {
       std::string Str("TLBI " + std::string(TLBI->Name) + " requires ");
       setRequiredFeatureString(TLBI->getRequiredFeatures(), Str);
       return TokError(Str.c_str());
     }
     createSysAlias(TLBI->Encoding, Operands, S);
+  } else if (Mnemonic == "cfp" || Mnemonic == "dvp" || Mnemonic == "cpp") {
+    const AArch64PRCTX::PRCTX *PRCTX = AArch64PRCTX::lookupPRCTXByName(Op);
+    if (!PRCTX)
+      return TokError("invalid operand for prediction restriction instruction");
+    else if (!PRCTX->haveFeatures(getSTI().getFeatureBits())) {
+      std::string Str(
+          Mnemonic.upper() + std::string(PRCTX->Name) + " requires ");
+      setRequiredFeatureString(PRCTX->getRequiredFeatures(), Str);
+      return TokError(Str.c_str());
+    }
+    uint16_t PRCTX_Op2 =
+      Mnemonic == "cfp" ? 4 :
+      Mnemonic == "dvp" ? 5 :
+      Mnemonic == "cpp" ? 7 :
+      0;
+    assert(PRCTX_Op2 && "Invalid mnemonic for prediction restriction instruction");
+    createSysAlias(PRCTX->Encoding << 3 | PRCTX_Op2 , Operands, S);
   }
 
   Parser.Lex(); // Eat operand.
 
   bool ExpectRegister = (Op.lower().find("all") == StringRef::npos);
   bool HasRegister = false;
 
   // Check for the optional register operand.
if (Parser.getTok().is(AsmToken::Identifier) &&
     // statement and don't need to match the 'instruction."
     return true;
   }
 
   // Create the leading tokens for the mnemonic, split by '.' characters.
   size_t Start = 0, Next = Name.find('.');
   StringRef Head = Name.slice(Start, Next);
 
-  // IC, DC, AT, and TLBI instructions are aliases for the SYS instruction.
-  if (Head == "ic" || Head == "dc" || Head == "at" || Head == "tlbi")
+  // IC, DC, AT, TLBI and Prediction invalidation instructions are aliases for
+  // the SYS instruction.
+  if (Head == "ic" || Head == "dc" || Head == "at" || Head == "tlbi" ||
+      Head == "cfp" || Head == "dvp" || Head == "cpp")
     return parseSysAlias(Head, NameLoc, Operands);
 
   Operands.push_back(
       AArch64Operand::CreateToken(Head, false, NameLoc, getContext()));
   Mnemonic = Head;
 
   // Handle condition codes for a branch mnemonic
   if (Head == "b" && Next != StringRef::npos) {

lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp

#endif
 
   bool NeedsReg;
   std::string Ins;
   std::string Name;
 
   if (CnVal == 7) {
     switch (CmVal) {
     default: return false;
+    // Maybe IC, maybe Prediction Restriction
+    case 1:
+      switch (Op1Val) {
+      default: return false;
+      case 0: goto Search_IC;
+      case 3: goto Search_PRCTX;
+      }
+    // Prediction Restriction aliases
+    case 3: {
+      Search_PRCTX:
+      const AArch64PRCTX::PRCTX *PRCTX = AArch64PRCTX::lookupPRCTXByEncoding(Encoding >> 3);
+      if (!PRCTX || !PRCTX->haveFeatures(STI.getFeatureBits()))
+        return false;
+
+      NeedsReg = PRCTX->NeedsReg;
+      switch (Op2Val) {
+      default: return false;
+      case 4: Ins = "cfp\t"; break;
+      case 5: Ins = "dvp\t"; break;
+      case 7: Ins = "cpp\t"; break;
+      }
+      Name = std::string(PRCTX->Name);
+    }
+    break;
     // IC aliases
-    case 1: case 5: {
+    case 5: {
+      Search_IC:
       const AArch64IC::IC *IC = AArch64IC::lookupICByEncoding(Encoding);
       if (!IC || !IC->haveFeatures(STI.getFeatureBits()))
         return false;
 
       NeedsReg = IC->NeedsReg;
       Ins = "ic\t";
       Name = std::string(IC->Name);
     }

lib/Target/AArch64/Utils/AArch64BaseInfo.h

 namespace AArch64TLBI {
   struct TLBI : SysAliasReg {
     using SysAliasReg::SysAliasReg;
   };
   #define GET_TLBI_DECL
   #include "AArch64GenSystemOperands.inc"
 }
 
+namespace AArch64PRCTX {
+  struct PRCTX : SysAliasReg {
+    using SysAliasReg::SysAliasReg;
+  };
+  #define GET_PRCTX_DECL
+  #include "AArch64GenSystemOperands.inc"
+}
+
 namespace AArch64II {
   /// Target Operand Flag enum.
   enum TOF {
     //===------------------------------------------------------------------===//
     // AArch64 Specific MachineOperand flags.
 
     MO_NO_FLAG,
 

lib/Target/AArch64/Utils/AArch64BaseInfo.cpp

 namespace llvm {
   namespace AArch64TSB {
 #define GET_TSB_IMPL
 #include "AArch64GenSystemOperands.inc"
   }
 }
 
 namespace llvm {
+  namespace AArch64PRCTX {
+#define GET_PRCTX_IMPL
+#include "AArch64GenSystemOperands.inc"
+  }
+}
+
+namespace llvm {
   namespace AArch64PRFM {
 #define GET_PRFM_IMPL
 #include "AArch64GenSystemOperands.inc"
   }
 }
 
 namespace llvm {
   namespace AArch64SVEPRFM {

test/MC/AArch64/armv8.5a-predctrl-error.s

+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predctrl < %s 2>&1| FileCheck %s
+
+cfp rctx
+dvp rctx
+cpp rctx
+
+// CHECK: specified cfp op requires a register
+// CHECK: specified dvp op requires a register
+// CHECK: specified cpp op requires a register
+
+cfp x0, x1
+dvp x1, x2
+cpp x2, x3
+
+// CHECK:      invalid operand for prediction restriction instruction
+// CHECK-NEXT: cfp
+// CHECK:      invalid operand for prediction restriction instruction
+// CHECK-NEXT: dvp
+// CHECK:      invalid operand for prediction restriction instruction
+// CHECK-NEXT: cpp

test/MC/AArch64/armv8.5a-predctrl.s

+// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predctrl < %s      | FileCheck %s
+// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-predctrl < %s 2>&1 | FileCheck %s --check-prefix=NOPREDCTRL
+
+cfp rctx, x0
+dvp rctx, x1
+cpp rctx, x2
+
+// CHECK: cfp rctx, x0      // encoding: [0x80,0x73,0x0b,0xd5]
+// CHECK: dvp rctx, x1      // encoding: [0xa1,0x73,0x0b,0xd5]
+// CHECK: cpp rctx, x2      // encoding: [0xe2,0x73,0x0b,0xd5]
+
+// NOPREDCTRL: CFPRCTX requires predctrl
+// NOPREDCTRL-NEXT: cfp
+// NOPREDCTRL: DVPRCTX requires predctrl
+// NOPREDCTRL-NEXT: dvp
+// NOPREDCTRL: CPPRCTX requires predctrl
+// NOPREDCTRL-NEXT: cpp

test/MC/Disassembler/AArch64/armv8.5a-predctrl.txt

+# RUN: llvm-mc -triple=aarch64 -mattr=+predctrl -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=aarch64 -mattr=-predctrl -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+
+[0x80 0x73 0x0b 0xd5]
+[0xa1 0x73 0x0b 0xd5]
+[0xe2 0x73 0x0b 0xd5]
+
+# CHECK: cfp rctx, x0
+# CHECK: dvp rctx, x1
+# CHECK: cpp rctx, x2
+
+# NOSB: sys #3, c7, c3, #4, x0
+# NOSB: sys #3, c7, c3, #5, x1
+# NOSB: sys #3, c7, c3, #7, x2