[AArch64][SVE] Add intrinsics for FFR manipulation

[kmclaughlin]

Summary

Implements the following intrinsics:

• llvm.aarch64.sve.setffr
• llvm.aarch64.sve.rdffr
• llvm.aarch64.sve.rdffr.z
• llvm.aarch64.sve.wrffr

[efriedma]

LGTM

I think it's fairly likely that we'll eventually decide that the correct model of the FFR register doesn't require these intrinsics at all. They're just register copies, which is not something we normally model explicitly in IR. Instead, it would be much simpler to reason about the semantics if the faulting load intrinsics took the FFR value explicitly as an operand/result. (clang can lower the ACLE intrinsics onto this model by storing "FFR" into a hidden alloca.)

In the meantime, though, this implementation looks correct, ignoring the issue with FFR getting clobbered we discussed before.

It would be nice if we could use COPY MachineInstrs instead of introducing new pseudo-instructions, but I guess that has the same issue of reading a register that wasn't previously defined.

[kmclaughlin]

Thanks for your input on this @efriedma, I will submit this patch for now and we will revisit the modelling of the FFR as also discussed in D71698

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
 def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
 def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
 def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
 
 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
 
 //
+// FFR manipulation
+//
+
+def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   Intrinsic<[llvm_nxv16i1_ty], []>;
+def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
+def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  Intrinsic<[], []>;
+def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   Intrinsic<[], [llvm_nxv16i1_ty]>;
+
+//
 // Saturating scalar arithmetic
 //
 
 def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
 def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
 def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
 def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
 

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 def SDT_AArch64Rev   : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
 def AArch64rev       : SDNode<"AArch64ISD::REV", SDT_AArch64Rev>;
 
 def SDT_AArch64PTest : SDTypeProfile<0, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
 def AArch64ptest     : SDNode<"AArch64ISD::PTEST", SDT_AArch64PTest>;
 
 let Predicates = [HasSVE] in {
 
-  def RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr">;
-  def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
-  def RDFFR_P    : sve_int_rdffr_unpred<"rdffr">;
-  def SETFFR     : sve_int_setffr<"setffr">;
-  def WRFFR      : sve_int_wrffr<"wrffr">;
+  defm RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
+  def  RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
+  defm RDFFR_P    : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>;
+  def  SETFFR     : sve_int_setffr<"setffr", int_aarch64_sve_setffr>;
+  def  WRFFR      : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>;
 
   defm ADD_ZZZ   : sve_int_bin_cons_arit_0<0b000, "add", add>;
   defm SUB_ZZZ   : sve_int_bin_cons_arit_0<0b001, "sub", sub>;
   defm SQADD_ZZZ : sve_int_bin_cons_arit_0<0b100, "sqadd", saddsat>;
   defm UQADD_ZZZ : sve_int_bin_cons_arit_0<0b101, "uqadd", uaddsat>;
   defm SQSUB_ZZZ : sve_int_bin_cons_arit_0<0b110, "sqsub", ssubsat>;
   defm UQSUB_ZZZ : sve_int_bin_cons_arit_0<0b111, "uqsub", usubsat>;
 

llvm/lib/Target/AArch64/SVEInstrFormats.td

: I<(outs PPR8:$Pd), (ins PPRAny:$Pg),
   let Inst{8-5}   = Pg;
   let Inst{4}     = 0;
   let Inst{3-0}   = Pd;
 
   let Defs = !if(!eq (s, 1), [NZCV], []);
   let Uses = [FFR];
 }
 
+multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> {
+  def _REAL : sve_int_rdffr_pred<s, asm>;
+
+  // We need a layer of indirection because early machine code passes balk at
+  // physical register (i.e. FFR) uses that have no previous definition.
+  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
+  def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>,
+           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>;
+  }
+}
+
 class sve_int_rdffr_unpred<string asm> : I<
   (outs PPR8:$Pd), (ins),
   asm, "\t$Pd",
   "",
   []>, Sched<[]> {
   bits<4> Pd;
   let Inst{31-4} = 0b0010010100011001111100000000;
   let Inst{3-0}   = Pd;
 
   let Uses = [FFR];
 }
 
-class sve_int_wrffr<string asm>
+multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> {
+  def _REAL : sve_int_rdffr_unpred<asm>;
+
+  // We need a layer of indirection because early machine code passes balk at
+  // physical register (i.e. FFR) uses that have no previous definition.
+  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
+  def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>,
+           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>;
+  }
+}
+
+class sve_int_wrffr<string asm, SDPatternOperator op>
 : I<(outs), (ins PPR8:$Pn),
   asm, "\t$Pn",
   "",
-  []>, Sched<[]> {
+  [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> {
   bits<4> Pn;
   let Inst{31-9} = 0b00100101001010001001000;
   let Inst{8-5}  = Pn;
   let Inst{4-0}  = 0b00000;
 
   let hasSideEffects = 1;
   let Defs = [FFR];
 }
 
-class sve_int_setffr<string asm>
+class sve_int_setffr<string asm, SDPatternOperator op>
 : I<(outs), (ins),
   asm, "",
   "",
-  []>, Sched<[]> {
+  [(op)]>, Sched<[]> {
   let Inst{31-0} = 0b00100101001011001001000000000000;
 
   let hasSideEffects = 1;
   let Defs = [FFR];
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Permute Vector - Predicated Group

llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; RDFFR
+;
+
+define <vscale x 16 x i1> @rdffr() {
+; CHECK-LABEL: rdffr:
+; CHECK: rdffr p0.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @rdffr_z(<vscale x 16 x i1> %pg) {
+; CHECK-LABEL: rdffr_z:
+; CHECK: rdffr p0.b, p0/z
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> %pg)
+  ret <vscale x 16 x i1> %out
+}
+
+;
+; SETFFR
+;
+
+define void @set_ffr() {
+; CHECK-LABEL: set_ffr:
+; CHECK: setffr
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.setffr()
+  ret void
+}
+
+;
+; WRFFR
+;
+
+define void @wrffr(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: wrffr:
+; CHECK: wrffr p0.b
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> %a)
+  ret void
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1>)
+declare void @llvm.aarch64.sve.setffr()
+declare void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1>)