diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1291,6 +1291,15 @@ def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; // +// FFR manipulation +// + +def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, Intrinsic<[llvm_nxv16i1_ty], []>; +def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>; +def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, Intrinsic<[], []>; +def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, Intrinsic<[], [llvm_nxv16i1_ty]>; + +// // Saturating scalar arithmetic // diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -98,11 +98,11 @@ let Predicates = [HasSVE] in { - def RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr">; - def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">; - def RDFFR_P : sve_int_rdffr_unpred<"rdffr">; - def SETFFR : sve_int_setffr<"setffr">; - def WRFFR : sve_int_wrffr<"wrffr">; + defm RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>; + def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">; + defm RDFFR_P : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>; + def SETFFR : sve_int_setffr<"setffr", int_aarch64_sve_setffr>; + def WRFFR : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>; defm ADD_ZZZ : sve_int_bin_cons_arit_0<0b000, "add", add>; defm SUB_ZZZ : sve_int_bin_cons_arit_0<0b001, "sub", sub>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -5105,6 +5105,17 @@ let Uses = [FFR]; } +multiclass sve_int_rdffr_pred { + def _REAL : sve_int_rdffr_pred; + + // We need a layer of indirection because early machine code passes balk at + // physical register (i.e. FFR) uses that have no previous definition. + let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { + def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, + PseudoInstExpansion<(!cast(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; + } +} + class sve_int_rdffr_unpred : I< (outs PPR8:$Pd), (ins), asm, "\t$Pd", @@ -5117,11 +5128,22 @@ let Uses = [FFR]; } -class sve_int_wrffr +multiclass sve_int_rdffr_unpred { + def _REAL : sve_int_rdffr_unpred; + + // We need a layer of indirection because early machine code passes balk at + // physical register (i.e. FFR) uses that have no previous definition. + let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { + def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, + PseudoInstExpansion<(!cast(NAME # _REAL) PPR8:$Pd)>; + } +} + +class sve_int_wrffr : I<(outs), (ins PPR8:$Pn), asm, "\t$Pn", "", - []>, Sched<[]> { + [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { bits<4> Pn; let Inst{31-9} = 0b00100101001010001001000; let Inst{8-5} = Pn; @@ -5131,11 +5153,11 @@ let Defs = [FFR]; } -class sve_int_setffr +class sve_int_setffr : I<(outs), (ins), asm, "", "", - []>, Sched<[]> { + [(op)]>, Sched<[]> { let Inst{31-0} = 0b00100101001011001001000000000000; let hasSideEffects = 1; diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll @@ -0,0 +1,50 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s + +; +; RDFFR +; + +define @rdffr() { +; CHECK-LABEL: rdffr: +; CHECK: rdffr p0.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.rdffr() + ret %out +} + +define @rdffr_z( %pg) { +; CHECK-LABEL: rdffr_z: +; CHECK: rdffr p0.b, p0/z +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.rdffr.z( %pg) + ret %out +} + +; +; SETFFR +; + +define void @set_ffr() { +; CHECK-LABEL: set_ffr: +; CHECK: setffr +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.setffr() + ret void +} + +; +; WRFFR +; + +define void @wrffr( %a) { +; CHECK-LABEL: wrffr: +; CHECK: wrffr p0.b +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.wrffr( %a) + ret void +} + +declare @llvm.aarch64.sve.rdffr() +declare @llvm.aarch64.sve.rdffr.z() +declare void @llvm.aarch64.sve.setffr() +declare void @llvm.aarch64.sve.wrffr()