[AArch64][SVE] Add intrinsics for bitwise permute instructions

[dancgr]

Summary

Add instrinsics and patterns for the following instructions:

• bext, bdep, bgrp

[dancgr]

llvm/test/CodeGen/AArch64/sve2-bit-perm.ll

1

Not sure I should keep both +sve2 and +sve2-bitperm or not.

[c-rhodes]

Thanks for the patch @dancgr but these intrinsics have already been implemented in D74912. @sdesmalen shared a spreadsheet [1] with the mailing list to coordinate the intrinsics upstreaming, it's probably not as useful now given they're almost all implemented but you might find it useful if you haven't seen it already.

[1] https://docs.google.com/spreadsheets/d/1leug3a0eHhi_bWtKJOR0MeZHvfQLoIPtP-um7AWafRM

llvm/test/CodeGen/AArch64/sve2-bit-perm.ll

1

+sve2-bitperm implies +sve2 so only the former is required.

[dancgr]

In D75197#1894971, @c-rhodes wrote:

Thanks for the patch @dancgr but these intrinsics have already been implemented in D74912. @sdesmalen shared a spreadsheet [1] with the mailing list to coordinate the intrinsics upstreaming, it's probably not as useful now given they're almost all implemented but you might find it useful if you haven't seen it already.

[1] https://docs.google.com/spreadsheets/d/1leug3a0eHhi_bWtKJOR0MeZHvfQLoIPtP-um7AWafRM

Thanks for the info @c-rhodes, it seems like we are done with codegen then.

llvm/include/llvm/IR/IntrinsicsAArch64.td

 // SVE2 bitwise ternary operations.
 def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
 
+// SVE2 bit permutations.
+def int_aarch64_sve_bext   : AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_bdep   : AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_bgrp   : AdvSIMD_2VectorArg_Intrinsic;
+
 //
 // SVE2 - Optional AES, SHA-3 and SM4
 //
 
 def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
                               Intrinsic<[llvm_nxv16i8_ty],
                                         [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
                                         [IntrNoMem]>;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 
 let Predicates = [HasSVE2SHA3] in {
   // SVE2 crypto constructive binary operations
   defm RAX1_ZZZ_D : sve2_crypto_cons_bin_op<0b1, "rax1", ZPR64, int_aarch64_sve_rax1, nxv2i64>;
 }
 
 let Predicates = [HasSVE2BitPerm] in {
   // SVE2 bitwise permute
-  defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext">;
-  defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep">;
-  defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp">;
+  defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext", int_aarch64_sve_bext>;
+  defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep", int_aarch64_sve_bdep>;
+  defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp", int_aarch64_sve_bgrp>;
 }

llvm/lib/Target/AArch64/SVEInstrFormats.td

: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),
   let Inst{21}    = 0b0;
   let Inst{20-16} = Zm;
   let Inst{15-14} = 0b10;
   let Inst{13-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 }
 
-multiclass sve2_misc_bitwise<bits<4> opc, string asm> {
+multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> {
   def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>;
   def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>;
   def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>;
   def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>;
+
+  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm,
                                                  SDPatternOperator op> {
   def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>;
   def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>;
   def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>;
 

llvm/test/CodeGen/AArch64/sve2-bit-perm.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -mattr=+sve2-bitperm < %s | FileCheck %s
+
+;
+; BEXT (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bext_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b) {
+; CHECK-LABEL: bext_i8
+; CHECK: bext z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bext.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bext_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b) {
+; CHECK-LABEL: bext_i16
+; CHECK: bext z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bext.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bext_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b) {
+; CHECK-LABEL: bext_i32
+; CHECK: bext z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bext.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bext_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b) {
+; CHECK-LABEL: bext_i64
+; CHECK: bext z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bext.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BDEP (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bdep_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b) {
+; CHECK-LABEL: bdep_i8
+; CHECK: bdep z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bdep.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bdep_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b) {
+; CHECK-LABEL: bdep_i16
+; CHECK: bdep z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bdep.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bdep_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b) {
+; CHECK-LABEL: bdep_i32
+; CHECK: bdep z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bdep.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bdep_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b) {
+; CHECK-LABEL: bdep_i64
+; CHECK: bdep z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bdep.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BGRP (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bgrp_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b) {
+; CHECK-LABEL: bgrp_i8
+; CHECK: bgrp z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bgrp.nxv16i8(<vscale x 16 x i8> %a,
+                                                                <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bgrp_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b) {
+; CHECK-LABEL: bgrp_i16
+; CHECK: bgrp z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bgrp.nxv8i16(<vscale x 8 x i16> %a,
+                                                                <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bgrp_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b) {
+; CHECK-LABEL: bgrp_i32
+; CHECK: bgrp z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bgrp.nxv4i32(<vscale x 4 x i32> %a,
+                                                                <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bgrp_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b) {
+; CHECK-LABEL: bgrp_i64
+; CHECK: bgrp z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bgrp.nxv2i64(<vscale x 2 x i64> %a,
+                                                                <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bext.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bext.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bext.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bext.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bdep.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bdep.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bdep.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bdep.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bgrp.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bgrp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bgrp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bgrp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)