[AArch64][SVE] Add intrinsics for SVE2 bitwise ternary operations

[dancgr]

Summary

Add intrinsics for the following operations:

• eor3, bcax
• bsl, bsl1n, bsl2n, nbsl

[efriedma]

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

285

Missing newline

Any particular reason you're only testing bsl with nxv2i64 operands?

[dancgr]

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

285

I didn't add other operand types for bsl because the class definition on SVEInstrFormats.td for bsl, bsl1n, bsl2n and nbsl were only defined for ZPR64. For eor3 and bcax there was instruction aliases for ZPR32, ZPR16 and ZPR8.

[efriedma]

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

285

Do you have any idea why bsl is defined differently?

[dancgr]

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

285

Actually no, should they be the same? I can unify the definition and apply the aliases to the bsl instructions as well.

[efriedma]

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

285

That makes sense to me.

[dancgr]

Merging all instructions into same multiclass.

[efriedma]

LGTM

[leonardchan]

Hi, I think this might've caused some test failures on our bots and others on buildbot:

FAIL: LLVM :: MC/AArch64/SVE2/bsl-diagnostics.s (23685 of 36141)
******************** TEST 'LLVM :: MC/AArch64/SVE2/bsl-diagnostics.s' FAILED ********************
Script:
--
: 'RUN: at line 1';   not /b/s/w/ir/k/recipe_cleanup/clangOZ9WCk/llvm_build_dir/bin/llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2  2>&1 < /b/s/w/ir/k/llvm-project/llvm/test/MC/AArch64/SVE2/bsl-diagnostics.s| /b/s/w/ir/k/recipe_cleanup/clangOZ9WCk/llvm_build_dir/bin/FileCheck /b/s/w/ir/k/llvm-project/llvm/test/MC/AArch64/SVE2/bsl-diagnostics.s
--
Exit Code: 1

Command Output (stderr):
--
/b/s/w/ir/k/llvm-project/llvm/test/MC/AArch64/SVE2/bsl-diagnostics.s:8:11: error: CHECK: expected string not found in input
// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
          ^
<stdin>:1:1: note: scanning from here
<stdin>:26:11: error: operand must match destination register
^
<stdin>:1:1: note: with "@LINE-1" equal to "7"
<stdin>:26:11: error: operand must match destination register
^
<stdin>:1:2: note: possible intended match here
<stdin>:26:11: error: operand must match destination register
 ^

...

Testing Time: 135.11s
********************
Failing Tests (4):
    LLVM :: MC/AArch64/SVE2/bsl-diagnostics.s
    LLVM :: MC/AArch64/SVE2/bsl1n-diagnostics.s
    LLVM :: MC/AArch64/SVE2/bsl2n-diagnostics.s
    LLVM :: MC/AArch64/SVE2/nbsl-diagnostics.s

Could you send out a fix or revert? Thanks.

Builder: https://ci.chromium.org/p/fuchsia/builders/ci/clang-linux-x64/b8887916992587484480

[thakis]

I reverted this in 6f4d9d10293d8e93d7cf3397e3a82ac19c8b629f due to breaking tests everywhere.

[dancgr]

Will look into that and update this review with the necessary changes.

Thanks @thakis for reverting the commit.

[dancgr]

Latest commit adds a change to the failing MC tests to be exactly the same as the MC tests for eor3 and bcax.

[sdesmalen]

LGTM, thanks @dancgr

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
 
 //
 // SVE2 - Polynomial arithmetic
 //
-
 def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
 def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
 def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
 
+// SVE2 bitwise ternary operations.
+def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
+def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
+def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
+def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
+def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
+def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
+
 }

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE2] in {
 
   // SVE2 floating-point multiply-add long
   defm FMLALB_ZZZ_SHH : sve2_fp_mla_long<0b00, "fmlalb", int_aarch64_sve_fmlalb>;
   defm FMLALT_ZZZ_SHH : sve2_fp_mla_long<0b01, "fmlalt", int_aarch64_sve_fmlalt>;
   defm FMLSLB_ZZZ_SHH : sve2_fp_mla_long<0b10, "fmlslb", int_aarch64_sve_fmlslb>;
   defm FMLSLT_ZZZ_SHH : sve2_fp_mla_long<0b11, "fmlslt", int_aarch64_sve_fmlslt>;
 
   // SVE2 bitwise ternary operations
-  defm EOR3_ZZZZ_D  : sve2_int_bitwise_ternary_op<0b000, "eor3">;
-  defm BCAX_ZZZZ_D  : sve2_int_bitwise_ternary_op<0b010, "bcax">;
-  def BSL_ZZZZ_D    : sve2_int_bitwise_ternary_op_d<0b001, "bsl">;
-  def BSL1N_ZZZZ_D  : sve2_int_bitwise_ternary_op_d<0b011, "bsl1n">;
-  def BSL2N_ZZZZ_D  : sve2_int_bitwise_ternary_op_d<0b101, "bsl2n">;
-  def NBSL_ZZZZ_D   : sve2_int_bitwise_ternary_op_d<0b111, "nbsl">;
+  defm EOR3_ZZZZ  : sve2_int_bitwise_ternary_op<0b000, "eor3",  int_aarch64_sve_eor3>;
+  defm BCAX_ZZZZ  : sve2_int_bitwise_ternary_op<0b010, "bcax",  int_aarch64_sve_bcax>;
+  defm BSL_ZZZZ   : sve2_int_bitwise_ternary_op<0b001, "bsl",   int_aarch64_sve_bsl>;
+  defm BSL1N_ZZZZ : sve2_int_bitwise_ternary_op<0b011, "bsl1n", int_aarch64_sve_bsl1n>;
+  defm BSL2N_ZZZZ : sve2_int_bitwise_ternary_op<0b101, "bsl2n", int_aarch64_sve_bsl2n>;
+  defm NBSL_ZZZZ  : sve2_int_bitwise_ternary_op<0b111, "nbsl",  int_aarch64_sve_nbsl>;
 
   // SVE2 bitwise xor and rotate right by immediate
   defm XAR_ZZZI : sve2_int_rotate_right_imm<"xar">;
 
   // SVE2 extract vector (immediate offset, constructive)
   def EXT_ZZI_B : sve2_int_perm_extract_i_cons<"ext">;
 
   // SVE2 non-temporal gather loads

llvm/lib/Target/AArch64/SVEInstrFormats.td

: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk),
   let Inst{9-5}   = Zk;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
 }
 
-multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm> {
+multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> {
   def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>;
 
   def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
                   (!cast<Instruction>(NAME) ZPR8:$Zdn,  ZPR8:$Zm,  ZPR8:$Zk),  1>;
   def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
                   (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>;
   def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
                   (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>;
+
+  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
+  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
+  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;
+  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;
 }
 
 class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm,
                                 ZPRRegOp zprty, Operand immtype>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm),
   asm, "\t$Zdn, $_Zdn, $Zm, $imm",
   "",
   []>, Sched<[]> {

llvm/test/CodeGen/AArch64/sve2-bitwise-ternary.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+
+;
+; EOR3 (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @eor3_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b,
+                                   <vscale x 16 x i8> %c) {
+; CHECK-LABEL: eor3_i8
+; CHECK: eor3 z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.eor3.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @eor3_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b,
+                                    <vscale x 8 x i16> %c) {
+; CHECK-LABEL: eor3_i16
+; CHECK: eor3 z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.eor3.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @eor3_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b,
+                                    <vscale x 4 x i32> %c) {
+; CHECK-LABEL: eor3_i32
+; CHECK: eor3 z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.eor3.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @eor3_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b,
+                                    <vscale x 2 x i64> %c) {
+; CHECK-LABEL: eor3_i64
+; CHECK: eor3 z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.eor3.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BCAX (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bcax_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b,
+                                   <vscale x 16 x i8> %c) {
+; CHECK-LABEL: bcax_i8
+; CHECK: bcax z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bcax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bcax_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b,
+                                    <vscale x 8 x i16> %c) {
+; CHECK-LABEL: bcax_i16
+; CHECK: bcax z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bcax.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bcax_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b,
+                                    <vscale x 4 x i32> %c) {
+; CHECK-LABEL: bcax_i32
+; CHECK: bcax z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bcax.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bcax_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b,
+                                    <vscale x 2 x i64> %c) {
+; CHECK-LABEL: bcax_i64
+; CHECK: bcax z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bcax.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BSL (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bsl_i8(<vscale x 16 x i8> %a,
+                                  <vscale x 16 x i8> %b,
+                                  <vscale x 16 x i8> %c) {
+; CHECK-LABEL: bsl_i8
+; CHECK: bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bsl.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bsl_i16(<vscale x 8 x i16> %a,
+                                   <vscale x 8 x i16> %b,
+                                   <vscale x 8 x i16> %c) {
+; CHECK-LABEL: bsl_i16
+; CHECK: bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bsl.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bsl_i32(<vscale x 4 x i32> %a,
+                                   <vscale x 4 x i32> %b,
+                                   <vscale x 4 x i32> %c) {
+; CHECK-LABEL: bsl_i32
+; CHECK: bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bsl.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bsl_i64(<vscale x 2 x i64> %a,
+                                   <vscale x 2 x i64> %b,
+                                   <vscale x 2 x i64> %c) {
+; CHECK-LABEL: bsl_i64
+; CHECK: bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bsl.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BSL1N (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bsl1n_i8(<vscale x 16 x i8> %a,
+                                    <vscale x 16 x i8> %b,
+                                    <vscale x 16 x i8> %c) {
+; CHECK-LABEL: bsl1n_i8
+; CHECK: bsl1n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bsl1n.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bsl1n_i16(<vscale x 8 x i16> %a,
+                                     <vscale x 8 x i16> %b,
+                                     <vscale x 8 x i16> %c) {
+; CHECK-LABEL: bsl1n_i16
+; CHECK: bsl1n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bsl1n.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bsl1n_i32(<vscale x 4 x i32> %a,
+                                     <vscale x 4 x i32> %b,
+                                     <vscale x 4 x i32> %c) {
+; CHECK-LABEL: bsl1n_i32
+; CHECK: bsl1n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bsl1n.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bsl1n_i64(<vscale x 2 x i64> %a,
+                                     <vscale x 2 x i64> %b,
+                                     <vscale x 2 x i64> %c) {
+; CHECK-LABEL: bsl1n_i64
+; CHECK: bsl1n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bsl1n.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; BSL2N (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @bsl2n_i8(<vscale x 16 x i8> %a,
+                                    <vscale x 16 x i8> %b,
+                                    <vscale x 16 x i8> %c) {
+; CHECK-LABEL: bsl2n_i8
+; CHECK: bsl2n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.bsl2n.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @bsl2n_i16(<vscale x 8 x i16> %a,
+                                     <vscale x 8 x i16> %b,
+                                     <vscale x 8 x i16> %c) {
+; CHECK-LABEL: bsl2n_i16
+; CHECK: bsl2n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.bsl2n.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @bsl2n_i32(<vscale x 4 x i32> %a,
+                                     <vscale x 4 x i32> %b,
+                                     <vscale x 4 x i32> %c) {
+; CHECK-LABEL: bsl2n_i32
+; CHECK: bsl2n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.bsl2n.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @bsl2n_i64(<vscale x 2 x i64> %a,
+                                     <vscale x 2 x i64> %b,
+                                     <vscale x 2 x i64> %c) {
+; CHECK-LABEL: bsl2n_i64
+; CHECK: bsl2n z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.bsl2n.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; NBSL (vector, bitwise, unpredicated)
+;
+define <vscale x 16 x i8> @nbsl_i8(<vscale x 16 x i8> %a,
+                                   <vscale x 16 x i8> %b,
+                                   <vscale x 16 x i8> %c) {
+; CHECK-LABEL: nbsl_i8
+; CHECK: nbsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.nbsl.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @nbsl_i16(<vscale x 8 x i16> %a,
+                                    <vscale x 8 x i16> %b,
+                                    <vscale x 8 x i16> %c) {
+; CHECK-LABEL: nbsl_i16
+; CHECK: nbsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.nbsl.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @nbsl_i32(<vscale x 4 x i32> %a,
+                                    <vscale x 4 x i32> %b,
+                                    <vscale x 4 x i32> %c) {
+; CHECK-LABEL: nbsl_i32
+; CHECK: nbsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.nbsl.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @nbsl_i64(<vscale x 2 x i64> %a,
+                                    <vscale x 2 x i64> %b,
+                                    <vscale x 2 x i64> %c) {
+; CHECK-LABEL: nbsl_i64
+; CHECK: nbsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.nbsl.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c)
+  ret <vscale x 2 x i64> %res
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.eor3.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.eor3.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.eor3.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.eor3.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bcax.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bcax.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bcax.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bcax.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bsl.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bsl.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bsl.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bsl.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bsl1n.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bsl1n.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bsl1n.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bsl1n.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.bsl2n.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.bsl2n.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.bsl2n.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.bsl2n.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.nbsl.nxv16i8(<vscale x 16 x i8>,<vscale x 16 x i8>,<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.nbsl.nxv8i16(<vscale x 8 x i16>,<vscale x 8 x i16>,<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.nbsl.nxv4i32(<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.nbsl.nxv2i64(<vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>)

llvm/test/MC/AArch64/SVE2/bsl-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element width
 
-bsl z0.b, z0.b, z1.b, z2.b
+bsl z0.b, z0.b, z1.s, z2.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl z0.b, z0.b, z1.b, z2.b
+// CHECK-NEXT: bsl z0.b, z0.b, z1.s, z2.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl z0.h, z0.h, z1.h, z2.h
+bsl z0.h, z0.h, z1.h, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl z0.h, z0.h, z1.h, z2.h
+// CHECK-NEXT: bsl z0.h, z0.h, z1.h, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl z0.s, z0.s, z1.s, z2.s
+bsl z0.d, z0.d, z1.s, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl z0.s, z0.s, z1.s, z2.s
+// CHECK-NEXT: bsl z0.d, z0.d, z1.s, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 
 // --------------------------------------------------------------------------//
 // Source and Destination Registers must match
 
 bsl z0.d, z1.d, z2.d, z3.d
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register

llvm/test/MC/AArch64/SVE2/bsl1n-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element width
 
-bsl1n z0.b, z0.b, z1.b, z2.b
+bsl1n z0.b, z0.b, z1.s, z2.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl1n z0.b, z0.b, z1.b, z2.b
+// CHECK-NEXT: bsl1n z0.b, z0.b, z1.s, z2.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl1n z0.h, z0.h, z1.h, z2.h
+bsl1n z0.h, z0.h, z1.h, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl1n z0.h, z0.h, z1.h, z2.h
+// CHECK-NEXT: bsl1n z0.h, z0.h, z1.h, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl1n z0.s, z0.s, z1.s, z2.s
+bsl1n z0.d, z0.d, z1.s, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl1n z0.s, z0.s, z1.s, z2.s
+// CHECK-NEXT: bsl1n z0.d, z0.d, z1.s, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 
 // --------------------------------------------------------------------------//
 // Source and Destination Registers must match
 
 bsl1n z0.d, z1.d, z2.d, z3.d
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register

llvm/test/MC/AArch64/SVE2/bsl2n-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element width
 
-bsl2n z0.b, z0.b, z1.b, z2.b
+bsl2n z0.b, z0.b, z1.s, z2.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl2n z0.b, z0.b, z1.b, z2.b
+// CHECK-NEXT: bsl2n z0.b, z0.b, z1.s, z2.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl2n z0.h, z0.h, z1.h, z2.h
+bsl2n z0.h, z0.h, z1.h, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl2n z0.h, z0.h, z1.h, z2.h
+// CHECK-NEXT: bsl2n z0.h, z0.h, z1.h, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-bsl2n z0.s, z0.s, z1.s, z2.s
+bsl2n z0.d, z0.d, z1.s, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: bsl2n z0.s, z0.s, z1.s, z2.s
+// CHECK-NEXT: bsl2n z0.d, z0.d, z1.s, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 
 // --------------------------------------------------------------------------//
 // Source and Destination Registers must match
 
 bsl2n z0.d, z1.d, z2.d, z3.d
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register

llvm/test/MC/AArch64/SVE2/nbsl-diagnostics.s

 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2  2>&1 < %s| FileCheck %s
 
 
 // ------------------------------------------------------------------------- //
 // Invalid element width
 
-nbsl z0.b, z0.b, z1.b, z2.b
+nbsl z0.b, z0.b, z1.s, z2.b
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: nbsl z0.b, z0.b, z1.b, z2.b
+// CHECK-NEXT: nbsl z0.b, z0.b, z1.s, z2.b
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-nbsl z0.h, z0.h, z1.h, z2.h
+nbsl z0.h, z0.h, z1.h, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: nbsl z0.h, z0.h, z1.h, z2.h
+// CHECK-NEXT: nbsl z0.h, z0.h, z1.h, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
-nbsl z0.s, z0.s, z1.s, z2.s
+nbsl z0.d, z0.d, z1.s, z2.s
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
-// CHECK-NEXT: nbsl z0.s, z0.s, z1.s, z2.s
+// CHECK-NEXT: nbsl z0.d, z0.d, z1.s, z2.s
 // CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
 
 
 // --------------------------------------------------------------------------//
 // Source and Destination Registers must match
 
 nbsl z0.d, z1.d, z2.d, z3.d
 // CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must match destination register