diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -3017,6 +3017,20 @@ def int_aarch64_sme_bmopa_za32 : SME_OuterProduct_Intrinsic; def int_aarch64_sme_bmops_za32 : SME_OuterProduct_Intrinsic; + // + // Multi-vector rounding shift left intrinsics + // + + def int_aarch64_sve_srshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; + def int_aarch64_sve_urshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; + def int_aarch64_sve_srshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; + def int_aarch64_sve_urshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; + + def int_aarch64_sve_srshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; + def int_aarch64_sve_urshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; + def int_aarch64_sve_srshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; + def int_aarch64_sve_urshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; + // Multi-vector saturating rounding shift right intrinsics def int_aarch64_sve_sqrshr_x2 : SME2_VG2_Multi_Imm_Intrinsic; diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -4956,6 +4956,62 @@ if (tryMULLV64LaneV128(IntNo, Node)) return; break; + case Intrinsic::aarch64_sve_srshl_single_x2: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H, + AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D})) + SelectDestructiveMultiIntrinsic(Node, 2, false, Op); + return; + case Intrinsic::aarch64_sve_srshl_single_x4: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H, + AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D})) + SelectDestructiveMultiIntrinsic(Node, 4, false, Op); + return; + case Intrinsic::aarch64_sve_urshl_single_x2: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H, + AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D})) + SelectDestructiveMultiIntrinsic(Node, 2, false, Op); + return; + case Intrinsic::aarch64_sve_urshl_single_x4: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H, + AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D})) + SelectDestructiveMultiIntrinsic(Node, 4, false, Op); + return; + case Intrinsic::aarch64_sve_srshl_x2: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H, + AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D})) + SelectDestructiveMultiIntrinsic(Node, 2, true, Op); + return; + case Intrinsic::aarch64_sve_srshl_x4: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H, + AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D})) + SelectDestructiveMultiIntrinsic(Node, 4, true, Op); + return; + case Intrinsic::aarch64_sve_urshl_x2: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H, + AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D})) + SelectDestructiveMultiIntrinsic(Node, 2, true, Op); + return; + case Intrinsic::aarch64_sve_urshl_x4: + if (auto Op = SelectOpcodeFromVT( + Node->getValueType(0), + {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H, + AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D})) + SelectDestructiveMultiIntrinsic(Node, 4, true, Op); + return; case Intrinsic::aarch64_sve_sqdmulh_single_vgx2: if (auto Op = SelectOpcodeFromVT( Node->getValueType(0), diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll @@ -0,0 +1,641 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s + +; SRSHL (Single, x2) + +define { , } @multi_vec_rounding_shl_single_x2_s8( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.b, z5.b }, { z4.b, z5.b }, z3.b +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.single.x2.nxv16i8( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_s16( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.h, z5.h }, { z4.h, z5.h }, z3.h +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.single.x2.nxv8i16( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_s32( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_s32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.s, z5.s }, { z4.s, z5.s }, z3.s +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.single.x2.nxv4i32( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_s64( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_s64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.d, z5.d }, { z4.d, z5.d }, z3.d +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.single.x2.nxv2i64( %zdn1, %zdn2, %zm) + ret { , } %res +} + +; SRSHL (Single, x4) + +define { , , , } +@multi_vec_rounding_shl_single_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.b - z27.b }, { z24.b - z27.b }, z5.b +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.single.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.h - z27.h }, { z24.h - z27.h }, z5.h +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.single.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.s - z27.s }, { z24.s - z27.s }, z5.s +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.single.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.d - z27.d }, { z24.d - z27.d }, z5.d +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.single.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +; URSHL (Single, x2) + +define { , } @multi_vec_rounding_shl_single_x2_u8( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.b, z5.b }, { z4.b, z5.b }, z3.b +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.single.x2.nxv16i8( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_u16( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.h, z5.h }, { z4.h, z5.h }, z3.h +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.single.x2.nxv8i16( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_u32( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_u32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.s, z5.s }, { z4.s, z5.s }, z3.s +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.single.x2.nxv4i32( %zdn1, %zdn2, %zm) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_single_x2_u64( %dummy, %zdn1, %zdn2, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x2_u64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.d, z5.d }, { z4.d, z5.d }, z3.d +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.single.x2.nxv2i64( %zdn1, %zdn2, %zm) + ret { , } %res +} + +; URSHL (Single, x4) + +define { , , , } +@multi_vec_rounding_shl_single_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.b - z27.b }, { z24.b - z27.b }, z5.b +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.single.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.h - z27.h }, { z24.h - z27.h }, z5.h +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.single.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.s - z27.s }, { z24.s - z27.s }, z5.s +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.single.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_single_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.d - z27.d }, { z24.d - z27.d }, z5.d +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.single.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm) + ret { , , , } %res +} + +; SRSHL (Multi, x2) + +define { , } @multi_vec_rounding_shl_x2_s8( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_shl_x2_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.b, z5.b }, { z4.b, z5.b }, { z6.b, z7.b } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.x2.nxv16i8( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_x2_s16( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_shl_x2_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.h, z5.h }, { z4.h, z5.h }, { z6.h, z7.h } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.x2.nxv8i16( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_x2_s32( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_shl_x2_s32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.s, z5.s }, { z4.s, z5.s }, { z6.s, z7.s } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.x2.nxv4i32( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_shl_x2_s64( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_shl_x2_s64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: srshl { z4.d, z5.d }, { z4.d, z5.d }, { z6.d, z7.d } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.srshl.x2.nxv2i64( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +; SRSHL (Multi, x4) + +define { , , , } +@multi_vec_rounding_shl_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_s8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_s16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_s32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_s64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: srshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.srshl.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +; URSHL (Multi, x2) + +define { , } @multi_vec_rounding_uhl_x2_u8( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_uhl_x2_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.b, z5.b }, { z4.b, z5.b }, { z6.b, z7.b } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.x2.nxv16i8( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_uhl_x2_u16( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_uhl_x2_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.h, z5.h }, { z4.h, z5.h }, { z6.h, z7.h } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.x2.nxv8i16( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_uhl_x2_u32( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_uhl_x2_u32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.s, z5.s }, { z4.s, z5.s }, { z6.s, z7.s } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.x2.nxv4i32( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +define { , } @multi_vec_rounding_uhl_x2_u64( %dummy, %zdn1, %zdn2, %zm1, %zm2) { +; CHECK-LABEL: multi_vec_rounding_uhl_x2_u64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: mov z5.d, z2.d +; CHECK-NEXT: mov z6.d, z3.d +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: urshl { z4.d, z5.d }, { z4.d, z5.d }, { z6.d, z7.d } +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %res = call { , } @llvm.aarch64.sve.urshl.x2.nxv2i64( %zdn1, %zdn2, %zm1, %zm2) + ret { , } %res +} + +; URSHL (Multi, x4) + +define { , , , } +@multi_vec_rounding_shl_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_u8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_u16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_u32: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +define { , , , } +@multi_vec_rounding_shl_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +; CHECK-LABEL: multi_vec_rounding_shl_x4_u64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z25.d, z2.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: urshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } +; CHECK-NEXT: mov z0.d, z24.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z26.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: ret + %res = call { , , , } + @llvm.aarch64.sve.urshl.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, + %zm1, %zm2, %zm3, %zm4) + ret { , , , } %res +} + +declare { , } @llvm.aarch64.sve.srshl.single.x2.nxv16i8(, , ) +declare { , } @llvm.aarch64.sve.srshl.single.x2.nxv8i16(, , ) +declare { , } @llvm.aarch64.sve.srshl.single.x2.nxv4i32(, , ) +declare { , } @llvm.aarch64.sve.srshl.single.x2.nxv2i64(, , ) + +declare { , , , } @llvm.aarch64.sve.srshl.single.x4.nxv16i8(, , , , ) +declare { , , , } @llvm.aarch64.sve.srshl.single.x4.nxv8i16(, , , , ) +declare { , , , } @llvm.aarch64.sve.srshl.single.x4.nxv4i32(, , , , ) +declare { , , , } @llvm.aarch64.sve.srshl.single.x4.nxv2i64(, , , , ) + +declare { , } @llvm.aarch64.sve.urshl.single.x2.nxv16i8(, , ) +declare { , } @llvm.aarch64.sve.urshl.single.x2.nxv8i16(, , ) +declare { , } @llvm.aarch64.sve.urshl.single.x2.nxv4i32(, , ) +declare { , } @llvm.aarch64.sve.urshl.single.x2.nxv2i64(, , ) + +declare { , , , } @llvm.aarch64.sve.urshl.single.x4.nxv16i8(, , , , ) +declare { , , , } @llvm.aarch64.sve.urshl.single.x4.nxv8i16(, , , , ) +declare { , , , } @llvm.aarch64.sve.urshl.single.x4.nxv4i32(, , , , ) +declare { , , , } @llvm.aarch64.sve.urshl.single.x4.nxv2i64(, , , , ) + +declare { , } @llvm.aarch64.sve.srshl.x2.nxv16i8(, , , ) +declare { , } @llvm.aarch64.sve.srshl.x2.nxv8i16(, , , ) +declare { , } @llvm.aarch64.sve.srshl.x2.nxv4i32(, , , ) +declare { , } @llvm.aarch64.sve.srshl.x2.nxv2i64(, , , ) + +declare { , , , } + @llvm.aarch64.sve.srshl.x4.nxv16i8(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.srshl.x4.nxv8i16(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.srshl.x4.nxv4i32(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.srshl.x4.nxv2i64(, , , , , , , ) + +declare { , } @llvm.aarch64.sve.urshl.x2.nxv16i8(, , , ) +declare { , } @llvm.aarch64.sve.urshl.x2.nxv8i16(, , , ) +declare { , } @llvm.aarch64.sve.urshl.x2.nxv4i32(, , , ) +declare { , } @llvm.aarch64.sve.urshl.x2.nxv2i64(, , , ) + +declare { , , , } + @llvm.aarch64.sve.urshl.x4.nxv16i8(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.urshl.x4.nxv8i16(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.urshl.x4.nxv4i32(, , , , , , , ) +declare { , , , } + @llvm.aarch64.sve.urshl.x4.nxv2i64(, , , , , , , )