Index: llvm/include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsAArch64.td +++ llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1053,6 +1053,13 @@ LLVMVectorOfBitcastsToInt<0>], [IntrNoMem]>; + class AdvSIMD_SVE2_TBX_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMVectorOfBitcastsToInt<0>], + [IntrNoMem]>; + class SVE2_1VectorArg_Long_Intrinsic : Intrinsic<[llvm_anyvector_ty], [LLVMSubdivide2VectorType<0>, @@ -2007,4 +2014,19 @@ def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; +// +// SVE2 - Extended table lookup/permute +// + +def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; +def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; + +// +// SVE2 - Optional bit permutation +// + +def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; + } Index: llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -200,6 +200,8 @@ /// unchanged; otherwise a REG_SEQUENCE value is returned. SDValue createDTuple(ArrayRef Vecs); SDValue createQTuple(ArrayRef Vecs); + // Same thing for SVE instructions making use of lists of Z registers + SDValue createZTuple(ArrayRef Vecs); /// Generic helper for the createDTuple/createQTuple /// functions. Those should almost always be called instead. @@ -208,6 +210,8 @@ void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt); + void SelectTableSVE2(SDNode *N, unsigned Opc); + bool tryIndexedLoad(SDNode *N); bool trySelectStackSlotTagP(SDNode *N); @@ -1154,6 +1158,15 @@ return createTuple(Regs, RegClassIDs, SubRegs); } +SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef Regs) { + static const unsigned RegClassIDs[] = { + AArch64::ZPR2RegClassID, AArch64::ZPR3RegClassID, AArch64::ZPR4RegClassID}; + static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1, + AArch64::zsub2, AArch64::zsub3}; + + return createTuple(Regs, RegClassIDs, SubRegs); +} + SDValue AArch64DAGToDAGISel::createTuple(ArrayRef Regs, const unsigned RegClassIDs[], const unsigned SubRegs[]) { @@ -1204,6 +1217,24 @@ ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops)); } +void AArch64DAGToDAGISel::SelectTableSVE2(SDNode *N, unsigned Opc) { + SDLoc DL(N); + EVT VT = N->getValueType(0); + + // We are only using this to select the aarch64_sve_tbl2 + // intrinsic currently, where NumVecs is always 2 + unsigned NumVecs = 2; + + // Form a REG_SEQUENCE to force register allocation. + SmallVector Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); + SDValue RegSeq = createZTuple(Regs); + + SmallVector Ops; + Ops.push_back(RegSeq); + Ops.push_back(N->getOperand(NumVecs + 1)); + ReplaceNode(N, CurDAG->getMachineNode(Opc, DL, VT, Ops)); +} + bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) { LoadSDNode *LD = cast(N); if (LD->isUnindexed()) @@ -3559,6 +3590,23 @@ if (tryMULLV64LaneV128(IntNo, Node)) return; break; + case Intrinsic::aarch64_sve_tbl2: + if (VT == MVT::nxv16i8) { + SelectTableSVE2(Node, AArch64::TBL_ZZZZ_B); + return; + } + if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) { + SelectTableSVE2(Node, AArch64::TBL_ZZZZ_H); + return; + } + if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { + SelectTableSVE2(Node, AArch64::TBL_ZZZZ_S); + return; + } + if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { + SelectTableSVE2(Node, AArch64::TBL_ZZZZ_D); + return; + } } break; } Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1814,7 +1814,7 @@ // SVE2 table lookup (three sources) defm TBL_ZZZZ : sve2_int_perm_tbl<"tbl">; - defm TBX_ZZZ : sve2_int_perm_tbx<"tbx">; + defm TBX_ZZZ : sve2_int_perm_tbx<"tbx", int_aarch64_sve_tbx>; // SVE2 integer compare scalar count and limit defm WHILEGE_PWW : sve_int_while4_rr<0b000, "whilege", int_aarch64_sve_whilege>; @@ -1862,7 +1862,7 @@ let Predicates = [HasSVE2BitPerm] in { // SVE2 bitwise permute - defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext">; - defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep">; - defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp">; + defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext", int_aarch64_sve_bext_x>; + defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep", int_aarch64_sve_bdep_x>; + defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp", int_aarch64_sve_bgrp_x>; } Index: llvm/lib/Target/AArch64/SVEInstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/SVEInstrFormats.td +++ llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -967,11 +967,20 @@ let Constraints = "$Zd = $_Zd"; } -multiclass sve2_int_perm_tbx { +multiclass sve2_int_perm_tbx { def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; + + def : SVE_3_Op_Pat(NAME # _B)>; + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; + + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; } class sve_int_perm_reverse_z sz8_64, string asm, ZPRRegOp zprty> @@ -2951,11 +2960,16 @@ let Inst{4-0} = Zd; } -multiclass sve2_misc_bitwise opc, string asm> { +multiclass sve2_misc_bitwise opc, string asm, SDPatternOperator op> { def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; + + def : SVE_2_Op_Pat(NAME # _B)>; + def : SVE_2_Op_Pat(NAME # _H)>; + def : SVE_2_Op_Pat(NAME # _S)>; + def : SVE_2_Op_Pat(NAME # _D)>; } multiclass sve2_misc_int_addsub_long_interleaved opc, string asm, Index: llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll @@ -0,0 +1,124 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2,+sve2-bitperm < %s | FileCheck %s + +; +; BDEP +; + +define @bdep_nxv16i8( %a, %b) { +; CHECK-LABEL: bdep_nxv16i8: +; CHECK: bdep z0.b, z0.b, z1.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bdep.x.nx16i8( %a, %b) + ret %out +} + +define @bdep_nxv8i16( %a, %b) { +; CHECK-LABEL: bdep_nxv8i16: +; CHECK: bdep z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bdep.x.nx8i16( %a, %b) + ret %out +} + +define @bdep_nxv4i32( %a, %b) { +; CHECK-LABEL: bdep_nxv4i32: +; CHECK: bdep z0.s, z0.s, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bdep.x.nx4i32( %a, %b) + ret %out +} + +define @bdep_nxv2i64( %a, %b) { +; CHECK-LABEL: bdep_nxv2i64: +; CHECK: bdep z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bdep.x.nx2i64( %a, %b) + ret %out +} + +; +; BEXT +; + +define @bext_nxv16i8( %a, %b) { +; CHECK-LABEL: bext_nxv16i8: +; CHECK: bext z0.b, z0.b, z1.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bext.x.nx16i8( %a, %b) + ret %out +} + +define @bext_nxv8i16( %a, %b) { +; CHECK-LABEL: bext_nxv8i16: +; CHECK: bext z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bext.x.nx8i16( %a, %b) + ret %out +} + +define @bext_nxv4i32( %a, %b) { +; CHECK-LABEL: bext_nxv4i32: +; CHECK: bext z0.s, z0.s, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bext.x.nx4i32( %a, %b) + ret %out +} + +define @bext_nxv2i64( %a, %b) { +; CHECK-LABEL: bext_nxv2i64: +; CHECK: bext z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bext.x.nx2i64( %a, %b) + ret %out +} + +; +; BGRP +; + +define @bgrp_nxv16i8( %a, %b) { +; CHECK-LABEL: bgrp_nxv16i8: +; CHECK: bgrp z0.b, z0.b, z1.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bgrp.x.nx16i8( %a, %b) + ret %out +} + +define @bgrp_nxv8i16( %a, %b) { +; CHECK-LABEL: bgrp_nxv8i16: +; CHECK: bgrp z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bgrp.x.nx8i16( %a, %b) + ret %out +} + +define @bgrp_nxv4i32( %a, %b) { +; CHECK-LABEL: bgrp_nxv4i32: +; CHECK: bgrp z0.s, z0.s, z1.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bgrp.x.nx4i32( %a, %b) + ret %out +} + +define @bgrp_nxv2i64( %a, %b) { +; CHECK-LABEL: bgrp_nxv2i64: +; CHECK: bgrp z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.bgrp.x.nx2i64( %a, %b) + ret %out +} + +declare @llvm.aarch64.sve.bdep.x.nx16i8( %a, %b) +declare @llvm.aarch64.sve.bdep.x.nx8i16( %a, %b) +declare @llvm.aarch64.sve.bdep.x.nx4i32( %a, %b) +declare @llvm.aarch64.sve.bdep.x.nx2i64( %a, %b) + +declare @llvm.aarch64.sve.bext.x.nx16i8( %a, %b) +declare @llvm.aarch64.sve.bext.x.nx8i16( %a, %b) +declare @llvm.aarch64.sve.bext.x.nx4i32( %a, %b) +declare @llvm.aarch64.sve.bext.x.nx2i64( %a, %b) + +declare @llvm.aarch64.sve.bgrp.x.nx16i8( %a, %b) +declare @llvm.aarch64.sve.bgrp.x.nx8i16( %a, %b) +declare @llvm.aarch64.sve.bgrp.x.nx4i32( %a, %b) +declare @llvm.aarch64.sve.bgrp.x.nx2i64( %a, %b) Index: llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve2-intrinsics-perm-tb.ll @@ -0,0 +1,167 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s + +; +; TBL2 +; + +define @tbl2_b( %a, %b, %c) { +; CHECK-LABEL: tbl2_b: +; CHECK: tbl z0.b, { z0.b, z1.b }, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv16i8( %a, + %b, + %c) + ret %out +} + +define @tbl2_h( %a, %b, %c) { +; CHECK-LABEL: tbl2_h: +; CHECK: tbl z0.h, { z0.h, z1.h }, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv8i16( %a, + %b, + %c) + ret %out +} + +define @tbl2_s( %a, %b, %c) { +; CHECK-LABEL: tbl2_s: +; CHECK: tbl z0.s, { z0.s, z1.s }, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv4i32( %a, + %b, + %c) + ret %out +} + +define @tbl2_d( %a, %b, %c) { +; CHECK-LABEL: tbl2_d: +; CHECK: tbl z0.d, { z0.d, z1.d }, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv2i64( %a, + %b, + %c) + ret %out +} + +define @tbl2_fh( %a, %b, %c) { +; CHECK-LABEL: tbl2_fh: +; CHECK: tbl z0.h, { z0.h, z1.h }, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv8f16( %a, + %b, + %c) + ret %out +} + +define @tbl2_fs( %a, %b, %c) { +; CHECK-LABEL: tbl2_fs: +; CHECK: tbl z0.s, { z0.s, z1.s }, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv4f32( %a, + %b, + %c) + ret %out +} + +define @tbl2_fd( %a, %b, %c) { +; CHECK-LABEL: tbl2_fd: +; CHECK: tbl z0.d, { z0.d, z1.d }, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbl2.nxv2f64( %a, + %b, + %c) + ret %out +} + +; +; TBX +; + +define @tbx_b( %a, %b, %c) { +; CHECK-LABEL: tbx_b: +; CHECK: tbx z0.b, z1.b, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv16i8( %a, + %b, + %c) + ret %out +} + +define @tbx_h( %a, %b, %c) { +; CHECK-LABEL: tbx_h: +; CHECK: tbx z0.h, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv8i16( %a, + %b, + %c) + ret %out +} + +define @ftbx_h( %a, %b, %c) { +; CHECK-LABEL: ftbx_h: +; CHECK: tbx z0.h, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv8f16( %a, + %b, + %c) + ret %out +} + +define @tbx_s( %a, %b, %c) { +; CHECK-LABEL: tbx_s: +; CHECK: tbx z0.s, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv4i32( %a, + %b, + %c) + ret %out +} + +define @ftbx_s( %a, %b, %c) { +; CHECK-LABEL: ftbx_s: +; CHECK: tbx z0.s, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv4f32( %a, + %b, + %c) + ret %out +} + +define @tbx_d( %a, %b, %c) { +; CHECK-LABEL: tbx_d: +; CHECK: tbx z0.d, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv2i64( %a, + %b, + %c) + ret %out +} + +define @ftbx_d( %a, %b, %c) { +; CHECK-LABEL: ftbx_d: +; CHECK: tbx z0.d, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.tbx.nxv2f64( %a, + %b, + %c) + ret %out +} + +declare @llvm.aarch64.sve.tbl2.nxv16i8(, , ) +declare @llvm.aarch64.sve.tbl2.nxv8i16(, , ) +declare @llvm.aarch64.sve.tbl2.nxv4i32(, , ) +declare @llvm.aarch64.sve.tbl2.nxv2i64(, , ) + +declare @llvm.aarch64.sve.tbl2.nxv8f16(, , ) +declare @llvm.aarch64.sve.tbl2.nxv4f32(, , ) +declare @llvm.aarch64.sve.tbl2.nxv2f64(, , ) + +declare @llvm.aarch64.sve.tbx.nxv16i8(, , ) +declare @llvm.aarch64.sve.tbx.nxv8i16(, , ) +declare @llvm.aarch64.sve.tbx.nxv4i32(, , ) +declare @llvm.aarch64.sve.tbx.nxv2i64(, , ) + +declare @llvm.aarch64.sve.tbx.nxv8f16(, , ) +declare @llvm.aarch64.sve.tbx.nxv4f32(, , ) +declare @llvm.aarch64.sve.tbx.nxv2f64(, , )