Index: llvm/include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsAArch64.td +++ llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -821,6 +821,11 @@ [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; +class AdvSIMD_Pred3VectorArg_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem]>; + // // Integer arithmetic @@ -852,6 +857,11 @@ def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; + def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -47,10 +47,10 @@ defm SQSUB_ZI : sve_int_arith_imm0<0b110, "sqsub">; defm UQSUB_ZI : sve_int_arith_imm0<0b111, "uqsub">; - defm MAD_ZPmZZ : sve_int_mladdsub_vvv_pred<0b0, "mad">; - defm MSB_ZPmZZ : sve_int_mladdsub_vvv_pred<0b1, "msb">; - defm MLA_ZPmZZ : sve_int_mlas_vvv_pred<0b0, "mla">; - defm MLS_ZPmZZ : sve_int_mlas_vvv_pred<0b1, "mls">; + defm MAD_ZPmZZ : sve_int_mladdsub_vvv_pred<0b0, "mad", int_aarch64_sve_mad>; + defm MSB_ZPmZZ : sve_int_mladdsub_vvv_pred<0b1, "msb", int_aarch64_sve_msb>; + defm MLA_ZPmZZ : sve_int_mlas_vvv_pred<0b0, "mla", int_aarch64_sve_mla>; + defm MLS_ZPmZZ : sve_int_mlas_vvv_pred<0b1, "mls", int_aarch64_sve_mls>; // SVE predicated integer reductions. defm SADDV_VPZ : sve_int_reduce_0_saddv<0b000, "saddv">; Index: llvm/lib/Target/AArch64/SVEInstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/SVEInstrFormats.td +++ llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1845,11 +1845,11 @@ //===----------------------------------------------------------------------===// class sve_int_mladdsub_vvv_pred sz8_64, bits<1> opc, string asm, - ZPRRegOp zprty> + ZPRRegOp zprty, ValueType pt, ValueType vt, SDPatternOperator op> : I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), asm, "\t$Zdn, $Pg/m, $Zm, $Za", "", - []>, Sched<[]> { + [(set (vt zprty:$Zdn), (op (pt PPR3bAny:$Pg), (vt zprty:$_Zdn), (vt zprty:$Zm), (vt zprty:$Za)))]>, Sched<[]> { bits<3> Pg; bits<5> Zdn; bits<5> Za; @@ -1869,19 +1869,19 @@ let ElementSize = zprty.ElementSize; } -multiclass sve_int_mladdsub_vvv_pred opc, string asm> { - def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; - def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; - def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; - def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; +multiclass sve_int_mladdsub_vvv_pred opc, string asm, SDPatternOperator op> { + def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8, nxv16i1, nxv16i8, op>; + def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16, nxv8i1, nxv8i16, op>; + def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32, nxv4i1, nxv4i32, op>; + def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64, nxv2i1, nxv2i64, op>; } class sve_int_mlas_vvv_pred sz8_64, bits<1> opc, string asm, - ZPRRegOp zprty> + ZPRRegOp zprty, ValueType pt, ValueType vt, SDPatternOperator op> : I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), asm, "\t$Zda, $Pg/m, $Zn, $Zm", "", - []>, Sched<[]> { + [(set (vt zprty:$Zda), (op (pt PPR3bAny:$Pg), (vt zprty:$_Zda), (vt zprty:$Zn), (vt zprty:$Zm)))]>, Sched<[]> { bits<3> Pg; bits<5> Zda; bits<5> Zm; @@ -1901,11 +1901,11 @@ let ElementSize = zprty.ElementSize; } -multiclass sve_int_mlas_vvv_pred opc, string asm> { - def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; - def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; - def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; - def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; +multiclass sve_int_mlas_vvv_pred opc, string asm, SDPatternOperator op> { + def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8, nxv16i1, nxv16i8, op>; + def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16, nxv8i1, nxv8i16, op>; + def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32, nxv4i1, nxv4i32, op>; + def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64, nxv2i1, nxv2i64, op>; } //===----------------------------------------------------------------------===// Index: llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-int-mad-pred.ll @@ -0,0 +1,199 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s + +define @mad_i8( %pg, %a, %b, %c) { +; CHECK-LABEL: mad_i8: +; CHECK: mad z0.b, p0/m, z1.b, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mad.nxv16i8( %pg, + %a, + %b, + %c) + ret %out +} + +define @mad_i16( %pg, %a, %b, %c) { +; CHECK-LABEL: mad_i16: +; CHECK: mad z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mad.nxv8i16( %pg, + %a, + %b, + %c) + ret %out +} + +define @mad_i32( %pg, %a, %b, %c) { +; CHECK-LABEL: mad_i32: +; CHECK: mad z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mad.nxv4i32( %pg, + %a, + %b, + %c) + ret %out +} + +define @mad_i64( %pg, %a, %b, %c) { +; CHECK-LABEL: mad_i64: +; CHECK: mad z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mad.nxv2i64( %pg, + %a, + %b, + %c) + ret %out +} + +define @msb_i8( %pg, %a, %b, %c) { +; CHECK-LABEL: msb_i8: +; CHECK: msb z0.b, p0/m, z1.b, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.msb.nxv16i8( %pg, + %a, + %b, + %c) + ret %out +} + +define @msb_i16( %pg, %a, %b, %c) { +; CHECK-LABEL: msb_i16: +; CHECK: msb z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.msb.nxv8i16( %pg, + %a, + %b, + %c) + ret %out +} + +define @msb_i32( %pg, %a, %b, %c) { +; CHECK-LABEL: msb_i32: +; CHECK: msb z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.msb.nxv4i32( %pg, + %a, + %b, + %c) + ret %out +} + +define @msb_i64( %pg, %a, %b, %c) { +; CHECK-LABEL: msb_i64: +; CHECK: msb z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.msb.nxv2i64( %pg, + %a, + %b, + %c) + ret %out +} + + +define @mla_i8( %pg, %a, %b, %c) { +; CHECK-LABEL: mla_i8: +; CHECK: mla z0.b, p0/m, z1.b, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mla.nxv16i8( %pg, + %a, + %b, + %c) + ret %out +} + +define @mla_i16( %pg, %a, %b, %c) { +; CHECK-LABEL: mla_i16: +; CHECK: mla z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mla.nxv8i16( %pg, + %a, + %b, + %c) + ret %out +} + +define @mla_i32( %pg, %a, %b, %c) { +; CHECK-LABEL: mla_i32: +; CHECK: mla z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mla.nxv4i32( %pg, + %a, + %b, + %c) + ret %out +} + +define @mla_i64( %pg, %a, %b, %c) { +; CHECK-LABEL: mla_i64: +; CHECK: mla z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mla.nxv2i64( %pg, + %a, + %b, + %c) + ret %out +} + + +define @mls_i8( %pg, %a, %b, %c) { +; CHECK-LABEL: mls_i8: +; CHECK: mls z0.b, p0/m, z1.b, z2.b +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mls.nxv16i8( %pg, + %a, + %b, + %c) + ret %out +} + +define @mls_i16( %pg, %a, %b, %c) { +; CHECK-LABEL: mls_i16: +; CHECK: mls z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mls.nxv8i16( %pg, + %a, + %b, + %c) + ret %out +} + +define @mls_i32( %pg, %a, %b, %c) { +; CHECK-LABEL: mls_i32: +; CHECK: mls z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mls.nxv4i32( %pg, + %a, + %b, + %c) + ret %out +} + +define @mls_i64( %pg, %a, %b, %c) { +; CHECK-LABEL: mls_i64: +; CHECK: mls z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.mls.nxv2i64( %pg, + %a, + %b, + %c) + ret %out +} + +declare @llvm.aarch64.sve.mad.nxv16i8(, ,,) +declare @llvm.aarch64.sve.mad.nxv8i16(, ,,) +declare @llvm.aarch64.sve.mad.nxv4i32(, ,,) +declare @llvm.aarch64.sve.mad.nxv2i64(, ,,) + +declare @llvm.aarch64.sve.msb.nxv16i8(, ,,) +declare @llvm.aarch64.sve.msb.nxv8i16(, ,,) +declare @llvm.aarch64.sve.msb.nxv4i32(, ,,) +declare @llvm.aarch64.sve.msb.nxv2i64(, ,,) + +declare @llvm.aarch64.sve.mla.nxv16i8(, ,,) +declare @llvm.aarch64.sve.mla.nxv8i16(, ,,) +declare @llvm.aarch64.sve.mla.nxv4i32(, ,,) +declare @llvm.aarch64.sve.mla.nxv2i64(, ,,) + +declare @llvm.aarch64.sve.mls.nxv16i8(, ,,) +declare @llvm.aarch64.sve.mls.nxv8i16(, ,,) +declare @llvm.aarch64.sve.mls.nxv4i32(, ,,) +declare @llvm.aarch64.sve.mls.nxv2i64(, ,,)