diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4628,6 +4628,20 @@ ID, Op.getOperand(1), Op.getOperand(2))); } return SDValue(); + case Intrinsic::aarch64_sve_cntp: + if (Op.getOperand(1).getValueType() == MVT::nxv1i1) { + // The other (unpacked lanes) may not be zeroed, so zero + // them explicitly here. + SDValue Op1 = getSVESafeBitCast(MVT::nxv2i1, Op.getOperand(1), DAG); + SDValue Op2 = getSVESafeBitCast(MVT::nxv2i1, Op.getOperand(2), DAG); + Op1 = DAG.getNode(ISD::AND, dl, MVT::nxv2i1, + DAG.getConstant(1, dl, MVT::nxv2i1), Op1); + return DAG.getNode( + ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(), + DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, dl, MVT::i64), + Op1, Op2); + } + return SDValue(); case Intrinsic::localaddress: { const auto &MF = DAG.getMachineFunction(); const auto *RegInfo = Subtarget->getRegisterInfo(); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2075,6 +2075,8 @@ (PTEST_PP PPR:$pg, PPR:$src)>; def : Pat<(AArch64ptest (nxv2i1 PPR:$pg), (nxv2i1 PPR:$src)), (PTEST_PP PPR:$pg, PPR:$src)>; + def : Pat<(AArch64ptest (nxv1i1 PPR:$pg), (nxv1i1 PPR:$src)), + (PTEST_PP PPR:$pg, PPR:$src)>; let AddedComplexity = 1 in { class LD1RPat; def : Pat<(nxv2i1 (and PPR:$Ps1, PPR:$Ps2)), (AND_PPzPP (PTRUE_D 31), PPR:$Ps1, PPR:$Ps2)>; + // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. + def : Pat<(nxv1i1 (and PPR:$Ps1, PPR:$Ps2)), + (AND_PPzPP (PTRUE_D 31), PPR:$Ps1, PPR:$Ps2)>; // Add more complex addressing modes here as required multiclass pred_load(NAME), PTRUE_S>; def : SVE_2_Op_AllActive_Pat(NAME), PTRUE_D>; + // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. + def : SVE_2_Op_AllActive_Pat(NAME), PTRUE_D>; } // An instance of sve_int_pred_log_and but uses op_nopred's first operand as the @@ -1683,6 +1686,9 @@ (!cast(NAME) $Op1, $Op1, $Op2)>; def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)), (!cast(NAME) $Op1, $Op1, $Op2)>; + // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. + def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)), + (!cast(NAME) $Op1, $Op1, $Op2)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve-int-log.ll b/llvm/test/CodeGen/AArch64/sve-int-log.ll --- a/llvm/test/CodeGen/AArch64/sve-int-log.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-log.ll @@ -46,6 +46,15 @@ ret %res } +define @and_pred_q( %a, %b) { +; CHECK-LABEL: and_pred_q: +; CHECK: // %bb.0: +; CHECK-NEXT: and p0.b, p0/z, p0.b, p1.b +; CHECK-NEXT: ret + %res = and %a, %b + ret %res +} + define @and_pred_d( %a, %b) { ; CHECK-LABEL: and_pred_d: ; CHECK: // %bb.0: @@ -126,6 +135,17 @@ ret %res } +define @bic_pred_q( %a, %b) { +; CHECK-LABEL: bic_pred_q: +; CHECK: // %bb.0: +; CHECK-NEXT: bic p0.b, p0/z, p0.b, p1.b +; CHECK-NEXT: ret + %allones = shufflevector insertelement( undef, i1 true, i32 0), undef, zeroinitializer + %not_b = xor %b, %allones + %res = and %a, %not_b + ret %res +} + define @bic_pred_d( %a, %b) { ; CHECK-LABEL: bic_pred_d: ; CHECK: // %bb.0: @@ -214,6 +234,15 @@ ret %res } +define @or_pred_q( %a, %b) { +; CHECK-LABEL: or_pred_q: +; CHECK: // %bb.0: +; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b +; CHECK-NEXT: ret + %res = or %a, %b + ret %res +} + define @or_pred_d( %a, %b) { ; CHECK-LABEL: or_pred_d: ; CHECK: // %bb.0: @@ -294,6 +323,16 @@ ret %res } +define @xor_pred_q( %a, %b) { +; CHECK-LABEL: xor_pred_q: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p2.d +; CHECK-NEXT: eor p0.b, p2/z, p0.b, p1.b +; CHECK-NEXT: ret + %res = xor %a, %b + ret %res +} + define @xor_pred_d( %a, %b) { ; CHECK-LABEL: xor_pred_d: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll --- a/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll @@ -51,6 +51,19 @@ ret i1 %res } +define i1 @reduce_and_nxv1i1( %vec) { +; CHECK-LABEL: reduce_and_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: eor p0.b, p1/z, p0.b, p2.b +; CHECK-NEXT: ptest p2, p0.b +; CHECK-NEXT: cset w0, eq +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.and.i1.nxv1i1( %vec) + ret i1 %res +} + ; ORV define i1 @reduce_or_nxv16i1( %vec) { @@ -93,6 +106,18 @@ ret i1 %res } +define i1 @reduce_or_nxv1i1( %vec) { +; CHECK-LABEL: reduce_or_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: ptest p1, p0.b +; CHECK-NEXT: cset w0, ne +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.or.i1.nxv1i1( %vec) + ret i1 %res +} + ; XORV define i1 @reduce_xor_nxv16i1( %vec) { @@ -139,6 +164,18 @@ ret i1 %res } +define i1 @reduce_xor_nxv1i1( %vec) { +; CHECK-LABEL: reduce_xor_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: cntp x8, p1, p0.d +; CHECK-NEXT: and w0, w8, #0x1 +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.xor.i1.nxv1i1( %vec) + ret i1 %res +} + ; SMAXV define i1 @reduce_smax_nxv16i1( %vec) { @@ -189,6 +226,19 @@ ret i1 %res } +define i1 @reduce_smax_nxv1i1( %vec) { +; CHECK-LABEL: reduce_smax_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: eor p0.b, p1/z, p0.b, p2.b +; CHECK-NEXT: ptest p2, p0.b +; CHECK-NEXT: cset w0, eq +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.smax.i1.nxv1i1( %vec) + ret i1 %res +} + ; SMINV define i1 @reduce_smin_nxv16i1( %vec) { @@ -231,6 +281,18 @@ ret i1 %res } +define i1 @reduce_smin_nxv1i1( %vec) { +; CHECK-LABEL: reduce_smin_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: ptest p1, p0.b +; CHECK-NEXT: cset w0, ne +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.smin.i1.nxv1i1( %vec) + ret i1 %res +} + ; UMAXV define i1 @reduce_umax_nxv16i1( %vec) { @@ -273,6 +335,18 @@ ret i1 %res } +define i1 @reduce_umax_nxv1i1( %vec) { +; CHECK-LABEL: reduce_umax_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: ptest p1, p0.b +; CHECK-NEXT: cset w0, ne +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.umax.i1.nxv1i1( %vec) + ret i1 %res +} + ; UMINV define i1 @reduce_umin_nxv16i1( %vec) { @@ -311,6 +385,19 @@ ret i1 %res } +define i1 @reduce_umin_nxv1i1( %vec) { +; CHECK-LABEL: reduce_umin_nxv1i1: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: eor p0.b, p1/z, p0.b, p2.b +; CHECK-NEXT: ptest p2, p0.b +; CHECK-NEXT: cset w0, eq +; CHECK-NEXT: ret + %res = call i1 @llvm.vector.reduce.umin.i1.nxv1i1( %vec) + ret i1 %res +} + define i1 @reduce_umin_nxv2i1( %vec) { ; CHECK-LABEL: reduce_umin_nxv2i1: ; CHECK: // %bb.0: @@ -327,33 +414,40 @@ declare i1 @llvm.vector.reduce.and.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.and.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.and.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.and.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.or.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.or.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.or.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.or.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.or.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.xor.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.xor.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.xor.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.xor.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.xor.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.smin.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.smin.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.smin.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.smin.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.smin.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.smax.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.smax.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.smax.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.smax.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.smax.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.umin.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.umin.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.umin.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.umin.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.umin.i1.nxv1i1( %vec) declare i1 @llvm.vector.reduce.umax.i1.nxv16i1( %vec) declare i1 @llvm.vector.reduce.umax.i1.nxv8i1( %vec) declare i1 @llvm.vector.reduce.umax.i1.nxv4i1( %vec) declare i1 @llvm.vector.reduce.umax.i1.nxv2i1( %vec) +declare i1 @llvm.vector.reduce.umax.i1.nxv1i1( %vec) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll @@ -44,6 +44,17 @@ ret %out } +define @reinterpret_bool_from_q( %arg) { +; CHECK-LABEL: reinterpret_bool_from_q: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: and p0.b, p0/z, p0.b, p1.b +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.convert.to.svbool.nxv1i1( %arg) + ret %res +} + ; ; Converting from svbool_t ; @@ -80,6 +91,14 @@ ret %out } +define @reinterpret_bool_to_q( %pg) { +; CHECK-LABEL: reinterpret_bool_to_q: +; CHECK: // %bb.0: +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.convert.from.svbool.nxv1i1( %pg) + ret %out +} + ; Reinterpreting a ptrue should not introduce an `and` instruction. define @reinterpret_ptrue() { ; CHECK-LABEL: reinterpret_ptrue: @@ -124,8 +143,10 @@ declare @llvm.aarch64.sve.convert.to.svbool.nxv8i1() declare @llvm.aarch64.sve.convert.to.svbool.nxv4i1() declare @llvm.aarch64.sve.convert.to.svbool.nxv2i1() +declare @llvm.aarch64.sve.convert.to.svbool.nxv1i1() declare @llvm.aarch64.sve.convert.from.svbool.nxv16i1() declare @llvm.aarch64.sve.convert.from.svbool.nxv8i1() declare @llvm.aarch64.sve.convert.from.svbool.nxv4i1() declare @llvm.aarch64.sve.convert.from.svbool.nxv2i1() +declare @llvm.aarch64.sve.convert.from.svbool.nxv1i1()