Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -997,6 +997,10 @@ // These operations are not supported on NEON but SVE can do them. setOperationAction(ISD::MUL, MVT::v1i64, Custom); setOperationAction(ISD::MUL, MVT::v2i64, Custom); + setOperationAction(ISD::SDIV, MVT::v2i32, Custom); + setOperationAction(ISD::SDIV, MVT::v4i32, Custom); + setOperationAction(ISD::SDIV, MVT::v1i64, Custom); + setOperationAction(ISD::SDIV, MVT::v2i64, Custom); setOperationAction(ISD::SMAX, MVT::v1i64, Custom); setOperationAction(ISD::SMAX, MVT::v2i64, Custom); setOperationAction(ISD::SMIN, MVT::v1i64, Custom); @@ -1129,6 +1133,12 @@ setOperationAction(ISD::UMIN, VT, Custom); setOperationAction(ISD::XOR, VT, Custom); setOperationAction(ISD::ZERO_EXTEND, VT, Custom); + + if (VT.getVectorElementType() == MVT::i32 || + VT.getVectorElementType() == MVT::i64) { + setOperationAction(ISD::SDIV, VT, Custom); + setOperationAction(ISD::UDIV, VT, Custom); + } } void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { @@ -8925,6 +8935,12 @@ EVT VT = Op.getValueType(); bool Signed = Op.getOpcode() == ISD::SDIV; unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED; + + if (useSVEForFixedLengthVectorVT(Op.getValueType(), /*OverrideNEON=*/true) && + (VT.getVectorElementType() == MVT::i32 || + VT.getVectorElementType() == MVT::i64)) + return LowerToPredicatedOp(Op, DAG, PredOpcode, /*OverrideNEON=*/true); + if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64) return LowerToPredicatedOp(Op, DAG, PredOpcode); Index: llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll +++ llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll @@ -419,6 +419,170 @@ ; VBYTES because the add tests already validate the legalisation code paths. ; +; +; SDIV +; + +; Vector v2i32 sdiv are not legal for NEON so use SVE when available. +define <2 x i32> @sdiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 { +; CHECK-LABEL: sdiv_v2i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),2)]] +; CHECK: sdiv z0.s, [[PG]]/m, z0.s, z1.s +; CHECK: ret + %res = sdiv <2 x i32> %op1, %op2 + ret <2 x i32> %res +} + +; Vector v4i32 sdiv are not legal for NEON so use SVE when available. +define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 { +; CHECK-LABEL: sdiv_v4i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),4)]] +; CHECK: sdiv z0.s, [[PG]]/m, z0.s, z1.s +; CHECK: ret + %res = sdiv <4 x i32> %op1, %op2 + ret <4 x i32> %res +} + +define void @sdiv_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { +; CHECK-LABEL: sdiv_v8i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]] +; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] +; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s +; CHECK: st1w { [[RES]].s }, [[PG]], [x0] +; CHECK: ret + %op1 = load <8 x i32>, <8 x i32>* %a + %op2 = load <8 x i32>, <8 x i32>* %b + %res = sdiv <8 x i32> %op1, %op2 + store <8 x i32> %res, <8 x i32>* %a + ret void +} + +define void @sdiv_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { +; CHECK-LABEL: sdiv_v16i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]] +; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] +; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s +; CHECK: st1w { [[RES]].s }, [[PG]], [x0] +; CHECK: ret + %op1 = load <16 x i32>, <16 x i32>* %a + %op2 = load <16 x i32>, <16 x i32>* %b + %res = sdiv <16 x i32> %op1, %op2 + store <16 x i32> %res, <16 x i32>* %a + ret void +} + +define void @sdiv_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 { +; CHECK-LABEL: sdiv_v32i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]] +; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] +; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s +; CHECK: st1w { [[RES]].s }, [[PG]], [x0] +; CHECK: ret + %op1 = load <32 x i32>, <32 x i32>* %a + %op2 = load <32 x i32>, <32 x i32>* %b + %res = sdiv <32 x i32> %op1, %op2 + store <32 x i32> %res, <32 x i32>* %a + ret void +} + +define void @sdiv_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 { +; CHECK-LABEL: sdiv_v64i32: +; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]] +; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] +; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s +; CHECK: st1w { [[RES]].s }, [[PG]], [x0] +; CHECK: ret + %op1 = load <64 x i32>, <64 x i32>* %a + %op2 = load <64 x i32>, <64 x i32>* %b + %res = sdiv <64 x i32> %op1, %op2 + store <64 x i32> %res, <64 x i32>* %a + ret void +} + +; Vector i64 sdiv are not legal for NEON so use SVE when available. +define <1 x i64> @sdiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 { +; CHECK-LABEL: sdiv_v1i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl1 +; CHECK: sdiv z0.d, [[PG]]/m, z0.d, z1.d +; CHECK: ret + %res = sdiv <1 x i64> %op1, %op2 + ret <1 x i64> %res +} + +; Vector i64 sdiv are not legal for NEON so use SVE when available. +define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 { +; CHECK-LABEL: sdiv_v2i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl2 +; CHECK: sdiv z0.d, [[PG]]/m, z0.d, z1.d +; CHECK: ret + %res = sdiv <2 x i64> %op1, %op2 + ret <2 x i64> %res +} + +define void @sdiv_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { +; CHECK-LABEL: sdiv_v4i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]] +; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] +; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d +; CHECK: st1d { [[RES]].d }, [[PG]], [x0] +; CHECK: ret + %op1 = load <4 x i64>, <4 x i64>* %a + %op2 = load <4 x i64>, <4 x i64>* %b + %res = sdiv <4 x i64> %op1, %op2 + store <4 x i64> %res, <4 x i64>* %a + ret void +} + +define void @sdiv_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { +; CHECK-LABEL: sdiv_v8i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]] +; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] +; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d +; CHECK: st1d { [[RES]].d }, [[PG]], [x0] +; CHECK: ret + %op1 = load <8 x i64>, <8 x i64>* %a + %op2 = load <8 x i64>, <8 x i64>* %b + %res = sdiv <8 x i64> %op1, %op2 + store <8 x i64> %res, <8 x i64>* %a + ret void +} + +define void @sdiv_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 { +; CHECK-LABEL: sdiv_v16i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]] +; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] +; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d +; CHECK: st1d { [[RES]].d }, [[PG]], [x0] +; CHECK: ret + %op1 = load <16 x i64>, <16 x i64>* %a + %op2 = load <16 x i64>, <16 x i64>* %b + %res = sdiv <16 x i64> %op1, %op2 + store <16 x i64> %res, <16 x i64>* %a + ret void +} + +define void @sdiv_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 { +; CHECK-LABEL: sdiv_v32i64: +; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]] +; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] +; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] +; CHECK: sdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d +; CHECK: st1d { [[RES]].d }, [[PG]], [x0] +; CHECK: ret + %op1 = load <32 x i64>, <32 x i64>* %a + %op2 = load <32 x i64>, <32 x i64>* %b + %res = sdiv <32 x i64> %op1, %op2 + store <32 x i64> %res, <32 x i64>* %a + ret void +} + ; ; MUL ;