This is an archive of the discontinued LLVM Phabricator instance.

Differential D84034

[SVE] Add lowering for fixed length vector fdiv, fma, fmul and fsub operations.
ClosedPublic

Authored by paulwalker-arm on Jul 17 2020, 8:19 AM.

Details

Reviewers
efriedma
cameron.mcinally
david-arm
Commits
rG6384ec409944: [SVE] Add lowering for fixed length vector fdiv, fma, fmul and fsub operations.

Diff Detail

Event Timeline

paulwalker-arm created this revision.Jul 17 2020, 8:19 AM
Herald added a reviewer: efriedma. · View Herald TranscriptJul 17 2020, 8:19 AM
Herald added a project: Restricted Project. · View Herald Transcript
Herald added subscribers: llvm-commits, psnobl, hiraditya, tschuett. · View Herald Transcript
paulwalker-arm added reviewers: cameron.mcinally, david-arm.Jul 17 2020, 8:20 AM

This is just a clone of the work already done for FADD.

Harbormaster failed remote builds in B64687: Diff 278780!Jul 17 2020, 8:49 AM
cameron.mcinally accepted this revision.Jul 17 2020, 1:05 PM

LGTM

Do we need to worry about fixed types less than 64b? E.g. <1 x float>.

This revision is now accepted and ready to land.Jul 17 2020, 1:05 PM
paulwalker-arm added a comment.EditedJul 17 2020, 3:49 PM
In D84034#2159377, @cameron.mcinally wrote:

Do we need to worry about fixed types less than 64b? E.g. <1 x float>.

At some point I think yes. Currently SVE fixed length code generation tries to remain compatible with the way NEON does things for the types it already supported. That means <1 x float> will be scalarised (depending on the type and operation widening is likely to be the next option). It's an area that really needs an ABI defining before we can break away from NEON.

Closed by commit rG6384ec409944: [SVE] Add lowering for fixed length vector fdiv, fma, fmul and fsub operations. (authored by paulwalker-arm). · Explain WhyJul 20 2020, 5:00 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
lib/
Target/
AArch64/
29 lines
test/
CodeGen/
AArch64/
1020 lines

Diff 279183

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 1,067 Lines • ▼ Show 20 Lines for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
setOperationAction(Op, VT, Expand); setOperationAction(Op, VT, Expand);
// We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one. // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
// Lower fixed length vector operations to scalable equivalents. // Lower fixed length vector operations to scalable equivalents.
setOperationAction(ISD::ADD, VT, Custom); setOperationAction(ISD::ADD, VT, Custom);
setOperationAction(ISD::FADD, VT, Custom); setOperationAction(ISD::FADD, VT, Custom);
setOperationAction(ISD::FDIV, VT, Custom);
setOperationAction(ISD::FMA, VT, Custom);
setOperationAction(ISD::FMUL, VT, Custom);
setOperationAction(ISD::FSUB, VT, Custom);
setOperationAction(ISD::LOAD, VT, Custom); setOperationAction(ISD::LOAD, VT, Custom);
setOperationAction(ISD::STORE, VT, Custom); setOperationAction(ISD::STORE, VT, Custom);
setOperationAction(ISD::TRUNCATE, VT, Custom); setOperationAction(ISD::TRUNCATE, VT, Custom);
} }
void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
addRegisterClass(VT, &AArch64::FPR64RegClass); addRegisterClass(VT, &AArch64::FPR64RegClass);
addTypeForNEON(VT, MVT::v2i32); addTypeForNEON(VT, MVT::v2i32);
▲ Show 20 Lines • Show All 2,385 Lines • ▼ Show 20 LinesSDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::SADDO: case ISD::SADDO:
case ISD::UADDO: case ISD::UADDO:
case ISD::SSUBO: case ISD::SSUBO:
case ISD::USUBO: case ISD::USUBO:
case ISD::SMULO: case ISD::SMULO:
case ISD::UMULO: case ISD::UMULO:
return LowerXALUO(Op, DAG); return LowerXALUO(Op, DAG);
case ISD::FADD: case ISD::FADD:
if (Op.getValueType().isScalableVector() || if (Op.getValueType() == MVT::f128)
useSVEForFixedLengthVectorVT(Op.getValueType()))
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
return LowerF128Call(Op, DAG, RTLIB::ADD_F128); return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
case ISD::FSUB: case ISD::FSUB:
if (Op.getValueType().isScalableVector()) if (Op.getValueType() == MVT::f128)
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
return LowerF128Call(Op, DAG, RTLIB::SUB_F128); return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
case ISD::FMUL: case ISD::FMUL:
if (Op.getValueType().isScalableVector()) if (Op.getValueType() == MVT::f128)
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
return LowerF128Call(Op, DAG, RTLIB::MUL_F128); return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
case ISD::FMA: case ISD::FMA:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED); return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED);
case ISD::FDIV: case ISD::FDIV:
if (Op.getValueType().isScalableVector()) if (Op.getValueType() == MVT::f128)
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
return LowerF128Call(Op, DAG, RTLIB::DIV_F128); return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
case ISD::FP_ROUND: case ISD::FP_ROUND:
case ISD::STRICT_FP_ROUND: case ISD::STRICT_FP_ROUND:
return LowerFP_ROUND(Op, DAG); return LowerFP_ROUND(Op, DAG);
case ISD::FP_EXTEND: case ISD::FP_EXTEND:
return LowerFP_EXTEND(Op, DAG); return LowerFP_EXTEND(Op, DAG);
case ISD::FRAMEADDR: case ISD::FRAMEADDR:
return LowerFRAMEADDR(Op, DAG); return LowerFRAMEADDR(Op, DAG);
case ISD::SPONENTRY: case ISD::SPONENTRY:
▲ Show 20 Lines • Show All 11,667 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll

Show All 19 Lines
; only power-of-2 vector lengths are considered legal, regardless of the ; only power-of-2 vector lengths are considered legal, regardless of the
; user specified vector length. ; user specified vector length.
target triple = "aarch64-unknown-linux-gnu" target triple = "aarch64-unknown-linux-gnu"
; Don't use SVE when its registers are no bigger than NEON. ; Don't use SVE when its registers are no bigger than NEON.
; NO_SVE-NOT: ptrue ; NO_SVE-NOT: ptrue
;
; FADD
;
; Don't use SVE for 64-bit vectors. ; Don't use SVE for 64-bit vectors.
define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) #0 { define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
; CHECK-LABEL: @fadd_v4f16 ; CHECK-LABEL: @fadd_v4f16
; CHECK: fadd v0.4h, v0.4h, v1.4h ; CHECK: fadd v0.4h, v0.4h, v1.4h
; CHECK: ret ; CHECK: ret
%res = fadd <4 x half> %op1, %op2 %res = fadd <4 x half> %op1, %op2
ret <4 x half> %res ret <4 x half> %res
} }
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines
; CHECK: ret ; CHECK: ret
%op1 = load <64 x half>, <64 x half>* %a %op1 = load <64 x half>, <64 x half>* %a
%op2 = load <64 x half>, <64 x half>* %b %op2 = load <64 x half>, <64 x half>* %b
%res = fadd <64 x half> %op1, %op2 %res = fadd <64 x half> %op1, %op2
store <64 x half> %res, <64 x half>* %a store <64 x half> %res, <64 x half>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v128f16(<128 x half>* %a, <128 x half>* %b) #0 { define void @fadd_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
; CHECK-LABEL: @fadd_v128f16 ; CHECK-LABEL: @fadd_v128f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]] ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0] ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1] ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h ; CHECK: fadd [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0] ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
Show All 33 Lines
; CHECK: ret ; CHECK: ret
%op1 = load <8 x float>, <8 x float>* %a %op1 = load <8 x float>, <8 x float>* %a
%op2 = load <8 x float>, <8 x float>* %b %op2 = load <8 x float>, <8 x float>* %b
%res = fadd <8 x float> %op1, %op2 %res = fadd <8 x float> %op1, %op2
store <8 x float> %res, <8 x float>* %a store <8 x float> %res, <8 x float>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v16f32(<16 x float>* %a, <16 x float>* %b) #0 { define void @fadd_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
; CHECK-LABEL: @fadd_v16f32 ; CHECK-LABEL: @fadd_v16f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]] ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s ; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0] ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret ; CHECK: ret
%op1 = load <16 x float>, <16 x float>* %a %op1 = load <16 x float>, <16 x float>* %a
%op2 = load <16 x float>, <16 x float>* %b %op2 = load <16 x float>, <16 x float>* %b
%res = fadd <16 x float> %op1, %op2 %res = fadd <16 x float> %op1, %op2
store <16 x float> %res, <16 x float>* %a store <16 x float> %res, <16 x float>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v32f32(<32 x float>* %a, <32 x float>* %b) #0 { define void @fadd_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
; CHECK-LABEL: @fadd_v32f32 ; CHECK-LABEL: @fadd_v32f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]] ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s ; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0] ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret ; CHECK: ret
%op1 = load <32 x float>, <32 x float>* %a %op1 = load <32 x float>, <32 x float>* %a
%op2 = load <32 x float>, <32 x float>* %b %op2 = load <32 x float>, <32 x float>* %b
%res = fadd <32 x float> %op1, %op2 %res = fadd <32 x float> %op1, %op2
store <32 x float> %res, <32 x float>* %a store <32 x float> %res, <32 x float>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v64f32(<64 x float>* %a, <64 x float>* %b) #0 { define void @fadd_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
; CHECK-LABEL: @fadd_v64f32 ; CHECK-LABEL: @fadd_v64f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]] ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0] ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1] ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s ; CHECK: fadd [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0] ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
Show All 33 Lines
; CHECK: ret ; CHECK: ret
%op1 = load <4 x double>, <4 x double>* %a %op1 = load <4 x double>, <4 x double>* %a
%op2 = load <4 x double>, <4 x double>* %b %op2 = load <4 x double>, <4 x double>* %b
%res = fadd <4 x double> %op1, %op2 %res = fadd <4 x double> %op1, %op2
store <4 x double> %res, <4 x double>* %a store <4 x double> %res, <4 x double>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v8f64(<8 x double>* %a, <8 x double>* %b) #0 { define void @fadd_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
; CHECK-LABEL: @fadd_v8f64 ; CHECK-LABEL: @fadd_v8f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]] ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d ; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0] ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret ; CHECK: ret
%op1 = load <8 x double>, <8 x double>* %a %op1 = load <8 x double>, <8 x double>* %a
%op2 = load <8 x double>, <8 x double>* %b %op2 = load <8 x double>, <8 x double>* %b
%res = fadd <8 x double> %op1, %op2 %res = fadd <8 x double> %op1, %op2
store <8 x double> %res, <8 x double>* %a store <8 x double> %res, <8 x double>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v16f64(<16 x double>* %a, <16 x double>* %b) #0 { define void @fadd_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
; CHECK-LABEL: @fadd_v16f64 ; CHECK-LABEL: @fadd_v16f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]] ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d ; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0] ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret ; CHECK: ret
%op1 = load <16 x double>, <16 x double>* %a %op1 = load <16 x double>, <16 x double>* %a
%op2 = load <16 x double>, <16 x double>* %b %op2 = load <16 x double>, <16 x double>* %b
%res = fadd <16 x double> %op1, %op2 %res = fadd <16 x double> %op1, %op2
store <16 x double> %res, <16 x double>* %a store <16 x double> %res, <16 x double>* %a
ret void ret void
} }
; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests ; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
; already cover the general legalisation cases. ; already cover the general legalisation cases.
define void @fadd_v32f64(<32 x double>* %a, <32 x double>* %b) #0 { define void @fadd_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
; CHECK-LABEL: @fadd_v32f64 ; CHECK-LABEL: @fadd_v32f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]] ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0] ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1] ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d ; CHECK: fadd [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0] ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret ; CHECK: ret
%op1 = load <32 x double>, <32 x double>* %a %op1 = load <32 x double>, <32 x double>* %a
%op2 = load <32 x double>, <32 x double>* %b %op2 = load <32 x double>, <32 x double>* %b
%res = fadd <32 x double> %op1, %op2 %res = fadd <32 x double> %op1, %op2
store <32 x double> %res, <32 x double>* %a store <32 x double> %res, <32 x double>* %a
ret void ret void
} }
;
; NOTE: Tests beyond this point only have CHECK lines to validate the first
; VBYTES because the fadd tests already validate the legalisation code paths.
;
;
; FDIV
;
; Don't use SVE for 64-bit vectors.
define <4 x half> @fdiv_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
; CHECK-LABEL: @fdiv_v4f16
; CHECK: fdiv v0.4h, v0.4h, v1.4h
; CHECK: ret
%res = fdiv <4 x half> %op1, %op2
ret <4 x half> %res
}
; Don't use SVE for 128-bit vectors.
define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
; CHECK-LABEL: @fdiv_v8f16
; CHECK: fdiv v0.8h, v0.8h, v1.8h
; CHECK: ret
%res = fdiv <8 x half> %op1, %op2
ret <8 x half> %res
}
define void @fdiv_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
; CHECK-LABEL: @fdiv_v16f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x half>, <16 x half>* %a
%op2 = load <16 x half>, <16 x half>* %b
%res = fdiv <16 x half> %op1, %op2
store <16 x half> %res, <16 x half>* %a
ret void
}
define void @fdiv_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
; CHECK-LABEL: @fdiv_v32f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x half>, <32 x half>* %a
%op2 = load <32 x half>, <32 x half>* %b
%res = fdiv <32 x half> %op1, %op2
store <32 x half> %res, <32 x half>* %a
ret void
}
define void @fdiv_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
; CHECK-LABEL: @fdiv_v64f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x half>, <64 x half>* %a
%op2 = load <64 x half>, <64 x half>* %b
%res = fdiv <64 x half> %op1, %op2
store <64 x half> %res, <64 x half>* %a
ret void
}
define void @fdiv_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
; CHECK-LABEL: @fdiv_v128f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <128 x half>, <128 x half>* %a
%op2 = load <128 x half>, <128 x half>* %b
%res = fdiv <128 x half> %op1, %op2
store <128 x half> %res, <128 x half>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <2 x float> @fdiv_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
; CHECK-LABEL: @fdiv_v2f32
; CHECK: fdiv v0.2s, v0.2s, v1.2s
; CHECK: ret
%res = fdiv <2 x float> %op1, %op2
ret <2 x float> %res
}
; Don't use SVE for 128-bit vectors.
define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
; CHECK-LABEL: @fdiv_v4f32
; CHECK: fdiv v0.4s, v0.4s, v1.4s
; CHECK: ret
%res = fdiv <4 x float> %op1, %op2
ret <4 x float> %res
}
define void @fdiv_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
; CHECK-LABEL: @fdiv_v8f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x float>, <8 x float>* %a
%op2 = load <8 x float>, <8 x float>* %b
%res = fdiv <8 x float> %op1, %op2
store <8 x float> %res, <8 x float>* %a
ret void
}
define void @fdiv_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
; CHECK-LABEL: @fdiv_v16f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x float>, <16 x float>* %a
%op2 = load <16 x float>, <16 x float>* %b
%res = fdiv <16 x float> %op1, %op2
store <16 x float> %res, <16 x float>* %a
ret void
}
define void @fdiv_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
; CHECK-LABEL: @fdiv_v32f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x float>, <32 x float>* %a
%op2 = load <32 x float>, <32 x float>* %b
%res = fdiv <32 x float> %op1, %op2
store <32 x float> %res, <32 x float>* %a
ret void
}
define void @fdiv_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
; CHECK-LABEL: @fdiv_v64f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x float>, <64 x float>* %a
%op2 = load <64 x float>, <64 x float>* %b
%res = fdiv <64 x float> %op1, %op2
store <64 x float> %res, <64 x float>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <1 x double> @fdiv_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
; CHECK-LABEL: @fdiv_v1f64
; CHECK: fdiv d0, d0, d1
; CHECK: ret
%res = fdiv <1 x double> %op1, %op2
ret <1 x double> %res
}
; Don't use SVE for 128-bit vectors.
define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
; CHECK-LABEL: @fdiv_v2f64
; CHECK: fdiv v0.2d, v0.2d, v1.2d
; CHECK: ret
%res = fdiv <2 x double> %op1, %op2
ret <2 x double> %res
}
define void @fdiv_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
; CHECK-LABEL: @fdiv_v4f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <4 x double>, <4 x double>* %a
%op2 = load <4 x double>, <4 x double>* %b
%res = fdiv <4 x double> %op1, %op2
store <4 x double> %res, <4 x double>* %a
ret void
}
define void @fdiv_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
; CHECK-LABEL: @fdiv_v8f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x double>, <8 x double>* %a
%op2 = load <8 x double>, <8 x double>* %b
%res = fdiv <8 x double> %op1, %op2
store <8 x double> %res, <8 x double>* %a
ret void
}
define void @fdiv_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
; CHECK-LABEL: @fdiv_v16f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x double>, <16 x double>* %a
%op2 = load <16 x double>, <16 x double>* %b
%res = fdiv <16 x double> %op1, %op2
store <16 x double> %res, <16 x double>* %a
ret void
}
define void @fdiv_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
; CHECK-LABEL: @fdiv_v32f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x double>, <32 x double>* %a
%op2 = load <32 x double>, <32 x double>* %b
%res = fdiv <32 x double> %op1, %op2
store <32 x double> %res, <32 x double>* %a
ret void
}
;
; FMA
;
; Don't use SVE for 64-bit vectors.
define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3) #0 {
; CHECK-LABEL: @fma_v4f16
; CHECK: fmla v2.4h, v1.4h, v0.4h
; CHECK: ret
%res = call <4 x half> @llvm.fma.v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)
ret <4 x half> %res
}
; Don't use SVE for 128-bit vectors.
define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) #0 {
; CHECK-LABEL: @fma_v8f16
; CHECK: fmla v2.8h, v1.8h, v0.8h
; CHECK: ret
%res = call <8 x half> @llvm.fma.v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)
ret <8 x half> %res
}
define void @fma_v16f16(<16 x half>* %a, <16 x half>* %b, <16 x half>* %c) #0 {
; CHECK-LABEL: @fma_v16f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x half>, <16 x half>* %a
%op2 = load <16 x half>, <16 x half>* %b
%op3 = load <16 x half>, <16 x half>* %c
%res = call <16 x half> @llvm.fma.v16f16(<16 x half> %op1, <16 x half> %op2, <16 x half> %op3)
store <16 x half> %res, <16 x half>* %a
ret void
}
define void @fma_v32f16(<32 x half>* %a, <32 x half>* %b, <32 x half>* %c) #0 {
; CHECK-LABEL: @fma_v32f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x half>, <32 x half>* %a
%op2 = load <32 x half>, <32 x half>* %b
%op3 = load <32 x half>, <32 x half>* %c
%res = call <32 x half> @llvm.fma.v32f16(<32 x half> %op1, <32 x half> %op2, <32 x half> %op3)
store <32 x half> %res, <32 x half>* %a
ret void
}
define void @fma_v64f16(<64 x half>* %a, <64 x half>* %b, <64 x half>* %c) #0 {
; CHECK-LABEL: @fma_v64f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x half>, <64 x half>* %a
%op2 = load <64 x half>, <64 x half>* %b
%op3 = load <64 x half>, <64 x half>* %c
%res = call <64 x half> @llvm.fma.v64f16(<64 x half> %op1, <64 x half> %op2, <64 x half> %op3)
store <64 x half> %res, <64 x half>* %a
ret void
}
define void @fma_v128f16(<128 x half>* %a, <128 x half>* %b, <128 x half>* %c) #0 {
; CHECK-LABEL: @fma_v128f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <128 x half>, <128 x half>* %a
%op2 = load <128 x half>, <128 x half>* %b
%op3 = load <128 x half>, <128 x half>* %c
%res = call <128 x half> @llvm.fma.v128f16(<128 x half> %op1, <128 x half> %op2, <128 x half> %op3)
store <128 x half> %res, <128 x half>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3) #0 {
; CHECK-LABEL: @fma_v2f32
; CHECK: fmla v2.2s, v1.2s, v0.2s
; CHECK: ret
%res = call <2 x float> @llvm.fma.v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3)
ret <2 x float> %res
}
; Don't use SVE for 128-bit vectors.
define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3) #0 {
; CHECK-LABEL: @fma_v4f32
; CHECK: fmla v2.4s, v1.4s, v0.4s
; CHECK: ret
%res = call <4 x float> @llvm.fma.v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3)
ret <4 x float> %res
}
define void @fma_v8f32(<8 x float>* %a, <8 x float>* %b, <8 x float>* %c) #0 {
; CHECK-LABEL: @fma_v8f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x float>, <8 x float>* %a
%op2 = load <8 x float>, <8 x float>* %b
%op3 = load <8 x float>, <8 x float>* %c
%res = call <8 x float> @llvm.fma.v8f32(<8 x float> %op1, <8 x float> %op2, <8 x float> %op3)
store <8 x float> %res, <8 x float>* %a
ret void
}
define void @fma_v16f32(<16 x float>* %a, <16 x float>* %b, <16 x float>* %c) #0 {
; CHECK-LABEL: @fma_v16f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x float>, <16 x float>* %a
%op2 = load <16 x float>, <16 x float>* %b
%op3 = load <16 x float>, <16 x float>* %c
%res = call <16 x float> @llvm.fma.v16f32(<16 x float> %op1, <16 x float> %op2, <16 x float> %op3)
store <16 x float> %res, <16 x float>* %a
ret void
}
define void @fma_v32f32(<32 x float>* %a, <32 x float>* %b, <32 x float>* %c) #0 {
; CHECK-LABEL: @fma_v32f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x float>, <32 x float>* %a
%op2 = load <32 x float>, <32 x float>* %b
%op3 = load <32 x float>, <32 x float>* %c
%res = call <32 x float> @llvm.fma.v32f32(<32 x float> %op1, <32 x float> %op2, <32 x float> %op3)
store <32 x float> %res, <32 x float>* %a
ret void
}
define void @fma_v64f32(<64 x float>* %a, <64 x float>* %b, <64 x float>* %c) #0 {
; CHECK-LABEL: @fma_v64f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x float>, <64 x float>* %a
%op2 = load <64 x float>, <64 x float>* %b
%op3 = load <64 x float>, <64 x float>* %c
%res = call <64 x float> @llvm.fma.v64f32(<64 x float> %op1, <64 x float> %op2, <64 x float> %op3)
store <64 x float> %res, <64 x float>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3) #0 {
; CHECK-LABEL: @fma_v1f64
; CHECK: fmadd d0, d0, d1, d2
; CHECK: ret
%res = call <1 x double> @llvm.fma.v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3)
ret <1 x double> %res
}
; Don't use SVE for 128-bit vectors.
define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3) #0 {
; CHECK-LABEL: @fma_v2f64
; CHECK: fmla v2.2d, v1.2d, v0.2d
; CHECK: ret
%res = call <2 x double> @llvm.fma.v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3)
ret <2 x double> %res
}
define void @fma_v4f64(<4 x double>* %a, <4 x double>* %b, <4 x double>* %c) #0 {
; CHECK-LABEL: @fma_v4f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <4 x double>, <4 x double>* %a
%op2 = load <4 x double>, <4 x double>* %b
%op3 = load <4 x double>, <4 x double>* %c
%res = call <4 x double> @llvm.fma.v4f64(<4 x double> %op1, <4 x double> %op2, <4 x double> %op3)
store <4 x double> %res, <4 x double>* %a
ret void
}
define void @fma_v8f64(<8 x double>* %a, <8 x double>* %b, <8 x double>* %c) #0 {
; CHECK-LABEL: @fma_v8f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x double>, <8 x double>* %a
%op2 = load <8 x double>, <8 x double>* %b
%op3 = load <8 x double>, <8 x double>* %c
%res = call <8 x double> @llvm.fma.v8f64(<8 x double> %op1, <8 x double> %op2, <8 x double> %op3)
store <8 x double> %res, <8 x double>* %a
ret void
}
define void @fma_v16f64(<16 x double>* %a, <16 x double>* %b, <16 x double>* %c) #0 {
; CHECK-LABEL: @fma_v16f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x double>, <16 x double>* %a
%op2 = load <16 x double>, <16 x double>* %b
%op3 = load <16 x double>, <16 x double>* %c
%res = call <16 x double> @llvm.fma.v16f64(<16 x double> %op1, <16 x double> %op2, <16 x double> %op3)
store <16 x double> %res, <16 x double>* %a
ret void
}
define void @fma_v32f64(<32 x double>* %a, <32 x double>* %b, <32 x double>* %c) #0 {
; CHECK-LABEL: @fma_v32f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x double>, <32 x double>* %a
%op2 = load <32 x double>, <32 x double>* %b
%op3 = load <32 x double>, <32 x double>* %c
%res = call <32 x double> @llvm.fma.v32f64(<32 x double> %op1, <32 x double> %op2, <32 x double> %op3)
store <32 x double> %res, <32 x double>* %a
ret void
}
;
; FMUL
;
; Don't use SVE for 64-bit vectors.
define <4 x half> @fmul_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
; CHECK-LABEL: @fmul_v4f16
; CHECK: fmul v0.4h, v0.4h, v1.4h
; CHECK: ret
%res = fmul <4 x half> %op1, %op2
ret <4 x half> %res
}
; Don't use SVE for 128-bit vectors.
define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
; CHECK-LABEL: @fmul_v8f16
; CHECK: fmul v0.8h, v0.8h, v1.8h
; CHECK: ret
%res = fmul <8 x half> %op1, %op2
ret <8 x half> %res
}
define void @fmul_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
; CHECK-LABEL: @fmul_v16f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x half>, <16 x half>* %a
%op2 = load <16 x half>, <16 x half>* %b
%res = fmul <16 x half> %op1, %op2
store <16 x half> %res, <16 x half>* %a
ret void
}
define void @fmul_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
; CHECK-LABEL: @fmul_v32f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x half>, <32 x half>* %a
%op2 = load <32 x half>, <32 x half>* %b
%res = fmul <32 x half> %op1, %op2
store <32 x half> %res, <32 x half>* %a
ret void
}
define void @fmul_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
; CHECK-LABEL: @fmul_v64f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x half>, <64 x half>* %a
%op2 = load <64 x half>, <64 x half>* %b
%res = fmul <64 x half> %op1, %op2
store <64 x half> %res, <64 x half>* %a
ret void
}
define void @fmul_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
; CHECK-LABEL: @fmul_v128f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <128 x half>, <128 x half>* %a
%op2 = load <128 x half>, <128 x half>* %b
%res = fmul <128 x half> %op1, %op2
store <128 x half> %res, <128 x half>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <2 x float> @fmul_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
; CHECK-LABEL: @fmul_v2f32
; CHECK: fmul v0.2s, v0.2s, v1.2s
; CHECK: ret
%res = fmul <2 x float> %op1, %op2
ret <2 x float> %res
}
; Don't use SVE for 128-bit vectors.
define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
; CHECK-LABEL: @fmul_v4f32
; CHECK: fmul v0.4s, v0.4s, v1.4s
; CHECK: ret
%res = fmul <4 x float> %op1, %op2
ret <4 x float> %res
}
define void @fmul_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
; CHECK-LABEL: @fmul_v8f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x float>, <8 x float>* %a
%op2 = load <8 x float>, <8 x float>* %b
%res = fmul <8 x float> %op1, %op2
store <8 x float> %res, <8 x float>* %a
ret void
}
define void @fmul_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
; CHECK-LABEL: @fmul_v16f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x float>, <16 x float>* %a
%op2 = load <16 x float>, <16 x float>* %b
%res = fmul <16 x float> %op1, %op2
store <16 x float> %res, <16 x float>* %a
ret void
}
define void @fmul_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
; CHECK-LABEL: @fmul_v32f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x float>, <32 x float>* %a
%op2 = load <32 x float>, <32 x float>* %b
%res = fmul <32 x float> %op1, %op2
store <32 x float> %res, <32 x float>* %a
ret void
}
define void @fmul_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
; CHECK-LABEL: @fmul_v64f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x float>, <64 x float>* %a
%op2 = load <64 x float>, <64 x float>* %b
%res = fmul <64 x float> %op1, %op2
store <64 x float> %res, <64 x float>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <1 x double> @fmul_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
; CHECK-LABEL: @fmul_v1f64
; CHECK: fmul d0, d0, d1
; CHECK: ret
%res = fmul <1 x double> %op1, %op2
ret <1 x double> %res
}
; Don't use SVE for 128-bit vectors.
define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
; CHECK-LABEL: @fmul_v2f64
; CHECK: fmul v0.2d, v0.2d, v1.2d
; CHECK: ret
%res = fmul <2 x double> %op1, %op2
ret <2 x double> %res
}
define void @fmul_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
; CHECK-LABEL: @fmul_v4f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <4 x double>, <4 x double>* %a
%op2 = load <4 x double>, <4 x double>* %b
%res = fmul <4 x double> %op1, %op2
store <4 x double> %res, <4 x double>* %a
ret void
}
define void @fmul_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
; CHECK-LABEL: @fmul_v8f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x double>, <8 x double>* %a
%op2 = load <8 x double>, <8 x double>* %b
%res = fmul <8 x double> %op1, %op2
store <8 x double> %res, <8 x double>* %a
ret void
}
define void @fmul_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
; CHECK-LABEL: @fmul_v16f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x double>, <16 x double>* %a
%op2 = load <16 x double>, <16 x double>* %b
%res = fmul <16 x double> %op1, %op2
store <16 x double> %res, <16 x double>* %a
ret void
}
define void @fmul_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
; CHECK-LABEL: @fmul_v32f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x double>, <32 x double>* %a
%op2 = load <32 x double>, <32 x double>* %b
%res = fmul <32 x double> %op1, %op2
store <32 x double> %res, <32 x double>* %a
ret void
}
;
; FSUB
;
; Don't use SVE for 64-bit vectors.
define <4 x half> @fsub_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
; CHECK-LABEL: @fsub_v4f16
; CHECK: fsub v0.4h, v0.4h, v1.4h
; CHECK: ret
%res = fsub <4 x half> %op1, %op2
ret <4 x half> %res
}
; Don't use SVE for 128-bit vectors.
define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
; CHECK-LABEL: @fsub_v8f16
; CHECK: fsub v0.8h, v0.8h, v1.8h
; CHECK: ret
%res = fsub <8 x half> %op1, %op2
ret <8 x half> %res
}
define void @fsub_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
; CHECK-LABEL: @fsub_v16f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x half>, <16 x half>* %a
%op2 = load <16 x half>, <16 x half>* %b
%res = fsub <16 x half> %op1, %op2
store <16 x half> %res, <16 x half>* %a
ret void
}
define void @fsub_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
; CHECK-LABEL: @fsub_v32f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x half>, <32 x half>* %a
%op2 = load <32 x half>, <32 x half>* %b
%res = fsub <32 x half> %op1, %op2
store <32 x half> %res, <32 x half>* %a
ret void
}
define void @fsub_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
; CHECK-LABEL: @fsub_v64f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x half>, <64 x half>* %a
%op2 = load <64 x half>, <64 x half>* %b
%res = fsub <64 x half> %op1, %op2
store <64 x half> %res, <64 x half>* %a
ret void
}
define void @fsub_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
; CHECK-LABEL: @fsub_v128f16
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
; CHECK: ret
%op1 = load <128 x half>, <128 x half>* %a
%op2 = load <128 x half>, <128 x half>* %b
%res = fsub <128 x half> %op1, %op2
store <128 x half> %res, <128 x half>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <2 x float> @fsub_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
; CHECK-LABEL: @fsub_v2f32
; CHECK: fsub v0.2s, v0.2s, v1.2s
; CHECK: ret
%res = fsub <2 x float> %op1, %op2
ret <2 x float> %res
}
; Don't use SVE for 128-bit vectors.
define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
; CHECK-LABEL: @fsub_v4f32
; CHECK: fsub v0.4s, v0.4s, v1.4s
; CHECK: ret
%res = fsub <4 x float> %op1, %op2
ret <4 x float> %res
}
define void @fsub_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
; CHECK-LABEL: @fsub_v8f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x float>, <8 x float>* %a
%op2 = load <8 x float>, <8 x float>* %b
%res = fsub <8 x float> %op1, %op2
store <8 x float> %res, <8 x float>* %a
ret void
}
define void @fsub_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
; CHECK-LABEL: @fsub_v16f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x float>, <16 x float>* %a
%op2 = load <16 x float>, <16 x float>* %b
%res = fsub <16 x float> %op1, %op2
store <16 x float> %res, <16 x float>* %a
ret void
}
define void @fsub_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
; CHECK-LABEL: @fsub_v32f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x float>, <32 x float>* %a
%op2 = load <32 x float>, <32 x float>* %b
%res = fsub <32 x float> %op1, %op2
store <32 x float> %res, <32 x float>* %a
ret void
}
define void @fsub_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
; CHECK-LABEL: @fsub_v64f32
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
; CHECK: ret
%op1 = load <64 x float>, <64 x float>* %a
%op2 = load <64 x float>, <64 x float>* %b
%res = fsub <64 x float> %op1, %op2
store <64 x float> %res, <64 x float>* %a
ret void
}
; Don't use SVE for 64-bit vectors.
define <1 x double> @fsub_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
; CHECK-LABEL: @fsub_v1f64
; CHECK: fsub d0, d0, d1
; CHECK: ret
%res = fsub <1 x double> %op1, %op2
ret <1 x double> %res
}
; Don't use SVE for 128-bit vectors.
define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
; CHECK-LABEL: @fsub_v2f64
; CHECK: fsub v0.2d, v0.2d, v1.2d
; CHECK: ret
%res = fsub <2 x double> %op1, %op2
ret <2 x double> %res
}
define void @fsub_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
; CHECK-LABEL: @fsub_v4f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <4 x double>, <4 x double>* %a
%op2 = load <4 x double>, <4 x double>* %b
%res = fsub <4 x double> %op1, %op2
store <4 x double> %res, <4 x double>* %a
ret void
}
define void @fsub_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
; CHECK-LABEL: @fsub_v8f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <8 x double>, <8 x double>* %a
%op2 = load <8 x double>, <8 x double>* %b
%res = fsub <8 x double> %op1, %op2
store <8 x double> %res, <8 x double>* %a
ret void
}
define void @fsub_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
; CHECK-LABEL: @fsub_v16f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <16 x double>, <16 x double>* %a
%op2 = load <16 x double>, <16 x double>* %b
%res = fsub <16 x double> %op1, %op2
store <16 x double> %res, <16 x double>* %a
ret void
}
define void @fsub_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
; CHECK-LABEL: @fsub_v32f64
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
; CHECK: ret
%op1 = load <32 x double>, <32 x double>* %a
%op2 = load <32 x double>, <32 x double>* %b
%res = fsub <32 x double> %op1, %op2
store <32 x double> %res, <32 x double>* %a
ret void
}
attributes #0 = { "target-features"="+sve" } attributes #0 = { "target-features"="+sve" }
declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)
declare <32 x half> @llvm.fma.v32f16(<32 x half>, <32 x half>, <32 x half>)
declare <64 x half> @llvm.fma.v64f16(<64 x half>, <64 x half>, <64 x half>)
declare <128 x half> @llvm.fma.v128f16(<128 x half>, <128 x half>, <128 x half>)
declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
declare <16 x float> @llvm.fma.v16f32(<16 x float>, <16 x float>, <16 x float>)
declare <32 x float> @llvm.fma.v32f32(<32 x float>, <32 x float>, <32 x float>)
declare <64 x float> @llvm.fma.v64f32(<64 x float>, <64 x float>, <64 x float>)
declare <1 x double> @llvm.fma.v1f64(<1 x double>, <1 x double>, <1 x double>)
declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
declare <8 x double> @llvm.fma.v8f64(<8 x double>, <8 x double>, <8 x double>)
declare <16 x double> @llvm.fma.v16f64(<16 x double>, <16 x double>, <16 x double>)
declare <32 x double> @llvm.fma.v32f64(<32 x double>, <32 x double>, <32 x double>)