This is an archive of the discontinued LLVM Phabricator instance.

Differential D154049

[DAG][AArch64] Handle vector types when expanding sdiv/udiv into mulh
ClosedPublic

Authored by dmgreen on Jun 29 2023, 1:40 AM.

Details

Reviewers
RKSimon
craig.topper
SjoerdMeijer
samtebbs
Commits
rGf55d96b9a25d: [DAG][AArch64] Handle vector types when expanding sdiv/udiv into mulh
Summary

The aarch64 backend will benefit from expanding 64vector sdiv/udiv into mulh using shift(mul(ext, ext)), as the larger type size is legal and the mul(ext, ext) can efficiently use smull/umull instructions. This extends the existing code in GetMULHS to handle vector types.

Diff Detail

Event Timeline

dmgreen created this revision.Jun 29 2023, 1:40 AM
Herald added a project: Restricted Project. · View Herald TranscriptJun 29 2023, 1:40 AM
Herald added subscribers: hiraditya, kristof.beyls. · View Herald Transcript
dmgreen requested review of this revision.Jun 29 2023, 1:40 AM
Herald added a project: Restricted Project. · View Herald TranscriptJun 29 2023, 1:40 AM
Harbormaster completed remote builds in B242001: Diff 535673.Jun 29 2023, 1:40 AM
samtebbs accepted this revision.Jun 29 2023, 1:55 AM

Looks good to me, nice one.

This revision is now accepted and ready to land.Jun 29 2023, 1:55 AM
dmgreen mentioned this in rG878e498f0575: [AArch64] Expand typesizes of tests for constant srem/urem. NFC.Jul 2 2023, 4:44 AM
This revision was landed with ongoing or failed builds.Jul 2 2023, 7:03 AM
Closed by commit rGf55d96b9a25d: [DAG][AArch64] Handle vector types when expanding sdiv/udiv into mulh (authored by dmgreen). · Explain Why
This revision was automatically updated to reflect the committed changes.
dmgreen added a commit: rGf55d96b9a25d: [DAG][AArch64] Handle vector types when expanding sdiv/udiv into mulh.

Revision Contents

PathSize
llvm/
lib/
CodeGen/
SelectionDAG/
46 lines
test/
CodeGen/
AArch64/
426 lines
299 lines

Diff 536589

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 5,990 Lines • ▼ Show 20 Lines auto GetMULHS = [&](SDValue X, SDValue Y) {
if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
return DAG.getNode(ISD::MULHS, dl, VT, X, Y); return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
SDValue LoHi = SDValue LoHi =
DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
return SDValue(LoHi.getNode(), 1); return SDValue(LoHi.getNode(), 1);
} }
// If type twice as wide legal, widen and use a mul plus a shift. // If type twice as wide legal, widen and use a mul plus a shift.
if (!VT.isVector()) { unsigned Size = VT.getScalarSizeInBits();
unsigned Size = VT.getSizeInBits();
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2); EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2);
if (isOperationLegal(ISD::MUL, WideVT)) { if (VT.isVector())
WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
VT.getVectorElementCount());
if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X); X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X);
Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y); Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y);
Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
DAG.getShiftAmountConstant(EltBits, WideVT, dl)); DAG.getShiftAmountConstant(EltBits, WideVT, dl));
return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
} }
}
return SDValue(); return SDValue();
}; };
SDValue Q = GetMULHS(N0, MagicFactor); SDValue Q = GetMULHS(N0, MagicFactor);
if (!Q) if (!Q)
return SDValue(); return SDValue();
Created.push_back(Q.getNode()); Created.push_back(Q.getNode());
▲ Show 20 Lines • Show All 158 Lines • ▼ Show 20 Lines auto GetMULHU = [&](SDValue X, SDValue Y) {
if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
return DAG.getNode(ISD::MULHU, dl, VT, X, Y); return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
SDValue LoHi = SDValue LoHi =
DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
return SDValue(LoHi.getNode(), 1); return SDValue(LoHi.getNode(), 1);
} }
// If type twice as wide legal, widen and use a mul plus a shift. // If type twice as wide legal, widen and use a mul plus a shift.
if (!VT.isVector()) { unsigned Size = VT.getScalarSizeInBits();
unsigned Size = VT.getSizeInBits();
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2); EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2);
if (isOperationLegal(ISD::MUL, WideVT)) { if (VT.isVector())
WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
VT.getVectorElementCount());
if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X); X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X);
Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y); Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y);
Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
DAG.getShiftAmountConstant(EltBits, WideVT, dl)); DAG.getShiftAmountConstant(EltBits, WideVT, dl));
return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
} }
}
return SDValue(); // No mulhu or equivalent return SDValue(); // No mulhu or equivalent
}; };
// Multiply the numerator (operand 0) by the magic value. // Multiply the numerator (operand 0) by the magic value.
Q = GetMULHU(Q, MagicFactor); Q = GetMULHU(Q, MagicFactor);
if (!Q) if (!Q)
return SDValue(); return SDValue();
▲ Show 20 Lines • Show All 4,510 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/srem-vector-lkk.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) { define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) {
; CHECK-LABEL: fold_srem_vec_1: ; CHECK-LABEL: fold_srem_vec_1:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI0_1
; CHECK-NEXT: smov w8, v0.h[1] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_1]
; CHECK-NEXT: smov w9, v0.h[0] ; CHECK-NEXT: adrp x8, .LCPI0_0
; CHECK-NEXT: mov w10, #63421 // =0xf7bd ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: mov w11, #37253 // =0x9185 ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: movk w10, #31710, lsl #16 ; CHECK-NEXT: adrp x8, .LCPI0_2
; CHECK-NEXT: movk w11, #44150, lsl #16 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: smov w13, v0.h[2] ; CHECK-NEXT: mla v1.4h, v0.4h, v2.4h
; CHECK-NEXT: mov w12, #33437 // =0x829d ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI0_2]
; CHECK-NEXT: smull x10, w8, w10 ; CHECK-NEXT: adrp x8, .LCPI0_3
; CHECK-NEXT: movk w12, #21399, lsl #16 ; CHECK-NEXT: sshl v1.4h, v1.4h, v2.4h
; CHECK-NEXT: smull x11, w9, w11 ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI0_3]
; CHECK-NEXT: lsr x10, x10, #32 ; CHECK-NEXT: usra v1.4h, v1.4h, #15
; CHECK-NEXT: lsr x11, x11, #32 ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: sub w10, w10, w8
; CHECK-NEXT: add w11, w11, w9
; CHECK-NEXT: asr w14, w10, #6
; CHECK-NEXT: asr w15, w11, #6
; CHECK-NEXT: add w10, w14, w10, lsr #31
; CHECK-NEXT: add w11, w15, w11, lsr #31
; CHECK-NEXT: mov w14, #95 // =0x5f
; CHECK-NEXT: mov w15, #-124 // =0xffffff84
; CHECK-NEXT: smull x12, w13, w12
; CHECK-NEXT: msub w9, w11, w14, w9
; CHECK-NEXT: msub w8, w10, w15, w8
; CHECK-NEXT: lsr x10, x12, #63
; CHECK-NEXT: asr x11, x12, #37
; CHECK-NEXT: smov w12, v0.h[3]
; CHECK-NEXT: add w10, w11, w10
; CHECK-NEXT: mov w11, #98 // =0x62
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: mov w9, #63249 // =0xf711
; CHECK-NEXT: movk w9, #48808, lsl #16
; CHECK-NEXT: msub w10, w10, w11, w13
; CHECK-NEXT: smull x9, w12, w9
; CHECK-NEXT: mov v0.h[1], w8
; CHECK-NEXT: lsr x8, x9, #63
; CHECK-NEXT: asr x9, x9, #40
; CHECK-NEXT: add w8, w9, w8
; CHECK-NEXT: mov w9, #-1003 // =0xfffffc15
; CHECK-NEXT: mov v0.h[2], w10
; CHECK-NEXT: msub w8, w8, w9, w12
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003> %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
ret <4 x i16> %1 ret <4 x i16> %1
} }
define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) { define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) {
; CHECK-LABEL: fold_srem_vec_2: ; CHECK-LABEL: fold_srem_vec_2:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: mov w8, #44151 // =0xac77
; CHECK-NEXT: smov w9, v0.h[0] ; CHECK-NEXT: movi v2.4h, #95
; CHECK-NEXT: mov w8, #37253 // =0x9185 ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: movk w8, #44150, lsl #16 ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: smov w10, v0.h[1] ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: smov w14, v0.h[2] ; CHECK-NEXT: add v1.4h, v1.4h, v0.4h
; CHECK-NEXT: mov w12, #95 // =0x5f ; CHECK-NEXT: sshr v1.4h, v1.4h, #6
; CHECK-NEXT: smull x11, w9, w8 ; CHECK-NEXT: usra v1.4h, v1.4h, #15
; CHECK-NEXT: smull x13, w10, w8 ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: add w11, w11, w9
; CHECK-NEXT: lsr x13, x13, #32
; CHECK-NEXT: asr w15, w11, #6
; CHECK-NEXT: add w13, w13, w10
; CHECK-NEXT: add w11, w15, w11, lsr #31
; CHECK-NEXT: smov w15, v0.h[3]
; CHECK-NEXT: asr w16, w13, #6
; CHECK-NEXT: msub w9, w11, w12, w9
; CHECK-NEXT: add w13, w16, w13, lsr #31
; CHECK-NEXT: smull x11, w14, w8
; CHECK-NEXT: msub w10, w13, w12, w10
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: smull x8, w15, w8
; CHECK-NEXT: add w11, w11, w14
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: asr w9, w11, #6
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: add w9, w9, w11, lsr #31
; CHECK-NEXT: add w8, w8, w15
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: asr w10, w8, #6
; CHECK-NEXT: msub w9, w9, w12, w14
; CHECK-NEXT: add w8, w10, w8, lsr #31
; CHECK-NEXT: msub w8, w8, w12, w15
; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if we can combine srem with sdiv. ; Don't fold if we can combine srem with sdiv.
define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) { define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) {
; CHECK-LABEL: combine_srem_sdiv: ; CHECK-LABEL: combine_srem_sdiv:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: mov w8, #44151 // =0xac77
; CHECK-NEXT: smov w9, v0.h[0] ; CHECK-NEXT: movi v2.4h, #95
; CHECK-NEXT: mov w8, #37253 // =0x9185 ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: movk w8, #44150, lsl #16 ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: smov w10, v0.h[1] ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: smov w11, v0.h[2] ; CHECK-NEXT: add v1.4h, v1.4h, v0.4h
; CHECK-NEXT: smov w12, v0.h[3] ; CHECK-NEXT: sshr v1.4h, v1.4h, #6
; CHECK-NEXT: mov w14, #95 // =0x5f ; CHECK-NEXT: usra v1.4h, v1.4h, #15
; CHECK-NEXT: smull x13, w9, w8 ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: smull x15, w10, w8
; CHECK-NEXT: lsr x13, x13, #32
; CHECK-NEXT: smull x16, w11, w8
; CHECK-NEXT: add w13, w13, w9
; CHECK-NEXT: lsr x15, x15, #32
; CHECK-NEXT: asr w17, w13, #6
; CHECK-NEXT: add w15, w15, w10
; CHECK-NEXT: add w13, w17, w13, lsr #31
; CHECK-NEXT: asr w17, w15, #6
; CHECK-NEXT: add w15, w17, w15, lsr #31
; CHECK-NEXT: smull x8, w12, w8
; CHECK-NEXT: msub w9, w13, w14, w9
; CHECK-NEXT: lsr x16, x16, #32
; CHECK-NEXT: add w16, w16, w11
; CHECK-NEXT: msub w10, w15, w14, w10
; CHECK-NEXT: asr w17, w16, #6
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: fmov s1, w13
; CHECK-NEXT: add w16, w17, w16, lsr #31
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: add w8, w8, w12
; CHECK-NEXT: asr w9, w8, #6
; CHECK-NEXT: add w8, w9, w8, lsr #31
; CHECK-NEXT: msub w9, w16, w14, w11
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: mov v1.h[1], w15
; CHECK-NEXT: msub w10, w8, w14, w12
; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: mov v1.h[2], w16
; CHECK-NEXT: mov v0.h[3], w10
; CHECK-NEXT: mov v1.h[3], w8
; CHECK-NEXT: add v0.4h, v0.4h, v1.4h ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
%2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
%3 = add <4 x i16> %1, %2 %3 = add <4 x i16> %1, %2
ret <4 x i16> %3 ret <4 x i16> %3
} }
; Don't fold for divisors that are a power of two. ; Don't fold for divisors that are a power of two.
define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) { define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_srem_power_of_two: ; CHECK-LABEL: dont_fold_srem_power_of_two:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI3_0
; CHECK-NEXT: smov w9, v0.h[1] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
; CHECK-NEXT: smov w10, v0.h[0] ; CHECK-NEXT: adrp x8, .LCPI3_1
; CHECK-NEXT: mov w8, #37253 // =0x9185 ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: smov w12, v0.h[2] ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI3_1]
; CHECK-NEXT: movk w8, #44150, lsl #16 ; CHECK-NEXT: adrp x8, .LCPI3_2
; CHECK-NEXT: negs w11, w9 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: and w9, w9, #0x1f ; CHECK-NEXT: add v1.4h, v1.4h, v0.4h
; CHECK-NEXT: and w11, w11, #0x1f ; CHECK-NEXT: sshl v1.4h, v1.4h, v2.4h
; CHECK-NEXT: csneg w9, w9, w11, mi ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI3_2]
; CHECK-NEXT: negs w11, w10 ; CHECK-NEXT: usra v1.4h, v1.4h, #15
; CHECK-NEXT: and w10, w10, #0x3f ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: and w11, w11, #0x3f
; CHECK-NEXT: csneg w10, w10, w11, mi
; CHECK-NEXT: smov w11, v0.h[3]
; CHECK-NEXT: fmov s0, w10
; CHECK-NEXT: negs w10, w12
; CHECK-NEXT: smull x8, w11, w8
; CHECK-NEXT: and w10, w10, #0x7
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v0.h[1], w9
; CHECK-NEXT: and w9, w12, #0x7
; CHECK-NEXT: add w8, w8, w11
; CHECK-NEXT: csneg w9, w9, w10, mi
; CHECK-NEXT: asr w10, w8, #6
; CHECK-NEXT: add w8, w10, w8, lsr #31
; CHECK-NEXT: mov w10, #95 // =0x5f
; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: msub w8, w8, w10, w11
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95> %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if the divisor is one. ; Don't fold if the divisor is one.
define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) { define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_srem_one: ; CHECK-LABEL: dont_fold_srem_one:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI4_0
; CHECK-NEXT: smov w8, v0.h[1] ; CHECK-NEXT: movi d2, #0x00ffff0000ffff
; CHECK-NEXT: mov w9, #30865 // =0x7891 ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
; CHECK-NEXT: movk w9, #51306, lsl #16 ; CHECK-NEXT: adrp x8, .LCPI4_1
; CHECK-NEXT: smov w10, v0.h[2] ; CHECK-NEXT: and v2.8b, v0.8b, v2.8b
; CHECK-NEXT: mov w11, #17097 // =0x42c9 ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: mov w12, #654 // =0x28e ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI4_1]
; CHECK-NEXT: movk w11, #45590, lsl #16 ; CHECK-NEXT: adrp x8, .LCPI4_2
; CHECK-NEXT: smull x9, w8, w9 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: smull x11, w10, w11 ; CHECK-NEXT: add v1.4h, v1.4h, v2.4h
; CHECK-NEXT: lsr x9, x9, #32 ; CHECK-NEXT: sshl v1.4h, v1.4h, v3.4h
; CHECK-NEXT: add w9, w9, w8 ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI4_2]
; CHECK-NEXT: lsr x11, x11, #32 ; CHECK-NEXT: ushr v2.4h, v1.4h, #15
; CHECK-NEXT: asr w13, w9, #9 ; CHECK-NEXT: mov v2.h[0], wzr
; CHECK-NEXT: add w11, w11, w10 ; CHECK-NEXT: add v1.4h, v1.4h, v2.4h
; CHECK-NEXT: add w9, w13, w9, lsr #31 ; CHECK-NEXT: mls v0.4h, v1.4h, v3.4h
; CHECK-NEXT: mov w13, #23 // =0x17
; CHECK-NEXT: msub w8, w9, w12, w8
; CHECK-NEXT: asr w9, w11, #4
; CHECK-NEXT: smov w12, v0.h[3]
; CHECK-NEXT: add w9, w9, w11, lsr #31
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: mov w11, #47143 // =0xb827
; CHECK-NEXT: movk w11, #24749, lsl #16
; CHECK-NEXT: msub w9, w9, w13, w10
; CHECK-NEXT: smull x10, w12, w11
; CHECK-NEXT: mov v0.h[1], w8
; CHECK-NEXT: lsr x8, x10, #63
; CHECK-NEXT: asr x10, x10, #43
; CHECK-NEXT: add w8, w10, w8
; CHECK-NEXT: mov w10, #5423 // =0x152f
; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: msub w8, w8, w10, w12
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423> %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if the divisor is 2^15. ; Don't fold if the divisor is 2^15.
define <4 x i16> @dont_fold_srem_i16_smax(<4 x i16> %x) { define <4 x i16> @dont_fold_srem_i16_smax(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_srem_i16_smax: ; CHECK-LABEL: dont_fold_srem_i16_smax:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI5_1
; CHECK-NEXT: smov w9, v0.h[2] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI5_1]
; CHECK-NEXT: mov w8, #17097 // =0x42c9 ; CHECK-NEXT: adrp x8, .LCPI5_0
; CHECK-NEXT: movk w8, #45590, lsl #16 ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: smov w10, v0.h[1] ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI5_0]
; CHECK-NEXT: smov w12, v0.h[3] ; CHECK-NEXT: adrp x8, .LCPI5_2
; CHECK-NEXT: mov w11, #23 // =0x17 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: mla v1.4h, v0.4h, v2.4h
; CHECK-NEXT: smull x8, w9, w8 ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI5_2]
; CHECK-NEXT: lsr x8, x8, #32 ; CHECK-NEXT: adrp x8, .LCPI5_3
; CHECK-NEXT: add w8, w8, w9 ; CHECK-NEXT: sshl v1.4h, v1.4h, v2.4h
; CHECK-NEXT: asr w13, w8, #4 ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI5_3]
; CHECK-NEXT: add w8, w13, w8, lsr #31 ; CHECK-NEXT: ushr v2.4h, v1.4h, #15
; CHECK-NEXT: negs w13, w10 ; CHECK-NEXT: mov v2.h[0], wzr
; CHECK-NEXT: and w10, w10, #0x7fff ; CHECK-NEXT: add v1.4h, v1.4h, v2.4h
; CHECK-NEXT: and w13, w13, #0x7fff ; CHECK-NEXT: mls v0.4h, v1.4h, v3.4h
; CHECK-NEXT: csneg w10, w10, w13, mi
; CHECK-NEXT: mov w13, #47143 // =0xb827
; CHECK-NEXT: movk w13, #24749, lsl #16
; CHECK-NEXT: msub w8, w8, w11, w9
; CHECK-NEXT: smull x9, w12, w13
; CHECK-NEXT: mov v1.h[1], w10
; CHECK-NEXT: lsr x10, x9, #63
; CHECK-NEXT: asr x9, x9, #43
; CHECK-NEXT: add w9, w9, w10
; CHECK-NEXT: mov w10, #5423 // =0x152f
; CHECK-NEXT: mov v1.h[2], w8
; CHECK-NEXT: msub w8, w9, w10, w12
; CHECK-NEXT: mov v1.h[3], w8
; CHECK-NEXT: fmov d0, d1
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423> %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold i64 srem. ; Don't fold i64 srem.
define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) { define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) {
; CHECK-LABEL: dont_fold_srem_i64: ; CHECK-LABEL: dont_fold_srem_i64:
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <16 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10> %1 = srem <16 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
ret <16 x i8> %1 ret <16 x i8> %1
} }
define <8 x i8> @fold_srem_v8i8(<8 x i8> %x) { define <8 x i8> @fold_srem_v8i8(<8 x i8> %x) {
; CHECK-LABEL: fold_srem_v8i8: ; CHECK-LABEL: fold_srem_v8i8:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: movi v1.8b, #103
; CHECK-NEXT: smov w10, v0.b[0] ; CHECK-NEXT: movi v2.8b, #10
; CHECK-NEXT: mov w8, #26215 // =0x6667 ; CHECK-NEXT: smull v1.8h, v0.8b, v1.8b
; CHECK-NEXT: movk w8, #26214, lsl #16 ; CHECK-NEXT: shrn v1.8b, v1.8h, #8
; CHECK-NEXT: smov w11, v0.b[1] ; CHECK-NEXT: sshr v1.8b, v1.8b, #2
; CHECK-NEXT: smull x9, w10, w8 ; CHECK-NEXT: usra v1.8b, v1.8b, #7
; CHECK-NEXT: smull x14, w11, w8 ; CHECK-NEXT: mls v0.8b, v1.8b, v2.8b
; CHECK-NEXT: lsr x12, x9, #63
; CHECK-NEXT: asr x13, x9, #34
; CHECK-NEXT: mov w9, #10 // =0xa
; CHECK-NEXT: add w12, w13, w12
; CHECK-NEXT: smov w13, v0.b[2]
; CHECK-NEXT: msub w10, w12, w9, w10
; CHECK-NEXT: lsr x12, x14, #63
; CHECK-NEXT: asr x14, x14, #34
; CHECK-NEXT: add w12, w14, w12
; CHECK-NEXT: smov w14, v0.b[3]
; CHECK-NEXT: smull x15, w13, w8
; CHECK-NEXT: msub w11, w12, w9, w11
; CHECK-NEXT: fmov s1, w10
; CHECK-NEXT: lsr x10, x15, #63
; CHECK-NEXT: asr x12, x15, #34
; CHECK-NEXT: add w10, w12, w10
; CHECK-NEXT: smull x12, w14, w8
; CHECK-NEXT: smov w15, v0.b[4]
; CHECK-NEXT: mov v1.b[1], w11
; CHECK-NEXT: msub w10, w10, w9, w13
; CHECK-NEXT: lsr x11, x12, #63
; CHECK-NEXT: asr x12, x12, #34
; CHECK-NEXT: add w11, w12, w11
; CHECK-NEXT: smov w13, v0.b[5]
; CHECK-NEXT: smull x12, w15, w8
; CHECK-NEXT: mov v1.b[2], w10
; CHECK-NEXT: msub w10, w11, w9, w14
; CHECK-NEXT: lsr x11, x12, #63
; CHECK-NEXT: asr x12, x12, #34
; CHECK-NEXT: add w11, w12, w11
; CHECK-NEXT: smull x12, w13, w8
; CHECK-NEXT: smov w14, v0.b[6]
; CHECK-NEXT: mov v1.b[3], w10
; CHECK-NEXT: msub w10, w11, w9, w15
; CHECK-NEXT: lsr x11, x12, #63
; CHECK-NEXT: asr x12, x12, #34
; CHECK-NEXT: add w11, w12, w11
; CHECK-NEXT: smov w15, v0.b[7]
; CHECK-NEXT: smull x12, w14, w8
; CHECK-NEXT: mov v1.b[4], w10
; CHECK-NEXT: msub w10, w11, w9, w13
; CHECK-NEXT: lsr x11, x12, #63
; CHECK-NEXT: asr x12, x12, #34
; CHECK-NEXT: add w11, w12, w11
; CHECK-NEXT: smull x8, w15, w8
; CHECK-NEXT: mov v1.b[5], w10
; CHECK-NEXT: msub w10, w11, w9, w14
; CHECK-NEXT: lsr x11, x8, #63
; CHECK-NEXT: asr x8, x8, #34
; CHECK-NEXT: add w8, w8, w11
; CHECK-NEXT: mov v1.b[6], w10
; CHECK-NEXT: msub w8, w8, w9, w15
; CHECK-NEXT: mov v1.b[7], w8
; CHECK-NEXT: fmov d0, d1
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <8 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10> %1 = srem <8 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
ret <8 x i8> %1 ret <8 x i8> %1
} }
define <8 x i16> @fold_srem_v8i16(<8 x i16> %x) { define <8 x i16> @fold_srem_v8i16(<8 x i16> %x) {
; CHECK-LABEL: fold_srem_v8i16: ; CHECK-LABEL: fold_srem_v8i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
Show All 9 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <8 x i16> %x, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10> %1 = srem <8 x i16> %x, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
ret <8 x i16> %1 ret <8 x i16> %1
} }
define <4 x i16> @fold_srem_v4i16(<4 x i16> %x) { define <4 x i16> @fold_srem_v4i16(<4 x i16> %x) {
; CHECK-LABEL: fold_srem_v4i16: ; CHECK-LABEL: fold_srem_v4i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: smov w9, v0.h[0]
; CHECK-NEXT: mov w8, #26215 // =0x6667 ; CHECK-NEXT: mov w8, #26215 // =0x6667
; CHECK-NEXT: movk w8, #26214, lsl #16 ; CHECK-NEXT: movi v2.4h, #10
; CHECK-NEXT: smov w10, v0.h[1] ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: mov w13, #10 // =0xa ; CHECK-NEXT: smull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: smull x11, w9, w8 ; CHECK-NEXT: sshr v1.4s, v1.4s, #18
; CHECK-NEXT: smull x14, w10, w8 ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: lsr x12, x11, #63 ; CHECK-NEXT: usra v1.4h, v1.4h, #15
; CHECK-NEXT: asr x11, x11, #34 ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: add w11, w11, w12
; CHECK-NEXT: smov w12, v0.h[2]
; CHECK-NEXT: msub w9, w11, w13, w9
; CHECK-NEXT: lsr x11, x14, #63
; CHECK-NEXT: asr x14, x14, #34
; CHECK-NEXT: add w11, w14, w11
; CHECK-NEXT: smov w14, v0.h[3]
; CHECK-NEXT: smull x15, w12, w8
; CHECK-NEXT: msub w10, w11, w13, w10
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: lsr x9, x15, #63
; CHECK-NEXT: asr x11, x15, #34
; CHECK-NEXT: add w9, w11, w9
; CHECK-NEXT: smull x8, w14, w8
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: msub w9, w9, w13, w12
; CHECK-NEXT: lsr x10, x8, #63
; CHECK-NEXT: asr x8, x8, #34
; CHECK-NEXT: add w8, w8, w10
; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: msub w8, w8, w13, w14
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i16> %x, <i16 10, i16 10, i16 10, i16 10> %1 = srem <4 x i16> %x, <i16 10, i16 10, i16 10, i16 10>
ret <4 x i16> %1 ret <4 x i16> %1
} }
define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) { define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) {
; CHECK-LABEL: fold_srem_v4i32: ; CHECK-LABEL: fold_srem_v4i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
Show All 10 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <4 x i32> %x, <i32 10, i32 10, i32 10, i32 10> %1 = srem <4 x i32> %x, <i32 10, i32 10, i32 10, i32 10>
ret <4 x i32> %1 ret <4 x i32> %1
} }
define <2 x i32> @fold_srem_v2i32(<2 x i32> %x) { define <2 x i32> @fold_srem_v2i32(<2 x i32> %x) {
; CHECK-LABEL: fold_srem_v2i32: ; CHECK-LABEL: fold_srem_v2i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: mov w8, #26215 // =0x6667 ; CHECK-NEXT: mov w8, #26215 // =0x6667
; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: movi v3.2s, #10
; CHECK-NEXT: movk w8, #26214, lsl #16 ; CHECK-NEXT: movk w8, #26214, lsl #16
; CHECK-NEXT: mov w10, v0.s[1] ; CHECK-NEXT: dup v1.2s, w8
; CHECK-NEXT: smull x11, w9, w8 ; CHECK-NEXT: smull v1.2d, v0.2s, v1.2s
; CHECK-NEXT: lsr x12, x11, #63 ; CHECK-NEXT: ushr v2.2d, v1.2d, #63
; CHECK-NEXT: asr x11, x11, #34 ; CHECK-NEXT: sshr v1.2d, v1.2d, #34
; CHECK-NEXT: add w11, w11, w12 ; CHECK-NEXT: xtn v2.2s, v2.2d
; CHECK-NEXT: mov w12, #10 // =0xa ; CHECK-NEXT: xtn v1.2s, v1.2d
; CHECK-NEXT: smull x8, w10, w8 ; CHECK-NEXT: add v1.2s, v1.2s, v2.2s
; CHECK-NEXT: msub w9, w11, w12, w9 ; CHECK-NEXT: mls v0.2s, v1.2s, v3.2s
; CHECK-NEXT: lsr x11, x8, #63
; CHECK-NEXT: asr x8, x8, #34
; CHECK-NEXT: add w8, w8, w11
; CHECK-NEXT: msub w8, w8, w12, w10
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = srem <2 x i32> %x, <i32 10, i32 10> %1 = srem <2 x i32> %x, <i32 10, i32 10>
ret <2 x i32> %1 ret <2 x i32> %1
} }
define <2 x i64> @fold_srem_v2i64(<2 x i64> %x) { define <2 x i64> @fold_srem_v2i64(<2 x i64> %x) {
; CHECK-LABEL: fold_srem_v2i64: ; CHECK-LABEL: fold_srem_v2i64:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
Show All 37 Lines

llvm/test/CodeGen/AArch64/urem-vector-lkk.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) { define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) {
; CHECK-LABEL: fold_urem_vec_1: ; CHECK-LABEL: fold_urem_vec_1:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI0_0
; CHECK-NEXT: umov w8, v0.h[0] ; CHECK-NEXT: adrp x9, .LCPI0_1
; CHECK-NEXT: mov w9, #55879 // =0xda47 ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: movk w9, #689, lsl #16 ; CHECK-NEXT: adrp x8, .LCPI0_2
; CHECK-NEXT: umov w10, v0.h[1] ; CHECK-NEXT: ldr d2, [x9, :lo12:.LCPI0_1]
; CHECK-NEXT: mov w11, #33826 // =0x8422 ; CHECK-NEXT: adrp x9, .LCPI0_4
; CHECK-NEXT: mov w12, #95 // =0x5f ; CHECK-NEXT: ushl v1.4h, v0.4h, v1.4h
; CHECK-NEXT: movk w11, #528, lsl #16 ; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h
; CHECK-NEXT: umov w13, v0.h[2] ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI0_2]
; CHECK-NEXT: umull x9, w8, w9 ; CHECK-NEXT: adrp x8, .LCPI0_3
; CHECK-NEXT: umull x11, w10, w11 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: lsr x9, x9, #32 ; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h
; CHECK-NEXT: lsr x11, x11, #32 ; CHECK-NEXT: umull v2.4s, v3.4h, v2.4h
; CHECK-NEXT: msub w8, w9, w12, w8 ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI0_3]
; CHECK-NEXT: mov w9, #48149 // =0xbc15 ; CHECK-NEXT: shrn v2.4h, v2.4s, #16
; CHECK-NEXT: movk w9, #668, lsl #16 ; CHECK-NEXT: add v1.4h, v2.4h, v1.4h
; CHECK-NEXT: mov w12, #124 // =0x7c ; CHECK-NEXT: ldr d2, [x9, :lo12:.LCPI0_4]
; CHECK-NEXT: umull x9, w13, w9 ; CHECK-NEXT: ushl v1.4h, v1.4h, v3.4h
; CHECK-NEXT: msub w10, w11, w12, w10 ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: umov w11, v0.h[3]
; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: mov w12, #22281 // =0x5709
; CHECK-NEXT: lsr x8, x9, #32
; CHECK-NEXT: mov w9, #98 // =0x62
; CHECK-NEXT: movk w12, #65, lsl #16
; CHECK-NEXT: msub w8, w8, w9, w13
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: umull x9, w11, w12
; CHECK-NEXT: mov w10, #1003 // =0x3eb
; CHECK-NEXT: lsr x9, x9, #32
; CHECK-NEXT: mov v0.h[2], w8
; CHECK-NEXT: msub w8, w9, w10, w11
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003> %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
ret <4 x i16> %1 ret <4 x i16> %1
} }
define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) { define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) {
; CHECK-LABEL: fold_urem_vec_2: ; CHECK-LABEL: fold_urem_vec_2:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: mov w8, #44151 // =0xac77
; CHECK-NEXT: umov w8, v0.h[0] ; CHECK-NEXT: movi v2.4h, #95
; CHECK-NEXT: mov w9, #55879 // =0xda47 ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: movk w9, #689, lsl #16 ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: umov w10, v0.h[1] ; CHECK-NEXT: ushr v1.4s, v1.4s, #22
; CHECK-NEXT: mov w12, #95 // =0x5f ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: umov w13, v0.h[2] ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: umull x11, w8, w9
; CHECK-NEXT: umull x14, w10, w9
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: msub w8, w11, w12, w8
; CHECK-NEXT: lsr x11, x14, #32
; CHECK-NEXT: umull x14, w13, w9
; CHECK-NEXT: msub w10, w11, w12, w10
; CHECK-NEXT: umov w11, v0.h[3]
; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: lsr x8, x14, #32
; CHECK-NEXT: msub w8, w8, w12, w13
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: umull x9, w11, w9
; CHECK-NEXT: lsr x9, x9, #32
; CHECK-NEXT: mov v0.h[2], w8
; CHECK-NEXT: msub w8, w9, w12, w11
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if we can combine urem with udiv. ; Don't fold if we can combine urem with udiv.
define <4 x i16> @combine_urem_udiv(<4 x i16> %x) { define <4 x i16> @combine_urem_udiv(<4 x i16> %x) {
; CHECK-LABEL: combine_urem_udiv: ; CHECK-LABEL: combine_urem_udiv:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: mov w8, #44151 // =0xac77
; CHECK-NEXT: umov w8, v0.h[0] ; CHECK-NEXT: movi v2.4h, #95
; CHECK-NEXT: mov w9, #55879 // =0xda47 ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: movk w9, #689, lsl #16 ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: umov w10, v0.h[1] ; CHECK-NEXT: ushr v1.4s, v1.4s, #22
; CHECK-NEXT: mov w12, #95 // =0x5f ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: umov w14, v0.h[2] ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: umov w15, v0.h[3]
; CHECK-NEXT: umull x11, w8, w9
; CHECK-NEXT: umull x13, w10, w9
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: lsr x13, x13, #32
; CHECK-NEXT: msub w8, w11, w12, w8
; CHECK-NEXT: msub w10, w13, w12, w10
; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: umull x8, w14, w9
; CHECK-NEXT: umull x9, w15, w9
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: lsr x9, x9, #32
; CHECK-NEXT: msub w10, w8, w12, w14
; CHECK-NEXT: mov v1.h[1], w13
; CHECK-NEXT: msub w11, w9, w12, w15
; CHECK-NEXT: mov v0.h[2], w10
; CHECK-NEXT: mov v1.h[2], w8
; CHECK-NEXT: mov v0.h[3], w11
; CHECK-NEXT: mov v1.h[3], w9
; CHECK-NEXT: add v0.4h, v0.4h, v1.4h ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
%2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95> %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
%3 = add <4 x i16> %1, %2 %3 = add <4 x i16> %1, %2
ret <4 x i16> %3 ret <4 x i16> %3
} }
; Don't fold for divisors that are a power of two. ; Don't fold for divisors that are a power of two.
define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) { define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_urem_power_of_two: ; CHECK-LABEL: dont_fold_urem_power_of_two:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI3_0
; CHECK-NEXT: umov w9, v0.h[0] ; CHECK-NEXT: adrp x9, .LCPI3_2
; CHECK-NEXT: umov w11, v0.h[1] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
; CHECK-NEXT: umov w10, v0.h[3] ; CHECK-NEXT: adrp x8, .LCPI3_1
; CHECK-NEXT: mov w8, #55879 // =0xda47 ; CHECK-NEXT: ldr d3, [x9, :lo12:.LCPI3_2]
; CHECK-NEXT: movk w8, #689, lsl #16 ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: and w9, w9, #0x3f ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI3_1]
; CHECK-NEXT: umull x8, w10, w8 ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: fmov s1, w9 ; CHECK-NEXT: ushl v1.4h, v1.4h, v2.4h
; CHECK-NEXT: and w9, w11, #0x1f ; CHECK-NEXT: mls v0.4h, v1.4h, v3.4h
; CHECK-NEXT: umov w11, v0.h[2]
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v1.h[1], w9
; CHECK-NEXT: mov w9, #95 // =0x5f
; CHECK-NEXT: and w11, w11, #0x7
; CHECK-NEXT: msub w8, w8, w9, w10
; CHECK-NEXT: mov v1.h[2], w11
; CHECK-NEXT: mov v1.h[3], w8
; CHECK-NEXT: fmov d0, d1
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95> %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if the divisor is one. ; Don't fold if the divisor is one.
define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) { define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_urem_one: ; CHECK-LABEL: dont_fold_urem_one:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI4_0
; CHECK-NEXT: umov w8, v0.h[1] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
; CHECK-NEXT: mov w9, #13629 // =0x353d ; CHECK-NEXT: adrp x8, .LCPI4_1
; CHECK-NEXT: movk w9, #100, lsl #16 ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: umov w10, v0.h[2] ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI4_1]
; CHECK-NEXT: mov w11, #25645 // =0x642d ; CHECK-NEXT: adrp x8, .LCPI4_2
; CHECK-NEXT: mov w12, #654 // =0x28e ; CHECK-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-NEXT: movk w11, #2849, lsl #16 ; CHECK-NEXT: ldr d4, [x8, :lo12:.LCPI4_2]
; CHECK-NEXT: mov w13, #5560 // =0x15b8 ; CHECK-NEXT: adrp x8, .LCPI4_3
; CHECK-NEXT: umull x9, w8, w9 ; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h
; CHECK-NEXT: movk w13, #12, lsl #16 ; CHECK-NEXT: umull v2.4s, v3.4h, v2.4h
; CHECK-NEXT: umull x11, w10, w11 ; CHECK-NEXT: movi d3, #0xffffffffffff0000
; CHECK-NEXT: lsr x9, x9, #32 ; CHECK-NEXT: shrn v2.4h, v2.4s, #16
; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: add v1.4h, v2.4h, v1.4h
; CHECK-NEXT: lsr x11, x11, #32 ; CHECK-NEXT: movi d2, #0x0000000000ffff
; CHECK-NEXT: msub w8, w9, w12, w8 ; CHECK-NEXT: ushl v1.4h, v1.4h, v4.4h
; CHECK-NEXT: umov w9, v0.h[3] ; CHECK-NEXT: ldr d4, [x8, :lo12:.LCPI4_3]
; CHECK-NEXT: mov w12, #23 // =0x17 ; CHECK-NEXT: and v1.8b, v1.8b, v3.8b
; CHECK-NEXT: msub w10, w11, w12, w10 ; CHECK-NEXT: and v2.8b, v0.8b, v2.8b
; CHECK-NEXT: mov w11, #5423 // =0x152f ; CHECK-NEXT: orr v1.8b, v2.8b, v1.8b
; CHECK-NEXT: mov v1.h[1], w8 ; CHECK-NEXT: mls v0.4h, v1.4h, v4.4h
; CHECK-NEXT: umull x8, w9, w13
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v1.h[2], w10
; CHECK-NEXT: msub w8, w8, w11, w9
; CHECK-NEXT: mov v1.h[3], w8
; CHECK-NEXT: fmov d0, d1
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423> %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
ret <4 x i16> %1 ret <4 x i16> %1
} }
; Don't fold if the divisor is 2^16. ; Don't fold if the divisor is 2^16.
define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) { define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) {
; CHECK-LABEL: dont_fold_urem_i16_smax: ; CHECK-LABEL: dont_fold_urem_i16_smax:
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <16 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10> %1 = urem <16 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
ret <16 x i8> %1 ret <16 x i8> %1
} }
define <8 x i8> @fold_urem_v8i8(<8 x i8> %x) { define <8 x i8> @fold_urem_v8i8(<8 x i8> %x) {
; CHECK-LABEL: fold_urem_v8i8: ; CHECK-LABEL: fold_urem_v8i8:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: movi v1.8b, #205
; CHECK-NEXT: umov w8, v0.b[0] ; CHECK-NEXT: movi v2.8b, #10
; CHECK-NEXT: mov w9, #39322 // =0x999a ; CHECK-NEXT: umull v1.8h, v0.8b, v1.8b
; CHECK-NEXT: movk w9, #6553, lsl #16 ; CHECK-NEXT: shrn v1.8b, v1.8h, #8
; CHECK-NEXT: umov w10, v0.b[1] ; CHECK-NEXT: ushr v1.8b, v1.8b, #3
; CHECK-NEXT: mov w12, #10 // =0xa ; CHECK-NEXT: mls v0.8b, v1.8b, v2.8b
; CHECK-NEXT: umov w13, v0.b[2]
; CHECK-NEXT: umull x11, w8, w9
; CHECK-NEXT: umull x14, w10, w9
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: umull x15, w13, w9
; CHECK-NEXT: lsr x14, x14, #32
; CHECK-NEXT: msub w8, w11, w12, w8
; CHECK-NEXT: umov w11, v0.b[3]
; CHECK-NEXT: msub w10, w14, w12, w10
; CHECK-NEXT: lsr x14, x15, #32
; CHECK-NEXT: msub w13, w14, w12, w13
; CHECK-NEXT: umov w14, v0.b[4]
; CHECK-NEXT: fmov s1, w8
; CHECK-NEXT: umull x8, w11, w9
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v1.b[1], w10
; CHECK-NEXT: umull x10, w14, w9
; CHECK-NEXT: msub w8, w8, w12, w11
; CHECK-NEXT: umov w11, v0.b[5]
; CHECK-NEXT: lsr x10, x10, #32
; CHECK-NEXT: mov v1.b[2], w13
; CHECK-NEXT: msub w10, w10, w12, w14
; CHECK-NEXT: umov w13, v0.b[6]
; CHECK-NEXT: mov v1.b[3], w8
; CHECK-NEXT: umull x8, w11, w9
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v1.b[4], w10
; CHECK-NEXT: umull x10, w13, w9
; CHECK-NEXT: msub w8, w8, w12, w11
; CHECK-NEXT: umov w11, v0.b[7]
; CHECK-NEXT: lsr x10, x10, #32
; CHECK-NEXT: msub w10, w10, w12, w13
; CHECK-NEXT: mov v1.b[5], w8
; CHECK-NEXT: umull x8, w11, w9
; CHECK-NEXT: lsr x8, x8, #32
; CHECK-NEXT: mov v1.b[6], w10
; CHECK-NEXT: msub w8, w8, w12, w11
; CHECK-NEXT: mov v1.b[7], w8
; CHECK-NEXT: fmov d0, d1
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <8 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10> %1 = urem <8 x i8> %x, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
ret <8 x i8> %1 ret <8 x i8> %1
} }
define <8 x i16> @fold_urem_v8i16(<8 x i16> %x) { define <8 x i16> @fold_urem_v8i16(<8 x i16> %x) {
; CHECK-LABEL: fold_urem_v8i16: ; CHECK-LABEL: fold_urem_v8i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w8, #52429 // =0xcccd ; CHECK-NEXT: mov w8, #52429 // =0xcccd
; CHECK-NEXT: dup v1.8h, w8 ; CHECK-NEXT: dup v1.8h, w8
; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h ; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: uzp2 v1.8h, v1.8h, v2.8h ; CHECK-NEXT: uzp2 v1.8h, v1.8h, v2.8h
; CHECK-NEXT: movi v2.8h, #10 ; CHECK-NEXT: movi v2.8h, #10
; CHECK-NEXT: ushr v1.8h, v1.8h, #3 ; CHECK-NEXT: ushr v1.8h, v1.8h, #3
; CHECK-NEXT: mls v0.8h, v1.8h, v2.8h ; CHECK-NEXT: mls v0.8h, v1.8h, v2.8h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <8 x i16> %x, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10> %1 = urem <8 x i16> %x, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
ret <8 x i16> %1 ret <8 x i16> %1
} }
define <4 x i16> @fold_urem_v4i16(<4 x i16> %x) { define <4 x i16> @fold_urem_v4i16(<4 x i16> %x) {
; CHECK-LABEL: fold_urem_v4i16: ; CHECK-LABEL: fold_urem_v4i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: mov w8, #52429 // =0xcccd
; CHECK-NEXT: umov w8, v0.h[0] ; CHECK-NEXT: movi v2.4h, #10
; CHECK-NEXT: mov w9, #39322 // =0x999a ; CHECK-NEXT: dup v1.4h, w8
; CHECK-NEXT: movk w9, #6553, lsl #16 ; CHECK-NEXT: umull v1.4s, v0.4h, v1.4h
; CHECK-NEXT: umov w10, v0.h[1] ; CHECK-NEXT: ushr v1.4s, v1.4s, #19
; CHECK-NEXT: mov w12, #10 // =0xa ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: umov w13, v0.h[2] ; CHECK-NEXT: mls v0.4h, v1.4h, v2.4h
; CHECK-NEXT: umull x11, w8, w9
; CHECK-NEXT: umull x14, w10, w9
; CHECK-NEXT: lsr x11, x11, #32
; CHECK-NEXT: msub w8, w11, w12, w8
; CHECK-NEXT: lsr x11, x14, #32
; CHECK-NEXT: umull x14, w13, w9
; CHECK-NEXT: msub w10, w11, w12, w10
; CHECK-NEXT: umov w11, v0.h[3]
; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: lsr x8, x14, #32
; CHECK-NEXT: msub w8, w8, w12, w13
; CHECK-NEXT: mov v0.h[1], w10
; CHECK-NEXT: umull x9, w11, w9
; CHECK-NEXT: lsr x9, x9, #32
; CHECK-NEXT: mov v0.h[2], w8
; CHECK-NEXT: msub w8, w9, w12, w11
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i16> %x, <i16 10, i16 10, i16 10, i16 10> %1 = urem <4 x i16> %x, <i16 10, i16 10, i16 10, i16 10>
ret <4 x i16> %1 ret <4 x i16> %1
} }
define <4 x i32> @fold_urem_v4i32(<4 x i32> %x) { define <4 x i32> @fold_urem_v4i32(<4 x i32> %x) {
; CHECK-LABEL: fold_urem_v4i32: ; CHECK-LABEL: fold_urem_v4i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
Show All 9 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <4 x i32> %x, <i32 10, i32 10, i32 10, i32 10> %1 = urem <4 x i32> %x, <i32 10, i32 10, i32 10, i32 10>
ret <4 x i32> %1 ret <4 x i32> %1
} }
define <2 x i32> @fold_urem_v2i32(<2 x i32> %x) { define <2 x i32> @fold_urem_v2i32(<2 x i32> %x) {
; CHECK-LABEL: fold_urem_v2i32: ; CHECK-LABEL: fold_urem_v2i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: mov w8, #52429 // =0xcccd ; CHECK-NEXT: mov w8, #52429 // =0xcccd
; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: movi v2.2s, #10
; CHECK-NEXT: movk w8, #52428, lsl #16 ; CHECK-NEXT: movk w8, #52428, lsl #16
; CHECK-NEXT: mov w10, v0.s[1] ; CHECK-NEXT: dup v1.2s, w8
; CHECK-NEXT: mov w12, #10 // =0xa ; CHECK-NEXT: umull v1.2d, v0.2s, v1.2s
; CHECK-NEXT: umull x11, w9, w8 ; CHECK-NEXT: ushr v1.2d, v1.2d, #35
; CHECK-NEXT: lsr x11, x11, #35 ; CHECK-NEXT: xtn v1.2s, v1.2d
; CHECK-NEXT: umull x8, w10, w8 ; CHECK-NEXT: mls v0.2s, v1.2s, v2.2s
; CHECK-NEXT: msub w9, w11, w12, w9
; CHECK-NEXT: lsr x8, x8, #35
; CHECK-NEXT: msub w8, w8, w12, w10
; CHECK-NEXT: fmov s0, w9
; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%1 = urem <2 x i32> %x, <i32 10, i32 10> %1 = urem <2 x i32> %x, <i32 10, i32 10>
ret <2 x i32> %1 ret <2 x i32> %1
} }
define <2 x i64> @fold_urem_v2i64(<2 x i64> %x) { define <2 x i64> @fold_urem_v2i64(<2 x i64> %x) {
; CHECK-LABEL: fold_urem_v2i64: ; CHECK-LABEL: fold_urem_v2i64:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
Show All 34 Lines