This is an archive of the discontinued LLVM Phabricator instance.

Differential D140287

[AArch64] Combine to UMULL if top bits are known zero
ClosedPublic

Authored by dmgreen on Dec 19 2022, 2:58 AM.

Details

Reviewers
SjoerdMeijer
bipmis
samtebbs
fhahn
Commits
rG3c0c24e0c1d3: [AArch64] Combine to UMULL if top bits are known zero
Summary

Given mul(zext(a), b), we can convert to a umull so long as we know that the top bits of b are zero. This uses MaskedValueIsZero to detect that case for NEON UMULL patterns.

Diff Detail

Event Timeline

dmgreen created this revision.Dec 19 2022, 2:58 AM
Herald added a project: Restricted Project. · View Herald TranscriptDec 19 2022, 2:58 AM
Herald added subscribers: hiraditya, kristof.beyls. · View Herald Transcript
dmgreen requested review of this revision.Dec 19 2022, 2:58 AM
Herald added a project: Restricted Project. · View Herald TranscriptDec 19 2022, 2:58 AM
Harbormaster completed remote builds in B203859: Diff 483888.Dec 19 2022, 2:59 AM
samtebbs accepted this revision.Dec 19 2022, 3:21 AM

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
4571

"extend" or "extends"?

This revision is now accepted and ready to land.Dec 19 2022, 3:21 AM
This revision was landed with ongoing or failed builds.Dec 20 2022, 5:50 AM
Closed by commit rG3c0c24e0c1d3: [AArch64] Combine to UMULL if top bits are known zero (authored by dmgreen). · Explain Why
This revision was automatically updated to reflect the committed changes.
dmgreen added a commit: rG3c0c24e0c1d3: [AArch64] Combine to UMULL if top bits are known zero.
dmgreen mentioned this in D154936: [AArch64] Improve cost of umull from known bits.Jul 11 2023, 2:56 AM
dmgreen mentioned this in rG1712ae670915: [AArch64] Improve cost of umull from known bits.Jul 12 2023, 5:13 AM

Revision Contents

PathSize
llvm/
lib/
Target/
AArch64/
34 lines
test/
CodeGen/
AArch64/
20 lines
92 lines

Diff 484240

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 4,561 Lines • ▼ Show 20 Lines if (DAG.SignBitIsZero(ZextOperand)) {
DAG.getSExtOrTrunc(ZextOperand, DL, N0->getValueType(0)).getNode(); DAG.getSExtOrTrunc(ZextOperand, DL, N0->getValueType(0)).getNode();
if (IsN0ZExt) if (IsN0ZExt)
N0 = NewSext; N0 = NewSext;
else else
N1 = NewSext; N1 = NewSext;
return AArch64ISD::SMULL; return AArch64ISD::SMULL;
} }
} }
// Select UMULL if we can replace the other operand with an extend.
samtebbsUnsubmitted
Not Done
ReplyInline Actions

"extend" or "extends"?

samtebbs: "extend" or "extends"?
if (IsN0ZExt || IsN1ZExt) {
EVT VT = N0->getValueType(0);
APInt Mask = APInt::getHighBitsSet(VT.getScalarSizeInBits(),
VT.getScalarSizeInBits() / 2);
if (DAG.MaskedValueIsZero(SDValue(IsN0ZExt ? N1 : N0, 0), Mask)) {
EVT HalfVT;
switch (VT.getSimpleVT().SimpleTy) {
case MVT::v2i64:
HalfVT = MVT::v2i32;
break;
case MVT::v4i32:
HalfVT = MVT::v4i16;
break;
case MVT::v8i16:
HalfVT = MVT::v8i8;
break;
default:
return 0;
}
// Truncate and then extend the result.
SDValue NewExt = DAG.getNode(ISD::TRUNCATE, DL, HalfVT,
SDValue(IsN0ZExt ? N1 : N0, 0));
NewExt = DAG.getZExtOrTrunc(NewExt, DL, VT);
if (IsN0ZExt)
N1 = NewExt.getNode();
else
N0 = NewExt.getNode();
return AArch64ISD::UMULL;
}
}
if (!IsN1SExt && !IsN1ZExt) if (!IsN1SExt && !IsN1ZExt)
return 0; return 0;
// Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
// into (s/zext A * s/zext C) + (s/zext B * s/zext C) // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
if (IsN1SExt && isAddSubSExt(N0, DAG)) { if (IsN1SExt && isAddSubSExt(N0, DAG)) {
IsMLA = true; IsMLA = true;
return AArch64ISD::SMULL; return AArch64ISD::SMULL;
} }
if (IsN1ZExt && isAddSubZExt(N0, DAG)) { if (IsN1ZExt && isAddSubZExt(N0, DAG)) {
IsMLA = true; IsMLA = true;
▲ Show 20 Lines • Show All 19,173 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll

Show First 20 Lines • Show All 110 Lines • ▼ Show 20 Linesentry:
%broadcast.splat = shufflevector <2 x i16> %broadcast.splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer %broadcast.splat = shufflevector <2 x i16> %broadcast.splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
%out = mul nsw <2 x i16> %broadcast.splat, %ext.b %out = mul nsw <2 x i16> %broadcast.splat, %ext.b
ret <2 x i16> %out ret <2 x i16> %out
} }
define <2 x i64> @dupzext_v2i16_v2i64(i16 %src, <2 x i16> %b) { define <2 x i64> @dupzext_v2i16_v2i64(i16 %src, <2 x i16> %b) {
; CHECK-LABEL: dupzext_v2i16_v2i64: ; CHECK-LABEL: dupzext_v2i16_v2i64:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi d1, #0x00ffff0000ffff
; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
; CHECK-NEXT: and x8, x0, #0xffff ; CHECK-NEXT: and x8, x0, #0xffff
; CHECK-NEXT: movi d1, #0x00ffff0000ffff
; CHECK-NEXT: dup v2.2d, x8
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: xtn v2.2s, v2.2d
; CHECK-NEXT: fmov x9, d0 ; CHECK-NEXT: umull v0.2d, v2.2s, v0.2s
; CHECK-NEXT: mov x10, v0.d[1]
; CHECK-NEXT: mul x9, x8, x9
; CHECK-NEXT: mul x8, x8, x10
; CHECK-NEXT: fmov d0, x9
; CHECK-NEXT: mov v0.d[1], x8
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in = zext i16 %src to i64 %in = zext i16 %src to i64
%ext.b = zext <2 x i16> %b to <2 x i64> %ext.b = zext <2 x i16> %b to <2 x i64>
%broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0 %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
%broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
%out = mul nuw <2 x i64> %broadcast.splat, %ext.b %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
ret <2 x i64> %out ret <2 x i64> %out
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines; CHECK-NEXT: ret
%tmp16 = sext <16 x i1> %tmp15 to <16 x i8> %tmp16 = sext <16 x i1> %tmp15 to <16 x i8>
store <16 x i8> %tmp16, ptr %p, align 1 store <16 x i8> %tmp16, ptr %p, align 1
ret void ret void
} }
define <8 x i16> @typei1_v8i1_v8i16(i1 %src, <8 x i1> %b) { define <8 x i16> @typei1_v8i1_v8i16(i1 %src, <8 x i1> %b) {
; CHECK-LABEL: typei1_v8i1_v8i16: ; CHECK-LABEL: typei1_v8i1_v8i16:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v1.8b, #1
; CHECK-NEXT: and w8, w0, #0x1 ; CHECK-NEXT: and w8, w0, #0x1
; CHECK-NEXT: movi v1.8b, #1
; CHECK-NEXT: dup v2.8h, w8
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: dup v1.8h, w8 ; CHECK-NEXT: xtn v2.8b, v2.8h
; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: umull v0.8h, v2.8b, v0.8b
; CHECK-NEXT: mul v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in = zext i1 %src to i16 %in = zext i1 %src to i16
%ext.b = zext <8 x i1> %b to <8 x i16> %ext.b = zext <8 x i1> %b to <8 x i16>
%broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0 %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
%broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
%out = mul nsw <8 x i16> %broadcast.splat, %ext.b %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
ret <8 x i16> %out ret <8 x i16> %out
▲ Show 20 Lines • Show All 104 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/aarch64-smull.ll

Show First 20 Lines • Show All 926 Lines • ▼ Show 20 Lines; CHECK-NEXT: ret
%and = and <4 x i64> %mul, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295> %and = and <4 x i64> %mul, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
ret <4 x i64> %and ret <4 x i64> %and
} }
define <8 x i16> @umull_and_v8i16(<8 x i8> %src1, <8 x i16> %src2) { define <8 x i16> @umull_and_v8i16(<8 x i8> %src1, <8 x i16> %src2) {
; CHECK-LABEL: umull_and_v8i16: ; CHECK-LABEL: umull_and_v8i16:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: bic v1.8h, #255, lsl #8
; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h ; CHECK-NEXT: xtn v1.8b, v1.8h
; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <8 x i8> %src1 to <8 x i16> %in1 = zext <8 x i8> %src1 to <8 x i16>
%in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255> %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
%out = mul nsw <8 x i16> %in1, %in2 %out = mul nsw <8 x i16> %in1, %in2
ret <8 x i16> %out ret <8 x i16> %out
} }
define <8 x i16> @umull_and_v8i16_c(<8 x i8> %src1, <8 x i16> %src2) { define <8 x i16> @umull_and_v8i16_c(<8 x i8> %src1, <8 x i16> %src2) {
; CHECK-LABEL: umull_and_v8i16_c: ; CHECK-LABEL: umull_and_v8i16_c:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: bic v1.8h, #255, lsl #8
; CHECK-NEXT: mul v0.8h, v1.8h, v0.8h ; CHECK-NEXT: xtn v1.8b, v1.8h
; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <8 x i8> %src1 to <8 x i16> %in1 = zext <8 x i8> %src1 to <8 x i16>
%in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255> %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
%out = mul nsw <8 x i16> %in2, %in1 %out = mul nsw <8 x i16> %in2, %in1
ret <8 x i16> %out ret <8 x i16> %out
} }
Show All 24 Linesentry:
ret <8 x i16> %out ret <8 x i16> %out
} }
define <8 x i16> @umull_smaller_v8i16(<8 x i4> %src1, <8 x i16> %src2) { define <8 x i16> @umull_smaller_v8i16(<8 x i4> %src1, <8 x i16> %src2) {
; CHECK-LABEL: umull_smaller_v8i16: ; CHECK-LABEL: umull_smaller_v8i16:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.8b, #15 ; CHECK-NEXT: movi v2.8b, #15
; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: bic v1.8h, #255, lsl #8
; CHECK-NEXT: xtn v1.8b, v1.8h
; CHECK-NEXT: and v0.8b, v0.8b, v2.8b ; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <8 x i4> %src1 to <8 x i16> %in1 = zext <8 x i4> %src1 to <8 x i16>
%in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255> %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
%out = mul nsw <8 x i16> %in1, %in2 %out = mul nsw <8 x i16> %in1, %in2
ret <8 x i16> %out ret <8 x i16> %out
} }
define <4 x i32> @umull_and_v4i32(<4 x i16> %src1, <4 x i32> %src2) { define <4 x i32> @umull_and_v4i32(<4 x i16> %src1, <4 x i32> %src2) {
; CHECK-LABEL: umull_and_v4i32: ; CHECK-LABEL: umull_and_v4i32:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.2d, #0x0000ff000000ff ; CHECK-NEXT: movi v2.2d, #0x0000ff000000ff
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <4 x i16> %src1 to <4 x i32> %in1 = zext <4 x i16> %src1 to <4 x i32>
%in2 = and <4 x i32> %src2, <i32 255, i32 255, i32 255, i32 255> %in2 = and <4 x i32> %src2, <i32 255, i32 255, i32 255, i32 255>
%out = mul nsw <4 x i32> %in1, %in2 %out = mul nsw <4 x i32> %in1, %in2
ret <4 x i32> %out ret <4 x i32> %out
} }
define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) { define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) {
; CHECK-LABEL: umull_and_v8i32: ; CHECK-LABEL: umull_and_v8i32:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v3.2d, #0x0000ff000000ff ; CHECK-NEXT: movi v3.2d, #0x0000ff000000ff
; CHECK-NEXT: ushll v4.4s, v0.4h, #0 ; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 ; CHECK-NEXT: and v2.16b, v2.16b, v3.16b
; CHECK-NEXT: and v5.16b, v1.16b, v3.16b ; CHECK-NEXT: and v1.16b, v1.16b, v3.16b
; CHECK-NEXT: and v1.16b, v2.16b, v3.16b ; CHECK-NEXT: xtn v1.4h, v1.4s
; CHECK-NEXT: mul v1.4s, v0.4s, v1.4s ; CHECK-NEXT: xtn v2.4h, v2.4s
; CHECK-NEXT: mul v0.4s, v4.4s, v5.4s ; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
; CHECK-NEXT: umull v1.4s, v4.4h, v2.4h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <8 x i16> %src1 to <8 x i32> %in1 = zext <8 x i16> %src1 to <8 x i32>
%in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255> %in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
%out = mul nsw <8 x i32> %in1, %in2 %out = mul nsw <8 x i32> %in1, %in2
ret <8 x i32> %out ret <8 x i32> %out
} }
define <8 x i32> @umull_and_v8i32_dup(<8 x i16> %src1, i32 %src2) { define <8 x i32> @umull_and_v8i32_dup(<8 x i16> %src1, i32 %src2) {
; CHECK-LABEL: umull_and_v8i32_dup: ; CHECK-LABEL: umull_and_v8i32_dup:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: and w8, w0, #0xff ; CHECK-NEXT: and w8, w0, #0xff
; CHECK-NEXT: ushll v1.4s, v0.4h, #0 ; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
; CHECK-NEXT: dup v2.4s, w8 ; CHECK-NEXT: dup v2.4s, w8
; CHECK-NEXT: mul v0.4s, v1.4s, v2.4s ; CHECK-NEXT: xtn v2.4h, v2.4s
; CHECK-NEXT: mul v1.4s, v3.4s, v2.4s ; CHECK-NEXT: umull v0.4s, v0.4h, v2.4h
; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <8 x i16> %src1 to <8 x i32> %in1 = zext <8 x i16> %src1 to <8 x i32>
%in2 = and i32 %src2, 255 %in2 = and i32 %src2, 255
%broadcast.splatinsert = insertelement <8 x i32> undef, i32 %in2, i64 0 %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %in2, i64 0
%broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
%out = mul nsw <8 x i32> %in1, %broadcast.splat %out = mul nsw <8 x i32> %in1, %broadcast.splat
ret <8 x i32> %out ret <8 x i32> %out
} }
define <2 x i64> @umull_and_v2i64(<2 x i32> %src1, <2 x i64> %src2) { define <2 x i64> @umull_and_v2i64(<2 x i32> %src1, <2 x i64> %src2) {
; CHECK-LABEL: umull_and_v2i64: ; CHECK-LABEL: umull_and_v2i64:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.2d, #0x000000000000ff ; CHECK-NEXT: movi v2.2d, #0x000000000000ff
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: fmov x10, d0
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: fmov x9, d1 ; CHECK-NEXT: xtn v1.2s, v1.2d
; CHECK-NEXT: mov x8, v1.d[1] ; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
; CHECK-NEXT: mov x11, v0.d[1]
; CHECK-NEXT: mul x9, x10, x9
; CHECK-NEXT: mul x8, x11, x8
; CHECK-NEXT: fmov d0, x9
; CHECK-NEXT: mov v0.d[1], x8
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <2 x i32> %src1 to <2 x i64> %in1 = zext <2 x i32> %src1 to <2 x i64>
%in2 = and <2 x i64> %src2, <i64 255, i64 255> %in2 = and <2 x i64> %src2, <i64 255, i64 255>
%out = mul nsw <2 x i64> %in1, %in2 %out = mul nsw <2 x i64> %in1, %in2
ret <2 x i64> %out ret <2 x i64> %out
} }
define <4 x i64> @umull_and_v4i64(<4 x i32> %src1, <4 x i64> %src2) { define <4 x i64> @umull_and_v4i64(<4 x i32> %src1, <4 x i64> %src2) {
; CHECK-LABEL: umull_and_v4i64: ; CHECK-LABEL: umull_and_v4i64:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v3.2d, #0x000000000000ff ; CHECK-NEXT: movi v3.2d, #0x000000000000ff
; CHECK-NEXT: ushll v4.2d, v0.2s, #0 ; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0
; CHECK-NEXT: fmov x14, d4
; CHECK-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-NEXT: and v2.16b, v2.16b, v3.16b
; CHECK-NEXT: fmov x11, d0 ; CHECK-NEXT: and v1.16b, v1.16b, v3.16b
; CHECK-NEXT: mov x9, v0.d[1] ; CHECK-NEXT: xtn v1.2s, v1.2d
; CHECK-NEXT: and v0.16b, v1.16b, v3.16b ; CHECK-NEXT: xtn v2.2s, v2.2d
; CHECK-NEXT: fmov x10, d2 ; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
; CHECK-NEXT: fmov x13, d0 ; CHECK-NEXT: umull v1.2d, v4.2s, v2.2s
; CHECK-NEXT: mov x8, v2.d[1]
; CHECK-NEXT: mov x12, v0.d[1]
; CHECK-NEXT: mul x10, x11, x10
; CHECK-NEXT: mov x15, v4.d[1]
; CHECK-NEXT: mul x11, x14, x13
; CHECK-NEXT: mul x8, x9, x8
; CHECK-NEXT: fmov d1, x10
; CHECK-NEXT: mul x9, x15, x12
; CHECK-NEXT: fmov d0, x11
; CHECK-NEXT: mov v1.d[1], x8
; CHECK-NEXT: mov v0.d[1], x9
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <4 x i32> %src1 to <4 x i64> %in1 = zext <4 x i32> %src1 to <4 x i64>
%in2 = and <4 x i64> %src2, <i64 255, i64 255, i64 255, i64 255> %in2 = and <4 x i64> %src2, <i64 255, i64 255, i64 255, i64 255>
%out = mul nsw <4 x i64> %in1, %in2 %out = mul nsw <4 x i64> %in1, %in2
ret <4 x i64> %out ret <4 x i64> %out
} }
define <4 x i64> @umull_and_v4i64_dup(<4 x i32> %src1, i64 %src2) { define <4 x i64> @umull_and_v4i64_dup(<4 x i32> %src1, i64 %src2) {
; CHECK-LABEL: umull_and_v4i64_dup: ; CHECK-LABEL: umull_and_v4i64_dup:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ushll2 v1.2d, v0.4s, #0
; CHECK-NEXT: and x8, x0, #0xff ; CHECK-NEXT: and x8, x0, #0xff
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-NEXT: fmov x9, d1 ; CHECK-NEXT: dup v2.2d, x8
; CHECK-NEXT: fmov x11, d0 ; CHECK-NEXT: xtn v2.2s, v2.2d
; CHECK-NEXT: mov x10, v1.d[1] ; CHECK-NEXT: umull v0.2d, v0.2s, v2.2s
; CHECK-NEXT: mov x12, v0.d[1] ; CHECK-NEXT: umull v1.2d, v1.2s, v2.2s
; CHECK-NEXT: mul x9, x9, x8
; CHECK-NEXT: mul x11, x11, x8
; CHECK-NEXT: mul x10, x10, x8
; CHECK-NEXT: mul x8, x12, x8
; CHECK-NEXT: fmov d1, x9
; CHECK-NEXT: fmov d0, x11
; CHECK-NEXT: mov v1.d[1], x10
; CHECK-NEXT: mov v0.d[1], x8
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%in1 = zext <4 x i32> %src1 to <4 x i64> %in1 = zext <4 x i32> %src1 to <4 x i64>
%in2 = and i64 %src2, 255 %in2 = and i64 %src2, 255
%broadcast.splatinsert = insertelement <4 x i64> undef, i64 %in2, i64 0 %broadcast.splatinsert = insertelement <4 x i64> undef, i64 %in2, i64 0
%broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer
%out = mul nsw <4 x i64> %in1, %broadcast.splat %out = mul nsw <4 x i64> %in1, %broadcast.splat
ret <4 x i64> %out ret <4 x i64> %out
} }