This is an archive of the discontinued LLVM Phabricator instance.

Differential D114431

[AArch64][SVE] Recognize all ones mask during fixed mask generation
ClosedPublic

Authored by bsmith on Nov 23 2021, 3:29 AM.

Details

Reviewers
paulwalker-arm
peterwaller-arm
david-arm
DavidTruby
efriedma
Commits
rG080ef0b6a698: [AArch64][SVE] Recognize all ones mask during fixed mask generation

Diff Detail

Event Timeline

bsmith created this revision.Nov 23 2021, 3:29 AM
Herald added a reviewer: efriedma. · View Herald TranscriptNov 23 2021, 3:29 AM
Herald added subscribers: psnobl, hiraditya, kristof.beyls, tschuett. · View Herald Transcript
bsmith requested review of this revision.Nov 23 2021, 3:29 AM
Herald added a project: Restricted Project. · View Herald TranscriptNov 23 2021, 3:29 AM
Herald added a subscriber: llvm-commits. · View Herald Transcript
paulwalker-arm accepted this revision.Nov 23 2021, 4:02 AM
paulwalker-arm added inline comments.
llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll
2

Given the patch, testing every vector length seems a bit extreme but I guess it's better to be safe than sorry.

This revision is now accepted and ready to land.Nov 23 2021, 4:02 AM
Harbormaster completed remote builds in B135592: Diff 389147.Nov 23 2021, 7:24 AM
Matt added a subscriber: Matt.Nov 23 2021, 9:38 AM
Closed by commit rG080ef0b6a698: [AArch64][SVE] Recognize all ones mask during fixed mask generation (authored by bsmith). · Explain WhyNov 24 2021, 6:03 AM
This revision was automatically updated to reflect the committed changes.
bsmith added a commit: rG080ef0b6a698: [AArch64][SVE] Recognize all ones mask during fixed mask generation.

Revision Contents

PathSize
llvm/
lib/
Target/
AArch64/
13 lines
test/
CodeGen/
AArch64/
450 lines
132 lines
128 lines

Diff 389480

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 4,500 Lines • ▼ Show 20 Lines if (MemVT.getScalarSizeInBits() <= IndexVT.getScalarSizeInBits()) {
IndexVT = getContainerForFixedLengthVector(DAG, IndexVT); IndexVT = getContainerForFixedLengthVector(DAG, IndexVT);
MemVT = IndexVT.changeVectorElementType(MemVT.getVectorElementType()); MemVT = IndexVT.changeVectorElementType(MemVT.getVectorElementType());
} else { } else {
MemVT = getContainerForFixedLengthVector(DAG, MemVT); MemVT = getContainerForFixedLengthVector(DAG, MemVT);
IndexVT = MemVT.changeTypeToInteger(); IndexVT = MemVT.changeTypeToInteger();
} }
InputVT = DAG.getValueType(MemVT.changeTypeToInteger()); InputVT = DAG.getValueType(MemVT.changeTypeToInteger());
Mask = DAG.getNode( Mask = DAG.getNode(
ISD::ZERO_EXTEND, DL, ISD::SIGN_EXTEND, DL,
VT.changeVectorElementType(IndexVT.getVectorElementType()), Mask); VT.changeVectorElementType(IndexVT.getVectorElementType()), Mask);
} }
if (PassThru->isUndef() || isZerosVector(PassThru.getNode())) if (PassThru->isUndef() || isZerosVector(PassThru.getNode()))
PassThru = SDValue(); PassThru = SDValue();
if (VT.isFloatingPoint() && !IsFixedLength) { if (VT.isFloatingPoint() && !IsFixedLength) {
// Handle FP data by using an integer gather and casting the result. // Handle FP data by using an integer gather and casting the result.
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Lines if (IsFixedLength) {
StoreVal = StoreVal =
DAG.getNode(ISD::BITCAST, DL, VT.changeTypeToInteger(), StoreVal); DAG.getNode(ISD::BITCAST, DL, VT.changeTypeToInteger(), StoreVal);
StoreVal = DAG.getNode( StoreVal = DAG.getNode(
ISD::ANY_EXTEND, DL, ISD::ANY_EXTEND, DL,
VT.changeVectorElementType(IndexVT.getVectorElementType()), StoreVal); VT.changeVectorElementType(IndexVT.getVectorElementType()), StoreVal);
StoreVal = convertToScalableVector(DAG, IndexVT, StoreVal); StoreVal = convertToScalableVector(DAG, IndexVT, StoreVal);
Mask = DAG.getNode( Mask = DAG.getNode(
ISD::ZERO_EXTEND, DL, ISD::SIGN_EXTEND, DL,
VT.changeVectorElementType(IndexVT.getVectorElementType()), Mask); VT.changeVectorElementType(IndexVT.getVectorElementType()), Mask);
} else if (VT.isFloatingPoint()) { } else if (VT.isFloatingPoint()) {
// Handle FP data by casting the data so an integer scatter can be used. // Handle FP data by casting the data so an integer scatter can be used.
EVT StoreValVT = getPackedSVEVectorVT(VT.getVectorElementCount()); EVT StoreValVT = getPackedSVEVectorVT(VT.getVectorElementCount());
StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG); StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG);
InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger()); InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
} }
▲ Show 20 Lines • Show All 13,992 Lines • ▼ Show 20 Lines
} }
static SDValue convertFixedMaskToScalableVector(SDValue Mask, static SDValue convertFixedMaskToScalableVector(SDValue Mask,
SelectionDAG &DAG) { SelectionDAG &DAG) {
SDLoc DL(Mask); SDLoc DL(Mask);
EVT InVT = Mask.getValueType(); EVT InVT = Mask.getValueType();
EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT); EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
auto Pg = getPredicateForFixedLengthVector(DAG, DL, InVT);
if (ISD::isBuildVectorAllOnes(Mask.getNode()))
return Pg;
auto Op1 = convertToScalableVector(DAG, ContainerVT, Mask); auto Op1 = convertToScalableVector(DAG, ContainerVT, Mask);
auto Op2 = DAG.getConstant(0, DL, ContainerVT); auto Op2 = DAG.getConstant(0, DL, ContainerVT);
auto Pg = getPredicateForFixedLengthVector(DAG, DL, InVT);
EVT CmpVT = Pg.getValueType(); return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, Pg.getValueType(),
return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, CmpVT,
{Pg, Op1, Op2, DAG.getCondCode(ISD::SETNE)}); {Pg, Op1, Op2, DAG.getCondCode(ISD::SETNE)});
} }
// Convert all fixed length vector loads larger than NEON to masked_loads. // Convert all fixed length vector loads larger than NEON to masked_loads.
SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE( SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE(
SDValue Op, SelectionDAG &DAG) const { SDValue Op, SelectionDAG &DAG) const {
auto Load = cast<MaskedLoadSDNode>(Op); auto Load = cast<MaskedLoadSDNode>(Op);
▲ Show 20 Lines • Show All 779 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll

  • This file was added.
; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_256
; RUN: llc -aarch64-sve-vector-bits-min=384 < %s | FileCheck %s -check-prefixes=CHECK
paulwalker-armUnsubmitted
Not Done
ReplyInline Actions

Given the patch, testing every vector length seems a bit extreme but I guess it's better to be safe than sorry.

paulwalker-arm: Given the patch, testing every vector length seems a bit extreme but I guess it's better to be…
; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=640 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=768 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=896 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1024,VBITS_GE_512
; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048,VBITS_GE_1024,VBITS_GE_512
target triple = "aarch64-unknown-linux-gnu"
;
; LD1B
;
define void @masked_gather_v2i8(<2 x i8>* %a, <2 x i8*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: ptrue p0.s, vl2
; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: st1b { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%ptrs = load <2 x i8*>, <2 x i8*>* %b
%vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i8> undef)
store <2 x i8> %vals, <2 x i8>* %a
ret void
}
define void @masked_gather_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: st1b { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%ptrs = load <4 x i8*>, <4 x i8*>* %b
%vals = call <4 x i8> @llvm.masked.gather.v4i8(<4 x i8*> %ptrs, i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
store <4 x i8> %vals, <4 x i8>* %a
ret void
}
define void @masked_gather_v8i8(<8 x i8>* %a, <8 x i8*>* %b) #0 {
; VBITS_EQ_256-LABEL: masked_gather_v8i8:
; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: ld1sb { z0.d }, p0/z, [z0.d]
; VBITS_EQ_256-NEXT: ld1sb { z1.d }, p0/z, [z1.d]
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h
; VBITS_EQ_256-NEXT: uzp1 v0.8b, v1.8b, v0.8b
; VBITS_EQ_256-NEXT: str d0, [x0]
; VBITS_EQ_256-NEXT: ret
;
; VBITS_GE_512-LABEL: masked_gather_v8i8:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [z0.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_512-NEXT: str d0, [x0]
; VBITS_GE_512-NEXT: ret
%ptrs = load <8 x i8*>, <8 x i8*>* %b
%vals = call <8 x i8> @llvm.masked.gather.v8i8(<8 x i8*> %ptrs, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i8> undef)
store <8 x i8> %vals, <8 x i8>* %a
ret void
}
define void @masked_gather_v16i8(<16 x i8>* %a, <16 x i8*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i8:
; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: ld1b { z0.d }, p0/z, [z0.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_1024-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_1024-NEXT: str q0, [x0]
; VBITS_GE_1024-NEXT: ret
%ptrs = load <16 x i8*>, <16 x i8*>* %b
%vals = call <16 x i8> @llvm.masked.gather.v16i8(<16 x i8*> %ptrs, i32 8, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
store <16 x i8> %vals, <16 x i8>* %a
ret void
}
define void @masked_gather_v32i8(<32 x i8>* %a, <32 x i8*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i8:
; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.d, vl32
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_2048-NEXT: ld1b { z0.d }, p0/z, [z0.d]
; VBITS_GE_2048-NEXT: ptrue p0.b, vl32
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_2048-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_2048-NEXT: ret
%ptrs = load <32 x i8*>, <32 x i8*>* %b
%vals = call <32 x i8> @llvm.masked.gather.v32i8(<32 x i8*> %ptrs, i32 8, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
store <32 x i8> %vals, <32 x i8>* %a
ret void
}
;
; LD1H
;
define void @masked_gather_v2i16(<2 x i16>* %a, <2 x i16*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: ptrue p0.s, vl2
; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: st1h { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%ptrs = load <2 x i16*>, <2 x i16*>* %b
%vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i16> undef)
store <2 x i16> %vals, <2 x i16>* %a
ret void
}
define void @masked_gather_v4i16(<4 x i16>* %a, <4 x i16*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret
%ptrs = load <4 x i16*>, <4 x i16*>* %b
%vals = call <4 x i16> @llvm.masked.gather.v4i16(<4 x i16*> %ptrs, i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef)
store <4 x i16> %vals, <4 x i16>* %a
ret void
}
define void @masked_gather_v8i16(<8 x i16>* %a, <8 x i16*>* %b) #0 {
; VBITS_EQ_256-LABEL: masked_gather_v8i16:
; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: ld1h { z0.d }, p0/z, [z0.d]
; VBITS_EQ_256-NEXT: ld1h { z1.d }, p0/z, [z1.d]
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h
; VBITS_EQ_256-NEXT: mov v1.d[1], v0.d[0]
; VBITS_EQ_256-NEXT: str q1, [x0]
; VBITS_EQ_256-NEXT: ret
;
; VBITS_GE_512-LABEL: masked_gather_v8i16:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z0.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: str q0, [x0]
; VBITS_GE_512-NEXT: ret
%ptrs = load <8 x i16*>, <8 x i16*>* %b
%vals = call <8 x i16> @llvm.masked.gather.v8i16(<8 x i16*> %ptrs, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
store <8 x i16> %vals, <8 x i16>* %a
ret void
}
define void @masked_gather_v16i16(<16 x i16>* %a, <16 x i16*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i16:
; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: ld1h { z0.d }, p0/z, [z0.d]
; VBITS_GE_1024-NEXT: ptrue p0.h, vl16
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_1024-NEXT: ret
%ptrs = load <16 x i16*>, <16 x i16*>* %b
%vals = call <16 x i16> @llvm.masked.gather.v16i16(<16 x i16*> %ptrs, i32 8, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i16> undef)
store <16 x i16> %vals, <16 x i16>* %a
ret void
}
define void @masked_gather_v32i16(<32 x i16>* %a, <32 x i16*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i16:
; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.d, vl32
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p0/z, [z0.d]
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret
%ptrs = load <32 x i16*>, <32 x i16*>* %b
%vals = call <32 x i16> @llvm.masked.gather.v32i16(<32 x i16*> %ptrs, i32 8, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i16> undef)
store <32 x i16> %vals, <32 x i16>* %a
ret void
}
;
; LD1W
;
define void @masked_gather_v2i32(<2 x i32>* %a, <2 x i32*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret
%ptrs = load <2 x i32*>, <2 x i32*>* %b
%vals = call <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
store <2 x i32> %vals, <2 x i32>* %a
ret void
}
define void @masked_gather_v4i32(<4 x i32>* %a, <4 x i32*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
%ptrs = load <4 x i32*>, <4 x i32*>* %b
%vals = call <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*> %ptrs, i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
store <4 x i32> %vals, <4 x i32>* %a
ret void
}
define void @masked_gather_v8i32(<8 x i32>* %a, <8 x i32*>* %b) #0 {
; VBITS_EQ_256-LABEL: masked_gather_v8i32:
; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; VBITS_EQ_256-NEXT: ld1w { z1.d }, p0/z, [z1.d]
; VBITS_EQ_256-NEXT: ptrue p0.s, vl4
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_EQ_256-NEXT: splice z1.s, p0, z1.s, z0.s
; VBITS_EQ_256-NEXT: ptrue p0.s, vl8
; VBITS_EQ_256-NEXT: st1w { z1.s }, p0, [x0]
; VBITS_EQ_256-NEXT: ret
;
; VBITS_GE_512-LABEL: masked_gather_v8i32:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; VBITS_GE_512-NEXT: ptrue p0.s, vl8
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%ptrs = load <8 x i32*>, <8 x i32*>* %b
%vals = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %ptrs, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
store <8 x i32> %vals, <8 x i32>* %a
ret void
}
define void @masked_gather_v16i32(<16 x i32>* %a, <16 x i32*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i32:
; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; VBITS_GE_1024-NEXT: ptrue p0.s, vl16
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_1024-NEXT: ret
%ptrs = load <16 x i32*>, <16 x i32*>* %b
%vals = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptrs, i32 8, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
store <16 x i32> %vals, <16 x i32>* %a
ret void
}
define void @masked_gather_v32i32(<32 x i32>* %a, <32 x i32*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i32:
; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.d, vl32
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret
%ptrs = load <32 x i32*>, <32 x i32*>* %b
%vals = call <32 x i32> @llvm.masked.gather.v32i32(<32 x i32*> %ptrs, i32 8, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i32> undef)
store <32 x i32> %vals, <32 x i32>* %a
ret void
}
;
; LD1D
;
define void @masked_gather_v2i64(<2 x i64>* %a, <2 x i64*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x1]
; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
%ptrs = load <2 x i64*>, <2 x i64*>* %b
%vals = call <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> undef)
store <2 x i64> %vals, <2 x i64>* %a
ret void
}
define void @masked_gather_v4i64(<4 x i64>* %a, <4 x i64*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]
; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%ptrs = load <4 x i64*>, <4 x i64*>* %b
%vals = call <4 x i64> @llvm.masked.gather.v4i64(<4 x i64*> %ptrs, i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> undef)
store <4 x i64> %vals, <4 x i64>* %a
ret void
}
define void @masked_gather_v8i64(<8 x i64>* %a, <8 x i64*>* %b) #0 {
; VBITS_EQ_256-LABEL: masked_gather_v8i64:
; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p0/z, [z1.d]
; VBITS_EQ_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
; VBITS_EQ_256-NEXT: st1d { z1.d }, p0, [x0]
; VBITS_EQ_256-NEXT: ret
;
; VBITS_GE_512-LABEL: masked_gather_v8i64:
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%ptrs = load <8 x i64*>, <8 x i64*>* %b
%vals = call <8 x i64> @llvm.masked.gather.v8i64(<8 x i64*> %ptrs, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i64> undef)
store <8 x i64> %vals, <8 x i64>* %a
ret void
}
define void @masked_gather_v16i64(<16 x i64>* %a, <16 x i64*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i64:
; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; VBITS_GE_1024-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_1024-NEXT: ret
%ptrs = load <16 x i64*>, <16 x i64*>* %b
%vals = call <16 x i64> @llvm.masked.gather.v16i64(<16 x i64*> %ptrs, i32 8, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i64> undef)
store <16 x i64> %vals, <16 x i64>* %a
ret void
}
define void @masked_gather_v32i64(<32 x i64>* %a, <32 x i64*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i64:
; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.d, vl32
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; VBITS_GE_2048-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_2048-NEXT: ret
%ptrs = load <32 x i64*>, <32 x i64*>* %b
%vals = call <32 x i64> @llvm.masked.gather.v32i64(<32 x i64*> %ptrs, i32 8, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true,
i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i64> undef)
store <32 x i64> %vals, <32 x i64>* %a
ret void
}
declare <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
declare <4 x i8> @llvm.masked.gather.v4i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
declare <8 x i8> @llvm.masked.gather.v8i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
declare <16 x i8> @llvm.masked.gather.v16i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
declare <32 x i8> @llvm.masked.gather.v32i8(<32 x i8*>, i32, <32 x i1>, <32 x i8>)
declare <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>)
declare <4 x i16> @llvm.masked.gather.v4i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
declare <8 x i16> @llvm.masked.gather.v8i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>)
declare <16 x i16> @llvm.masked.gather.v16i16(<16 x i16*>, i32, <16 x i1>, <16 x i16>)
declare <32 x i16> @llvm.masked.gather.v32i16(<32 x i16*>, i32, <32 x i1>, <32 x i16>)
declare <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
declare <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
declare <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>)
declare <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>)
declare <32 x i32> @llvm.masked.gather.v32i32(<32 x i32*>, i32, <32 x i1>, <32 x i32>)
declare <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*>, i32, <2 x i1>, <2 x i64>)
declare <4 x i64> @llvm.masked.gather.v4i64(<4 x i64*>, i32, <4 x i1>, <4 x i64>)
declare <8 x i64> @llvm.masked.gather.v8i64(<8 x i64*>, i32, <8 x i1>, <8 x i64>)
declare <16 x i64> @llvm.masked.gather.v16i64(<16 x i64*>, i32, <16 x i1>, <16 x i64>)
declare <32 x i64> @llvm.masked.gather.v32i64(<32 x i64*>, i32, <32 x i1>, <32 x i64>)
attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll

Show All 28 Lines
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldrb w8, [x0] ; CHECK-NEXT: ldrb w8, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: ldr q1, [x1]
; CHECK-NEXT: fmov s0, w8 ; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: ldrb w8, [x0, #1] ; CHECK-NEXT: ldrb w8, [x0, #1]
; CHECK-NEXT: mov v0.s[1], w8 ; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: cmeq v0.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v0.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ptrue p0.s, vl2
; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: st1b { z0.s }, p0, [x0] ; CHECK-NEXT: st1b { z0.s }, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <2 x i8>, <2 x i8>* %a %cval = load <2 x i8>, <2 x i8>* %a
%ptrs = load <2 x i8*>, <2 x i8*>* %b %ptrs = load <2 x i8*>, <2 x i8*>* %b
%mask = icmp eq <2 x i8> %cval, zeroinitializer %mask = icmp eq <2 x i8> %cval, zeroinitializer
%vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*> %ptrs, i32 8, <2 x i1> %mask, <2 x i8> undef) %vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*> %ptrs, i32 8, <2 x i1> %mask, <2 x i8> undef)
store <2 x i8> %vals, <2 x i8>* %a store <2 x i8> %vals, <2 x i8>* %a
ret void ret void
} }
define void @masked_gather_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 { define void @masked_gather_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i8: ; CHECK-LABEL: masked_gather_v4i8:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr s0, [x0] ; CHECK-NEXT: ldr s0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1sb { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: st1b { z0.h }, p0, [x0] ; CHECK-NEXT: st1b { z0.h }, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <4 x i8>, <4 x i8>* %a %cval = load <4 x i8>, <4 x i8>* %a
Show All 15 Lines
; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] ; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z3.d }, p0/z, [x1] ; VBITS_EQ_256-NEXT: ld1d { z3.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: zip2 v1.8b, v0.8b, v0.8b ; VBITS_EQ_256-NEXT: zip2 v1.8b, v0.8b, v0.8b
; VBITS_EQ_256-NEXT: zip1 v0.8b, v0.8b, v0.8b ; VBITS_EQ_256-NEXT: zip1 v0.8b, v0.8b, v0.8b
; VBITS_EQ_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_EQ_256-NEXT: shl v1.4h, v1.4h, #8
; VBITS_EQ_256-NEXT: shl v0.4h, v0.4h, #8 ; VBITS_EQ_256-NEXT: shl v0.4h, v0.4h, #8
; VBITS_EQ_256-NEXT: sshr v1.4h, v1.4h, #8 ; VBITS_EQ_256-NEXT: sshr v1.4h, v1.4h, #8
; VBITS_EQ_256-NEXT: sshr v0.4h, v0.4h, #8 ; VBITS_EQ_256-NEXT: sshr v0.4h, v0.4h, #8
; VBITS_EQ_256-NEXT: uunpklo z1.s, z1.h ; VBITS_EQ_256-NEXT: sunpklo z1.s, z1.h
; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h ; VBITS_EQ_256-NEXT: sunpklo z0.s, z0.h
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: sunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z1.d, #0
; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z0.d, #0
; VBITS_EQ_256-NEXT: ld1sb { z0.d }, p1/z, [z2.d] ; VBITS_EQ_256-NEXT: ld1sb { z0.d }, p1/z, [z2.d]
; VBITS_EQ_256-NEXT: ld1sb { z1.d }, p0/z, [z3.d] ; VBITS_EQ_256-NEXT: ld1sb { z1.d }, p0/z, [z3.d]
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h
; VBITS_EQ_256-NEXT: uzp1 v0.8b, v1.8b, v0.8b ; VBITS_EQ_256-NEXT: uzp1 v0.8b, v1.8b, v0.8b
; VBITS_EQ_256-NEXT: str d0, [x0] ; VBITS_EQ_256-NEXT: str d0, [x0]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_gather_v8i8: ; VBITS_GE_512-LABEL: masked_gather_v8i8:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr d0, [x0] ; VBITS_GE_512-NEXT: ldr d0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_512-NEXT: cmeq v0.8b, v0.8b, #0
; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [z1.d] ; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [z1.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_512-NEXT: str d0, [x0] ; VBITS_GE_512-NEXT: str d0, [x0]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%cval = load <8 x i8>, <8 x i8>* %a %cval = load <8 x i8>, <8 x i8>* %a
%ptrs = load <8 x i8*>, <8 x i8*>* %b %ptrs = load <8 x i8*>, <8 x i8*>* %b
%mask = icmp eq <8 x i8> %cval, zeroinitializer %mask = icmp eq <8 x i8> %cval, zeroinitializer
%vals = call <8 x i8> @llvm.masked.gather.v8i8(<8 x i8*> %ptrs, i32 8, <8 x i1> %mask, <8 x i8> undef) %vals = call <8 x i8> @llvm.masked.gather.v8i8(<8 x i8*> %ptrs, i32 8, <8 x i1> %mask, <8 x i8> undef)
store <8 x i8> %vals, <8 x i8>* %a store <8 x i8> %vals, <8 x i8>* %a
ret void ret void
} }
define void @masked_gather_v16i8(<16 x i8>* %a, <16 x i8*>* %b) #0 { define void @masked_gather_v16i8(<16 x i8>* %a, <16 x i8*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i8: ; VBITS_GE_1024-LABEL: masked_gather_v16i8:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ldr q0, [x0] ; VBITS_GE_1024-NEXT: ldr q0, [x0]
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: cmeq v0.16b, v0.16b, #0 ; VBITS_GE_1024-NEXT: cmeq v0.16b, v0.16b, #0
; VBITS_GE_1024-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_1024-NEXT: sunpklo z0.h, z0.b
; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_1024-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p0/z, z0.d, #0
; VBITS_GE_1024-NEXT: ld1b { z0.d }, p0/z, [z1.d] ; VBITS_GE_1024-NEXT: ld1b { z0.d }, p0/z, [z1.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_1024-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_1024-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_1024-NEXT: str q0, [x0] ; VBITS_GE_1024-NEXT: str q0, [x0]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%cval = load <16 x i8>, <16 x i8>* %a %cval = load <16 x i8>, <16 x i8>* %a
%ptrs = load <16 x i8*>, <16 x i8*>* %b %ptrs = load <16 x i8*>, <16 x i8*>* %b
%mask = icmp eq <16 x i8> %cval, zeroinitializer %mask = icmp eq <16 x i8> %cval, zeroinitializer
%vals = call <16 x i8> @llvm.masked.gather.v16i8(<16 x i8*> %ptrs, i32 8, <16 x i1> %mask, <16 x i8> undef) %vals = call <16 x i8> @llvm.masked.gather.v16i8(<16 x i8*> %ptrs, i32 8, <16 x i1> %mask, <16 x i8> undef)
store <16 x i8> %vals, <16 x i8>* %a store <16 x i8> %vals, <16 x i8>* %a
ret void ret void
} }
define void @masked_gather_v32i8(<32 x i8>* %a, <32 x i8*>* %b) #0 { define void @masked_gather_v32i8(<32 x i8>* %a, <32 x i8*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i8: ; VBITS_GE_2048-LABEL: masked_gather_v32i8:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.b, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.b, vl32
; VBITS_GE_2048-NEXT: ptrue p2.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p2.d, vl32
; VBITS_GE_2048-NEXT: ld1b { z0.b }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1b { z0.b }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p2/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p2/z, [x1]
; VBITS_GE_2048-NEXT: cmpeq p1.b, p0/z, z0.b, #0 ; VBITS_GE_2048-NEXT: cmpeq p1.b, p0/z, z0.b, #0
; VBITS_GE_2048-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_2048-NEXT: sunpklo z0.h, z0.b
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p2/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p2/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1b { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1b { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_2048-NEXT: uzp1 z0.b, z0.b, z0.b
; VBITS_GE_2048-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_2048-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cval = load <32 x i8>, <32 x i8>* %a %cval = load <32 x i8>, <32 x i8>* %a
Show All 13 Lines
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldrh w8, [x0] ; CHECK-NEXT: ldrh w8, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: ldr q1, [x1]
; CHECK-NEXT: fmov s0, w8 ; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: ldrh w8, [x0, #2] ; CHECK-NEXT: ldrh w8, [x0, #2]
; CHECK-NEXT: mov v0.s[1], w8 ; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: cmeq v0.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v0.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1sh { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ptrue p0.s, vl2
; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: st1h { z0.s }, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <2 x i16>, <2 x i16>* %a %cval = load <2 x i16>, <2 x i16>* %a
%ptrs = load <2 x i16*>, <2 x i16*>* %b %ptrs = load <2 x i16*>, <2 x i16*>* %b
%mask = icmp eq <2 x i16> %cval, zeroinitializer %mask = icmp eq <2 x i16> %cval, zeroinitializer
%vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*> %ptrs, i32 8, <2 x i1> %mask, <2 x i16> undef) %vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*> %ptrs, i32 8, <2 x i1> %mask, <2 x i16> undef)
store <2 x i16> %vals, <2 x i16>* %a store <2 x i16> %vals, <2 x i16>* %a
ret void ret void
} }
define void @masked_gather_v4i16(<4 x i16>* %a, <4 x i16*>* %b) #0 { define void @masked_gather_v4i16(<4 x i16>* %a, <4 x i16*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i16: ; CHECK-LABEL: masked_gather_v4i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: str d0, [x0] ; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <4 x i16>, <4 x i16>* %a %cval = load <4 x i16>, <4 x i16>* %a
%ptrs = load <4 x i16*>, <4 x i16*>* %b %ptrs = load <4 x i16*>, <4 x i16*>* %b
Show All 9 Lines
; VBITS_EQ_256: // %bb.0: ; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: ldr q0, [x0] ; VBITS_EQ_256-NEXT: ldr q0, [x0]
; VBITS_EQ_256-NEXT: mov x8, #4 ; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4 ; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_EQ_256-NEXT: cmeq v0.8h, v0.8h, #0
; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] ; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z3.d }, p0/z, [x1] ; VBITS_EQ_256-NEXT: ld1d { z3.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8 ; VBITS_EQ_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h ; VBITS_EQ_256-NEXT: sunpklo z0.s, z0.h
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: sunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z0.d, #0
; VBITS_EQ_256-NEXT: ld1h { z0.d }, p1/z, [z3.d] ; VBITS_EQ_256-NEXT: ld1h { z0.d }, p1/z, [z3.d]
; VBITS_EQ_256-NEXT: uunpklo z1.s, z1.h ; VBITS_EQ_256-NEXT: sunpklo z1.s, z1.h
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0
; VBITS_EQ_256-NEXT: ld1h { z1.d }, p0/z, [z2.d] ; VBITS_EQ_256-NEXT: ld1h { z1.d }, p0/z, [z2.d]
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_EQ_256-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_EQ_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_EQ_256-NEXT: uzp1 z1.h, z1.h, z1.h
; VBITS_EQ_256-NEXT: mov v0.d[1], v1.d[0] ; VBITS_EQ_256-NEXT: mov v0.d[1], v1.d[0]
; VBITS_EQ_256-NEXT: str q0, [x0] ; VBITS_EQ_256-NEXT: str q0, [x0]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_gather_v8i16: ; VBITS_GE_512-LABEL: masked_gather_v8i16:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr q0, [x0] ; VBITS_GE_512-NEXT: ldr q0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_512-NEXT: cmeq v0.8h, v0.8h, #0
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] ; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: str q0, [x0] ; VBITS_GE_512-NEXT: str q0, [x0]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%cval = load <8 x i16>, <8 x i16>* %a %cval = load <8 x i16>, <8 x i16>* %a
%ptrs = load <8 x i16*>, <8 x i16*>* %b %ptrs = load <8 x i16*>, <8 x i16*>* %b
%mask = icmp eq <8 x i16> %cval, zeroinitializer %mask = icmp eq <8 x i16> %cval, zeroinitializer
%vals = call <8 x i16> @llvm.masked.gather.v8i16(<8 x i16*> %ptrs, i32 8, <8 x i1> %mask, <8 x i16> undef) %vals = call <8 x i16> @llvm.masked.gather.v8i16(<8 x i16*> %ptrs, i32 8, <8 x i1> %mask, <8 x i16> undef)
store <8 x i16> %vals, <8 x i16>* %a store <8 x i16> %vals, <8 x i16>* %a
ret void ret void
} }
define void @masked_gather_v16i16(<16 x i16>* %a, <16 x i16*>* %b) #0 { define void @masked_gather_v16i16(<16 x i16>* %a, <16 x i16*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i16: ; VBITS_GE_1024-LABEL: masked_gather_v16i16:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.h, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.h, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_1024-NEXT: cmpeq p2.h, p0/z, z0.h, #0 ; VBITS_GE_1024-NEXT: cmpeq p2.h, p0/z, z0.h, #0
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z0.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_1024-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_1024-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_1024-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_1024-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_1024-NEXT: ld1h { z0.d }, p1/z, [z0.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%cval = load <16 x i16>, <16 x i16>* %a %cval = load <16 x i16>, <16 x i16>* %a
%ptrs = load <16 x i16*>, <16 x i16*>* %b %ptrs = load <16 x i16*>, <16 x i16*>* %b
%mask = icmp eq <16 x i16> %cval, zeroinitializer %mask = icmp eq <16 x i16> %cval, zeroinitializer
%vals = call <16 x i16> @llvm.masked.gather.v16i16(<16 x i16*> %ptrs, i32 8, <16 x i1> %mask, <16 x i16> undef) %vals = call <16 x i16> @llvm.masked.gather.v16i16(<16 x i16*> %ptrs, i32 8, <16 x i1> %mask, <16 x i16> undef)
store <16 x i16> %vals, <16 x i16>* %a store <16 x i16> %vals, <16 x i16>* %a
ret void ret void
} }
define void @masked_gather_v32i16(<32 x i16>* %a, <32 x i16*>* %b) #0 { define void @masked_gather_v32i16(<32 x i16>* %a, <32 x i16*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i16: ; VBITS_GE_2048-LABEL: masked_gather_v32i16:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: cmpeq p2.h, p0/z, z0.h, #0 ; VBITS_GE_2048-NEXT: cmpeq p2.h, p0/z, z0.h, #0
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [z0.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cval = load <32 x i16>, <32 x i16>* %a %cval = load <32 x i16>, <32 x i16>* %a
%ptrs = load <32 x i16*>, <32 x i16*>* %b %ptrs = load <32 x i16*>, <32 x i16*>* %b
Show All 9 Lines
define void @masked_gather_v2i32(<2 x i32>* %a, <2 x i32*>* %b) #0 { define void @masked_gather_v2i32(<2 x i32>* %a, <2 x i32*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2i32: ; CHECK-LABEL: masked_gather_v2i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: ldr q1, [x1]
; CHECK-NEXT: cmeq v0.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v0.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: str d0, [x0] ; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <2 x i32>, <2 x i32>* %a %cval = load <2 x i32>, <2 x i32>* %a
%ptrs = load <2 x i32*>, <2 x i32*>* %b %ptrs = load <2 x i32*>, <2 x i32*>* %b
%mask = icmp eq <2 x i32> %cval, zeroinitializer %mask = icmp eq <2 x i32> %cval, zeroinitializer
%vals = call <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %ptrs, i32 8, <2 x i1> %mask, <2 x i32> undef) %vals = call <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %ptrs, i32 8, <2 x i1> %mask, <2 x i32> undef)
store <2 x i32> %vals, <2 x i32>* %a store <2 x i32> %vals, <2 x i32>* %a
ret void ret void
} }
define void @masked_gather_v4i32(<4 x i32>* %a, <4 x i32*>* %b) #0 { define void @masked_gather_v4i32(<4 x i32>* %a, <4 x i32*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4i32: ; CHECK-LABEL: masked_gather_v4i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <4 x i32>, <4 x i32>* %a %cval = load <4 x i32>, <4 x i32>* %a
%ptrs = load <4 x i32*>, <4 x i32*>* %b %ptrs = load <4 x i32*>, <4 x i32*>* %b
%mask = icmp eq <4 x i32> %cval, zeroinitializer %mask = icmp eq <4 x i32> %cval, zeroinitializer
Show All 9 Lines
; VBITS_EQ_256-NEXT: ptrue p0.s, vl8 ; VBITS_EQ_256-NEXT: ptrue p0.s, vl8
; VBITS_EQ_256-NEXT: mov x8, #4 ; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_EQ_256-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_EQ_256-NEXT: ptrue p1.d, vl4 ; VBITS_EQ_256-NEXT: ptrue p1.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z1.d }, p1/z, [x1, x8, lsl #3] ; VBITS_EQ_256-NEXT: ld1d { z1.d }, p1/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z2.d }, p1/z, [x1] ; VBITS_EQ_256-NEXT: ld1d { z2.d }, p1/z, [x1]
; VBITS_EQ_256-NEXT: cmpeq p2.s, p0/z, z0.s, #0 ; VBITS_EQ_256-NEXT: cmpeq p2.s, p0/z, z0.s, #0
; VBITS_EQ_256-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_EQ_256-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_EQ_256-NEXT: uunpklo z3.d, z0.s ; VBITS_EQ_256-NEXT: sunpklo z3.d, z0.s
; VBITS_EQ_256-NEXT: ext z0.b, z0.b, z0.b, #16 ; VBITS_EQ_256-NEXT: ext z0.b, z0.b, z0.b, #16
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: sunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: cmpne p2.d, p1/z, z3.d, #0 ; VBITS_EQ_256-NEXT: cmpne p2.d, p1/z, z3.d, #0
; VBITS_EQ_256-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_EQ_256-NEXT: ld1w { z2.d }, p2/z, [z2.d] ; VBITS_EQ_256-NEXT: ld1w { z2.d }, p2/z, [z2.d]
; VBITS_EQ_256-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_EQ_256-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_EQ_256-NEXT: ptrue p1.s, vl4 ; VBITS_EQ_256-NEXT: ptrue p1.s, vl4
; VBITS_EQ_256-NEXT: uzp1 z1.s, z2.s, z2.s ; VBITS_EQ_256-NEXT: uzp1 z1.s, z2.s, z2.s
; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_EQ_256-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_EQ_256-NEXT: splice z1.s, p1, z1.s, z0.s ; VBITS_EQ_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_EQ_256-NEXT: st1w { z1.s }, p0, [x0] ; VBITS_EQ_256-NEXT: st1w { z1.s }, p0, [x0]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_gather_v8i32: ; VBITS_GE_512-LABEL: masked_gather_v8i32:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ptrue p1.d, vl8
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p2.s, p0/z, z0.s, #0 ; VBITS_GE_512-NEXT: cmpeq p2.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_512-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%cval = load <8 x i32>, <8 x i32>* %a %cval = load <8 x i32>, <8 x i32>* %a
%ptrs = load <8 x i32*>, <8 x i32*>* %b %ptrs = load <8 x i32*>, <8 x i32*>* %b
%mask = icmp eq <8 x i32> %cval, zeroinitializer %mask = icmp eq <8 x i32> %cval, zeroinitializer
%vals = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %ptrs, i32 8, <8 x i1> %mask, <8 x i32> undef) %vals = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %ptrs, i32 8, <8 x i1> %mask, <8 x i32> undef)
store <8 x i32> %vals, <8 x i32>* %a store <8 x i32> %vals, <8 x i32>* %a
ret void ret void
} }
define void @masked_gather_v16i32(<16 x i32>* %a, <16 x i32*>* %b) #0 { define void @masked_gather_v16i32(<16 x i32>* %a, <16 x i32*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16i32: ; VBITS_GE_1024-LABEL: masked_gather_v16i32:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.s, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.s, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: cmpeq p2.s, p0/z, z0.s, #0 ; VBITS_GE_1024-NEXT: cmpeq p2.s, p0/z, z0.s, #0
; VBITS_GE_1024-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_1024-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_1024-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%cval = load <16 x i32>, <16 x i32>* %a %cval = load <16 x i32>, <16 x i32>* %a
%ptrs = load <16 x i32*>, <16 x i32*>* %b %ptrs = load <16 x i32*>, <16 x i32*>* %b
%mask = icmp eq <16 x i32> %cval, zeroinitializer %mask = icmp eq <16 x i32> %cval, zeroinitializer
%vals = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptrs, i32 8, <16 x i1> %mask, <16 x i32> undef) %vals = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptrs, i32 8, <16 x i1> %mask, <16 x i32> undef)
store <16 x i32> %vals, <16 x i32>* %a store <16 x i32> %vals, <16 x i32>* %a
ret void ret void
} }
define void @masked_gather_v32i32(<32 x i32>* %a, <32 x i32*>* %b) #0 { define void @masked_gather_v32i32(<32 x i32>* %a, <32 x i32*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32i32: ; VBITS_GE_2048-LABEL: masked_gather_v32i32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: cmpeq p2.s, p0/z, z0.s, #0 ; VBITS_GE_2048-NEXT: cmpeq p2.s, p0/z, z0.s, #0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cval = load <32 x i32>, <32 x i32>* %a %cval = load <32 x i32>, <32 x i32>* %a
%ptrs = load <32 x i32*>, <32 x i32*>* %b %ptrs = load <32 x i32*>, <32 x i32*>* %b
%mask = icmp eq <32 x i32> %cval, zeroinitializer %mask = icmp eq <32 x i32> %cval, zeroinitializer
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines
; CHECK-NEXT: sshr v1.2s, v1.2s, #16 ; CHECK-NEXT: sshr v1.2s, v1.2s, #16
; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: fmov w8, s1
; CHECK-NEXT: mov w9, v1.s[1] ; CHECK-NEXT: mov w9, v1.s[1]
; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: ldr q1, [x1]
; CHECK-NEXT: mov v0.h[0], w8 ; CHECK-NEXT: mov v0.h[0], w8
; CHECK-NEXT: mov v0.h[1], w9 ; CHECK-NEXT: mov v0.h[1], w9
; CHECK-NEXT: shl v0.4h, v0.4h, #15 ; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: sshr v0.4h, v0.4h, #15 ; CHECK-NEXT: sshr v0.4h, v0.4h, #15
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: str s0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <2 x half>, <2 x half>* %a %cval = load <2 x half>, <2 x half>* %a
%ptrs = load <2 x half*>, <2 x half*>* %b %ptrs = load <2 x half*>, <2 x half*>* %b
%mask = fcmp oeq <2 x half> %cval, zeroinitializer %mask = fcmp oeq <2 x half> %cval, zeroinitializer
%vals = call <2 x half> @llvm.masked.gather.v2f16(<2 x half*> %ptrs, i32 8, <2 x i1> %mask, <2 x half> undef) %vals = call <2 x half> @llvm.masked.gather.v2f16(<2 x half*> %ptrs, i32 8, <2 x i1> %mask, <2 x half> undef)
store <2 x half> %vals, <2 x half>* %a store <2 x half> %vals, <2 x half>* %a
ret void ret void
} }
define void @masked_gather_v4f16(<4 x half>* %a, <4 x half*>* %b) #0 { define void @masked_gather_v4f16(<4 x half>* %a, <4 x half*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4f16: ; CHECK-LABEL: masked_gather_v4f16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: fcmeq v0.4h, v0.4h, #0.0 ; CHECK-NEXT: fcmeq v0.4h, v0.4h, #0.0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: str d0, [x0] ; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <4 x half>, <4 x half>* %a %cval = load <4 x half>, <4 x half>* %a
%ptrs = load <4 x half*>, <4 x half*>* %b %ptrs = load <4 x half*>, <4 x half*>* %b
%mask = fcmp oeq <4 x half> %cval, zeroinitializer %mask = fcmp oeq <4 x half> %cval, zeroinitializer
%vals = call <4 x half> @llvm.masked.gather.v4f16(<4 x half*> %ptrs, i32 8, <4 x i1> %mask, <4 x half> undef) %vals = call <4 x half> @llvm.masked.gather.v4f16(<4 x half*> %ptrs, i32 8, <4 x i1> %mask, <4 x half> undef)
store <4 x half> %vals, <4 x half>* %a store <4 x half> %vals, <4 x half>* %a
ret void ret void
} }
define void @masked_gather_v8f16(<8 x half>* %a, <8 x half*>* %b) #0 { define void @masked_gather_v8f16(<8 x half>* %a, <8 x half*>* %b) #0 {
; VBITS_GE_512-LABEL: masked_gather_v8f16: ; VBITS_GE_512-LABEL: masked_gather_v8f16:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr q0, [x0] ; VBITS_GE_512-NEXT: ldr q0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: fcmeq v0.8h, v0.8h, #0.0 ; VBITS_GE_512-NEXT: fcmeq v0.8h, v0.8h, #0.0
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] ; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: str q0, [x0] ; VBITS_GE_512-NEXT: str q0, [x0]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%cval = load <8 x half>, <8 x half>* %a %cval = load <8 x half>, <8 x half>* %a
%ptrs = load <8 x half*>, <8 x half*>* %b %ptrs = load <8 x half*>, <8 x half*>* %b
%mask = fcmp oeq <8 x half> %cval, zeroinitializer %mask = fcmp oeq <8 x half> %cval, zeroinitializer
%vals = call <8 x half> @llvm.masked.gather.v8f16(<8 x half*> %ptrs, i32 8, <8 x i1> %mask, <8 x half> undef) %vals = call <8 x half> @llvm.masked.gather.v8f16(<8 x half*> %ptrs, i32 8, <8 x i1> %mask, <8 x half> undef)
store <8 x half> %vals, <8 x half>* %a store <8 x half> %vals, <8 x half>* %a
ret void ret void
} }
define void @masked_gather_v16f16(<16 x half>* %a, <16 x half*>* %b) #0 { define void @masked_gather_v16f16(<16 x half>* %a, <16 x half*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16f16: ; VBITS_GE_1024-LABEL: masked_gather_v16f16:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.h, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.h, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_1024-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_1024-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_1024-NEXT: ld1d { z0.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z0.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_1024-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_1024-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_1024-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_1024-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_1024-NEXT: ld1h { z0.d }, p1/z, [z0.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_1024-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%cval = load <16 x half>, <16 x half>* %a %cval = load <16 x half>, <16 x half>* %a
%ptrs = load <16 x half*>, <16 x half*>* %b %ptrs = load <16 x half*>, <16 x half*>* %b
%mask = fcmp oeq <16 x half> %cval, zeroinitializer %mask = fcmp oeq <16 x half> %cval, zeroinitializer
%vals = call <16 x half> @llvm.masked.gather.v16f16(<16 x half*> %ptrs, i32 8, <16 x i1> %mask, <16 x half> undef) %vals = call <16 x half> @llvm.masked.gather.v16f16(<16 x half*> %ptrs, i32 8, <16 x i1> %mask, <16 x half> undef)
store <16 x half> %vals, <16 x half>* %a store <16 x half> %vals, <16 x half>* %a
ret void ret void
} }
define void @masked_gather_v32f16(<32 x half>* %a, <32 x half*>* %b) #0 { define void @masked_gather_v32f16(<32 x half>* %a, <32 x half*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32f16: ; VBITS_GE_2048-LABEL: masked_gather_v32f16:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: ld1d { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [z0.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cval = load <32 x half>, <32 x half>* %a %cval = load <32 x half>, <32 x half>* %a
%ptrs = load <32 x half*>, <32 x half*>* %b %ptrs = load <32 x half*>, <32 x half*>* %b
Show All 9 Lines
define void @masked_gather_v2f32(<2 x float>* %a, <2 x float*>* %b) #0 { define void @masked_gather_v2f32(<2 x float>* %a, <2 x float*>* %b) #0 {
; CHECK-LABEL: masked_gather_v2f32: ; CHECK-LABEL: masked_gather_v2f32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: ldr q1, [x1]
; CHECK-NEXT: fcmeq v0.2s, v0.2s, #0.0 ; CHECK-NEXT: fcmeq v0.2s, v0.2s, #0.0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: str d0, [x0] ; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <2 x float>, <2 x float>* %a %cval = load <2 x float>, <2 x float>* %a
%ptrs = load <2 x float*>, <2 x float*>* %b %ptrs = load <2 x float*>, <2 x float*>* %b
%mask = fcmp oeq <2 x float> %cval, zeroinitializer %mask = fcmp oeq <2 x float> %cval, zeroinitializer
%vals = call <2 x float> @llvm.masked.gather.v2f32(<2 x float*> %ptrs, i32 8, <2 x i1> %mask, <2 x float> undef) %vals = call <2 x float> @llvm.masked.gather.v2f32(<2 x float*> %ptrs, i32 8, <2 x i1> %mask, <2 x float> undef)
store <2 x float> %vals, <2 x float>* %a store <2 x float> %vals, <2 x float>* %a
ret void ret void
} }
define void @masked_gather_v4f32(<4 x float>* %a, <4 x float*>* %b) #0 { define void @masked_gather_v4f32(<4 x float>* %a, <4 x float*>* %b) #0 {
; CHECK-LABEL: masked_gather_v4f32: ; CHECK-LABEL: masked_gather_v4f32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: fcmeq v0.4s, v0.4s, #0.0 ; CHECK-NEXT: fcmeq v0.4s, v0.4s, #0.0
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] ; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%cval = load <4 x float>, <4 x float>* %a %cval = load <4 x float>, <4 x float>* %a
%ptrs = load <4 x float*>, <4 x float*>* %b %ptrs = load <4 x float*>, <4 x float*>* %b
%mask = fcmp oeq <4 x float> %cval, zeroinitializer %mask = fcmp oeq <4 x float> %cval, zeroinitializer
%vals = call <4 x float> @llvm.masked.gather.v4f32(<4 x float*> %ptrs, i32 8, <4 x i1> %mask, <4 x float> undef) %vals = call <4 x float> @llvm.masked.gather.v4f32(<4 x float*> %ptrs, i32 8, <4 x i1> %mask, <4 x float> undef)
store <4 x float> %vals, <4 x float>* %a store <4 x float> %vals, <4 x float>* %a
ret void ret void
} }
define void @masked_gather_v8f32(<8 x float>* %a, <8 x float*>* %b) #0 { define void @masked_gather_v8f32(<8 x float>* %a, <8 x float*>* %b) #0 {
; VBITS_GE_512-LABEL: masked_gather_v8f32: ; VBITS_GE_512-LABEL: masked_gather_v8f32:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ptrue p1.d, vl8
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_512-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_512-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_512-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_512-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%cval = load <8 x float>, <8 x float>* %a %cval = load <8 x float>, <8 x float>* %a
%ptrs = load <8 x float*>, <8 x float*>* %b %ptrs = load <8 x float*>, <8 x float*>* %b
%mask = fcmp oeq <8 x float> %cval, zeroinitializer %mask = fcmp oeq <8 x float> %cval, zeroinitializer
%vals = call <8 x float> @llvm.masked.gather.v8f32(<8 x float*> %ptrs, i32 8, <8 x i1> %mask, <8 x float> undef) %vals = call <8 x float> @llvm.masked.gather.v8f32(<8 x float*> %ptrs, i32 8, <8 x i1> %mask, <8 x float> undef)
store <8 x float> %vals, <8 x float>* %a store <8 x float> %vals, <8 x float>* %a
ret void ret void
} }
define void @masked_gather_v16f32(<16 x float>* %a, <16 x float*>* %b) #0 { define void @masked_gather_v16f32(<16 x float>* %a, <16 x float*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_gather_v16f32: ; VBITS_GE_1024-LABEL: masked_gather_v16f32:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.s, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.s, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_1024-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_1024-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_1024-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_1024-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_1024-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_1024-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%cval = load <16 x float>, <16 x float>* %a %cval = load <16 x float>, <16 x float>* %a
%ptrs = load <16 x float*>, <16 x float*>* %b %ptrs = load <16 x float*>, <16 x float*>* %b
%mask = fcmp oeq <16 x float> %cval, zeroinitializer %mask = fcmp oeq <16 x float> %cval, zeroinitializer
%vals = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %ptrs, i32 8, <16 x i1> %mask, <16 x float> undef) %vals = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %ptrs, i32 8, <16 x i1> %mask, <16 x float> undef)
store <16 x float> %vals, <16 x float>* %a store <16 x float> %vals, <16 x float>* %a
ret void ret void
} }
define void @masked_gather_v32f32(<32 x float>* %a, <32 x float*>* %b) #0 { define void @masked_gather_v32f32(<32 x float>* %a, <32 x float*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_gather_v32f32: ; VBITS_GE_2048-LABEL: masked_gather_v32f32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cval = load <32 x float>, <32 x float>* %a %cval = load <32 x float>, <32 x float>* %a
%ptrs = load <32 x float*>, <32 x float*>* %b %ptrs = load <32 x float*>, <32 x float*>* %b
%mask = fcmp oeq <32 x float> %cval, zeroinitializer %mask = fcmp oeq <32 x float> %cval, zeroinitializer
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines
; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_sext_f16: ; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_sext_f16:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: ld1sw { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d, lsl #1] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d, lsl #1]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x half>, <32 x half>* %a %cvals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
Show All 10 Lines
; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_sext_f32: ; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_sext_f32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d, lsl #2] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d, lsl #2]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = sext <32 x i32> %idxs to <32 x i64> %ext = sext <32 x i32> %idxs to <32 x i64>
Show All 30 Lines
; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_zext: ; VBITS_GE_2048-LABEL: masked_gather_32b_scaled_zext:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d, lsl #1] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d, lsl #1]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x half>, <32 x half>* %a %cvals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
Show All 10 Lines
; VBITS_GE_2048-LABEL: masked_gather_32b_unscaled_sext: ; VBITS_GE_2048-LABEL: masked_gather_32b_unscaled_sext:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: ld1sw { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x half>, <32 x half>* %a %cvals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
Show All 11 Lines
; VBITS_GE_2048-LABEL: masked_gather_32b_unscaled_zext: ; VBITS_GE_2048-LABEL: masked_gather_32b_unscaled_zext:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d] ; VBITS_GE_2048-NEXT: ld1h { z0.d }, p1/z, [x2, z0.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x half>, <32 x half>* %a %cvals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
Show All 10 Lines
; VBITS_GE_2048-LABEL: masked_gather_64b_scaled: ; VBITS_GE_2048-LABEL: masked_gather_64b_scaled:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d, lsl #2] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d, lsl #2]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i64>, <32 x i64>* %b %idxs = load <32 x i64>, <32 x i64>* %b
%ptrs = getelementptr float, float* %base, <32 x i64> %idxs %ptrs = getelementptr float, float* %base, <32 x i64> %idxs
%mask = fcmp oeq <32 x float> %cvals, zeroinitializer %mask = fcmp oeq <32 x float> %cvals, zeroinitializer
%vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x float*> %ptrs, i32 8, <32 x i1> %mask, <32 x float> undef) %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x float*> %ptrs, i32 8, <32 x i1> %mask, <32 x float> undef)
store <32 x float> %vals, <32 x float>* %a store <32 x float> %vals, <32 x float>* %a
ret void ret void
} }
define void @masked_gather_64b_unscaled(<32 x float>* %a, <32 x i64>* %b, i8* %base) #0 { define void @masked_gather_64b_unscaled(<32 x float>* %a, <32 x i64>* %b, i8* %base) #0 {
; VBITS_GE_2048-LABEL: masked_gather_64b_unscaled: ; VBITS_GE_2048-LABEL: masked_gather_64b_unscaled:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [x2, z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i64>, <32 x i64>* %b %idxs = load <32 x i64>, <32 x i64>* %b
%byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %idxs %byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %idxs
Show All 11 Lines
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z2.d, x2 ; VBITS_GE_2048-NEXT: mov z2.d, x2
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d ; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%bases = load <32 x i8*>, <32 x i8*>* %b %bases = load <32 x i8*>, <32 x i8*>* %b
%byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off
Show All 11 Lines
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z2.d, #4 // =0x4 ; VBITS_GE_2048-NEXT: mov z2.d, #4 // =0x4
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d ; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%bases = load <32 x i8*>, <32 x i8*>* %b %bases = load <32 x i8*>, <32 x i8*>* %b
%byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4 %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
%mask = fcmp oeq <32 x float> %cvals, zeroinitializer %mask = fcmp oeq <32 x float> %cvals, zeroinitializer
%vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x float*> %ptrs, i32 8, <32 x i1> %mask, <32 x float> undef) %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x float*> %ptrs, i32 8, <32 x i1> %mask, <32 x float> undef)
store <32 x float> %vals, <32 x float>* %a store <32 x float> %vals, <32 x float>* %a
ret void ret void
} }
define void @masked_gather_passthru(<32 x float>* %a, <32 x float*>* %b, <32 x float>* %c) #0 { define void @masked_gather_passthru(<32 x float>* %a, <32 x float*>* %b, <32 x float>* %c) #0 {
; VBITS_GE_2048-LABEL: masked_gather_passthru: ; VBITS_GE_2048-LABEL: masked_gather_passthru:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x2] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x2]
; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_2048-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_2048-NEXT: mov z0.s, p2/m, z1.s ; VBITS_GE_2048-NEXT: mov z0.s, p2/m, z1.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
Show All 9 Lines
; VBITS_GE_2048-LABEL: masked_gather_passthru_0: ; VBITS_GE_2048-LABEL: masked_gather_passthru_0:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p2.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z0.s, p2/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0 ; VBITS_GE_2048-NEXT: cmpne p1.d, p1/z, z0.d, #0
; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d] ; VBITS_GE_2048-NEXT: ld1w { z0.d }, p1/z, [z1.d]
; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%cvals = load <32 x float>, <32 x float>* %a %cvals = load <32 x float>, <32 x float>* %a
%ptrs = load <32 x float*>, <32 x float*>* %b %ptrs = load <32 x float*>, <32 x float*>* %b
%mask = fcmp oeq <32 x float> %cvals, zeroinitializer %mask = fcmp oeq <32 x float> %cvals, zeroinitializer
▲ Show 20 Lines • Show All 50 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll

Show All 29 Lines
; CHECK-NEXT: ldrb w8, [x0] ; CHECK-NEXT: ldrb w8, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q2, [x1] ; CHECK-NEXT: ldr q2, [x1]
; CHECK-NEXT: fmov s0, w8 ; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: ldrb w8, [x0, #1] ; CHECK-NEXT: ldrb w8, [x0, #1]
; CHECK-NEXT: mov v0.s[1], w8 ; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: cmeq v1.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v1.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0
; CHECK-NEXT: st1b { z0.d }, p0, [z2.d] ; CHECK-NEXT: st1b { z0.d }, p0, [z2.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <2 x i8>, <2 x i8>* %a %vals = load <2 x i8>, <2 x i8>* %a
%ptrs = load <2 x i8*>, <2 x i8*>* %b %ptrs = load <2 x i8*>, <2 x i8*>* %b
%mask = icmp eq <2 x i8> %vals, zeroinitializer %mask = icmp eq <2 x i8> %vals, zeroinitializer
call void @llvm.masked.scatter.v2i8(<2 x i8> %vals, <2 x i8*> %ptrs, i32 8, <2 x i1> %mask) call void @llvm.masked.scatter.v2i8(<2 x i8> %vals, <2 x i8*> %ptrs, i32 8, <2 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 { define void @masked_scatter_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v4i8: ; CHECK-LABEL: masked_scatter_v4i8:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr s0, [x0] ; CHECK-NEXT: ldr s0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: cmeq v2.4h, v0.4h, #0 ; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: uunpklo z2.s, z2.h ; CHECK-NEXT: sunpklo z2.s, z2.h
; CHECK-NEXT: uunpklo z2.d, z2.s ; CHECK-NEXT: sunpklo z2.d, z2.s
; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0
; CHECK-NEXT: st1b { z0.d }, p0, [z1.d] ; CHECK-NEXT: st1b { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <4 x i8>, <4 x i8>* %a %vals = load <4 x i8>, <4 x i8>* %a
%ptrs = load <4 x i8*>, <4 x i8*>* %b %ptrs = load <4 x i8*>, <4 x i8*>* %b
%mask = icmp eq <4 x i8> %vals, zeroinitializer %mask = icmp eq <4 x i8> %vals, zeroinitializer
call void @llvm.masked.scatter.v4i8(<4 x i8> %vals, <4 x i8*> %ptrs, i32 8, <4 x i1> %mask) call void @llvm.masked.scatter.v4i8(<4 x i8> %vals, <4 x i8*> %ptrs, i32 8, <4 x i1> %mask)
ret void ret void
Show All 14 Lines
; VBITS_EQ_256-NEXT: zip2 v1.8b, v1.8b, v0.8b ; VBITS_EQ_256-NEXT: zip2 v1.8b, v1.8b, v0.8b
; VBITS_EQ_256-NEXT: zip2 v0.8b, v0.8b, v0.8b ; VBITS_EQ_256-NEXT: zip2 v0.8b, v0.8b, v0.8b
; VBITS_EQ_256-NEXT: shl v2.4h, v2.4h, #8 ; VBITS_EQ_256-NEXT: shl v2.4h, v2.4h, #8
; VBITS_EQ_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_EQ_256-NEXT: shl v1.4h, v1.4h, #8
; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h ; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: sshr v2.4h, v2.4h, #8 ; VBITS_EQ_256-NEXT: sshr v2.4h, v2.4h, #8
; VBITS_EQ_256-NEXT: sshr v1.4h, v1.4h, #8 ; VBITS_EQ_256-NEXT: sshr v1.4h, v1.4h, #8
; VBITS_EQ_256-NEXT: uunpklo z2.s, z2.h ; VBITS_EQ_256-NEXT: sunpklo z2.s, z2.h
; VBITS_EQ_256-NEXT: uunpklo z1.s, z1.h ; VBITS_EQ_256-NEXT: sunpklo z1.s, z1.h
; VBITS_EQ_256-NEXT: uunpklo z2.d, z2.s ; VBITS_EQ_256-NEXT: sunpklo z2.d, z2.s
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0
; VBITS_EQ_256-NEXT: uunpklo z1.s, z5.h ; VBITS_EQ_256-NEXT: uunpklo z1.s, z5.h
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: st1b { z1.d }, p1, [z4.d] ; VBITS_EQ_256-NEXT: st1b { z1.d }, p1, [z4.d]
; VBITS_EQ_256-NEXT: st1b { z0.d }, p0, [z3.d] ; VBITS_EQ_256-NEXT: st1b { z0.d }, p0, [z3.d]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_scatter_v8i8: ; VBITS_GE_512-LABEL: masked_scatter_v8i8:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr d0, [x0] ; VBITS_GE_512-NEXT: ldr d0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0 ; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0
; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: uunpklo z2.h, z2.b ; VBITS_GE_512-NEXT: sunpklo z2.h, z2.b
; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_512-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_512-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0
; VBITS_GE_512-NEXT: st1b { z0.d }, p0, [z1.d] ; VBITS_GE_512-NEXT: st1b { z0.d }, p0, [z1.d]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%vals = load <8 x i8>, <8 x i8>* %a %vals = load <8 x i8>, <8 x i8>* %a
%ptrs = load <8 x i8*>, <8 x i8*>* %b %ptrs = load <8 x i8*>, <8 x i8*>* %b
%mask = icmp eq <8 x i8> %vals, zeroinitializer %mask = icmp eq <8 x i8> %vals, zeroinitializer
call void @llvm.masked.scatter.v8i8(<8 x i8> %vals, <8 x i8*> %ptrs, i32 8, <8 x i1> %mask) call void @llvm.masked.scatter.v8i8(<8 x i8> %vals, <8 x i8*> %ptrs, i32 8, <8 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v16i8(<16 x i8>* %a, <16 x i8*>* %b) #0 { define void @masked_scatter_v16i8(<16 x i8>* %a, <16 x i8*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_scatter_v16i8: ; VBITS_GE_1024-LABEL: masked_scatter_v16i8:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ldr q0, [x0] ; VBITS_GE_1024-NEXT: ldr q0, [x0]
; VBITS_GE_1024-NEXT: ptrue p0.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.d, vl16
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_1024-NEXT: cmeq v2.16b, v0.16b, #0 ; VBITS_GE_1024-NEXT: cmeq v2.16b, v0.16b, #0
; VBITS_GE_1024-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_1024-NEXT: uunpklo z0.h, z0.b
; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: uunpklo z2.h, z2.b ; VBITS_GE_1024-NEXT: sunpklo z2.h, z2.b
; VBITS_GE_1024-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_1024-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_1024-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_1024-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p0/z, z2.d, #0
; VBITS_GE_1024-NEXT: st1b { z0.d }, p0, [z1.d] ; VBITS_GE_1024-NEXT: st1b { z0.d }, p0, [z1.d]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%vals = load <16 x i8>, <16 x i8>* %a %vals = load <16 x i8>, <16 x i8>* %a
%ptrs = load <16 x i8*>, <16 x i8*>* %b %ptrs = load <16 x i8*>, <16 x i8*>* %b
%mask = icmp eq <16 x i8> %vals, zeroinitializer %mask = icmp eq <16 x i8> %vals, zeroinitializer
call void @llvm.masked.scatter.v16i8(<16 x i8> %vals, <16 x i8*> %ptrs, i32 8, <16 x i1> %mask) call void @llvm.masked.scatter.v16i8(<16 x i8> %vals, <16 x i8*> %ptrs, i32 8, <16 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v32i8(<32 x i8>* %a, <32 x i8*>* %b) #0 { define void @masked_scatter_v32i8(<32 x i8>* %a, <32 x i8*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_v32i8: ; VBITS_GE_2048-LABEL: masked_scatter_v32i8:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.b, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.b, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1b { z0.b }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1b { z0.b }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z2.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z2.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: cmpeq p0.b, p0/z, z0.b, #0 ; VBITS_GE_2048-NEXT: cmpeq p0.b, p0/z, z0.b, #0
; VBITS_GE_2048-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_2048-NEXT: uunpklo z0.h, z0.b
; VBITS_GE_2048-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: uunpklo z1.h, z1.b ; VBITS_GE_2048-NEXT: sunpklo z1.h, z1.b
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_2048-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT: uunpklo z1.d, z1.s ; VBITS_GE_2048-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z1.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z1.d, #0
; VBITS_GE_2048-NEXT: st1b { z0.d }, p0, [z2.d] ; VBITS_GE_2048-NEXT: st1b { z0.d }, p0, [z2.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x i8>, <32 x i8>* %a %vals = load <32 x i8>, <32 x i8>* %a
%ptrs = load <32 x i8*>, <32 x i8*>* %b %ptrs = load <32 x i8*>, <32 x i8*>* %b
%mask = icmp eq <32 x i8> %vals, zeroinitializer %mask = icmp eq <32 x i8> %vals, zeroinitializer
call void @llvm.masked.scatter.v32i8(<32 x i8> %vals, <32 x i8*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32i8(<32 x i8> %vals, <32 x i8*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
Show All 9 Lines
; CHECK-NEXT: ldrh w8, [x0] ; CHECK-NEXT: ldrh w8, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q2, [x1] ; CHECK-NEXT: ldr q2, [x1]
; CHECK-NEXT: fmov s0, w8 ; CHECK-NEXT: fmov s0, w8
; CHECK-NEXT: ldrh w8, [x0, #2] ; CHECK-NEXT: ldrh w8, [x0, #2]
; CHECK-NEXT: mov v0.s[1], w8 ; CHECK-NEXT: mov v0.s[1], w8
; CHECK-NEXT: cmeq v1.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v1.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0
; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] ; CHECK-NEXT: st1h { z0.d }, p0, [z2.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <2 x i16>, <2 x i16>* %a %vals = load <2 x i16>, <2 x i16>* %a
%ptrs = load <2 x i16*>, <2 x i16*>* %b %ptrs = load <2 x i16*>, <2 x i16*>* %b
%mask = icmp eq <2 x i16> %vals, zeroinitializer %mask = icmp eq <2 x i16> %vals, zeroinitializer
call void @llvm.masked.scatter.v2i16(<2 x i16> %vals, <2 x i16*> %ptrs, i32 8, <2 x i1> %mask) call void @llvm.masked.scatter.v2i16(<2 x i16> %vals, <2 x i16*> %ptrs, i32 8, <2 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v4i16(<4 x i16>* %a, <4 x i16*>* %b) #0 { define void @masked_scatter_v4i16(<4 x i16>* %a, <4 x i16*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v4i16: ; CHECK-LABEL: masked_scatter_v4i16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: cmeq v2.4h, v0.4h, #0 ; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: uunpklo z2.s, z2.h ; CHECK-NEXT: sunpklo z2.s, z2.h
; CHECK-NEXT: uunpklo z2.d, z2.s ; CHECK-NEXT: sunpklo z2.d, z2.s
; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0
; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <4 x i16>, <4 x i16>* %a %vals = load <4 x i16>, <4 x i16>* %a
%ptrs = load <4 x i16*>, <4 x i16*>* %b %ptrs = load <4 x i16*>, <4 x i16*>* %b
%mask = icmp eq <4 x i16> %vals, zeroinitializer %mask = icmp eq <4 x i16> %vals, zeroinitializer
call void @llvm.masked.scatter.v4i16(<4 x i16> %vals, <4 x i16*> %ptrs, i32 8, <4 x i1> %mask) call void @llvm.masked.scatter.v4i16(<4 x i16> %vals, <4 x i16*> %ptrs, i32 8, <4 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v8i16(<8 x i16>* %a, <8 x i16*>* %b) #0 { define void @masked_scatter_v8i16(<8 x i16>* %a, <8 x i16*>* %b) #0 {
; Ensure sensible type legalisation. ; Ensure sensible type legalisation.
; VBITS_EQ_256-LABEL: masked_scatter_v8i16: ; VBITS_EQ_256-LABEL: masked_scatter_v8i16:
; VBITS_EQ_256: // %bb.0: ; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: ldr q0, [x0] ; VBITS_EQ_256-NEXT: ldr q0, [x0]
; VBITS_EQ_256-NEXT: mov x8, #4 ; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ptrue p0.d, vl4 ; VBITS_EQ_256-NEXT: ptrue p0.d, vl4
; VBITS_EQ_256-NEXT: cmeq v1.8h, v0.8h, #0 ; VBITS_EQ_256-NEXT: cmeq v1.8h, v0.8h, #0
; VBITS_EQ_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_EQ_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ext v3.16b, v0.16b, v0.16b, #8 ; VBITS_EQ_256-NEXT: ext v3.16b, v0.16b, v0.16b, #8
; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h ; VBITS_EQ_256-NEXT: uunpklo z0.s, z0.h
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: uunpklo z2.s, z1.h ; VBITS_EQ_256-NEXT: sunpklo z2.s, z1.h
; VBITS_EQ_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8 ; VBITS_EQ_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; VBITS_EQ_256-NEXT: uunpklo z2.d, z2.s ; VBITS_EQ_256-NEXT: sunpklo z2.d, z2.s
; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_EQ_256-NEXT: ld1d { z2.d }, p0/z, [x1]
; VBITS_EQ_256-NEXT: uunpklo z3.s, z3.h ; VBITS_EQ_256-NEXT: uunpklo z3.s, z3.h
; VBITS_EQ_256-NEXT: uunpklo z1.s, z1.h ; VBITS_EQ_256-NEXT: sunpklo z1.s, z1.h
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: st1h { z0.d }, p1, [z2.d] ; VBITS_EQ_256-NEXT: st1h { z0.d }, p1, [z2.d]
; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; VBITS_EQ_256-NEXT: cmpne p0.d, p0/z, z1.d, #0
; VBITS_EQ_256-NEXT: uunpklo z1.d, z3.s ; VBITS_EQ_256-NEXT: uunpklo z1.d, z3.s
; VBITS_EQ_256-NEXT: st1h { z1.d }, p0, [z4.d] ; VBITS_EQ_256-NEXT: st1h { z1.d }, p0, [z4.d]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_scatter_v8i16: ; VBITS_GE_512-LABEL: masked_scatter_v8i16:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr q0, [x0] ; VBITS_GE_512-NEXT: ldr q0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0 ; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_512-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_512-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0
; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%vals = load <8 x i16>, <8 x i16>* %a %vals = load <8 x i16>, <8 x i16>* %a
%ptrs = load <8 x i16*>, <8 x i16*>* %b %ptrs = load <8 x i16*>, <8 x i16*>* %b
%mask = icmp eq <8 x i16> %vals, zeroinitializer %mask = icmp eq <8 x i16> %vals, zeroinitializer
call void @llvm.masked.scatter.v8i16(<8 x i16> %vals, <8 x i16*> %ptrs, i32 8, <8 x i1> %mask) call void @llvm.masked.scatter.v8i16(<8 x i16> %vals, <8 x i16*> %ptrs, i32 8, <8 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v16i16(<16 x i16>* %a, <16 x i16*>* %b) #0 { define void @masked_scatter_v16i16(<16 x i16>* %a, <16 x i16*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_scatter_v16i16: ; VBITS_GE_1024-LABEL: masked_scatter_v16i16:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.h, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.h, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; VBITS_GE_1024-NEXT: cmpeq p0.h, p0/z, z0.h, #0
; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_1024-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_1024-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_1024-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_1024-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_1024-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_1024-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%vals = load <16 x i16>, <16 x i16>* %a %vals = load <16 x i16>, <16 x i16>* %a
%ptrs = load <16 x i16*>, <16 x i16*>* %b %ptrs = load <16 x i16*>, <16 x i16*>* %b
%mask = icmp eq <16 x i16> %vals, zeroinitializer %mask = icmp eq <16 x i16> %vals, zeroinitializer
call void @llvm.masked.scatter.v16i16(<16 x i16> %vals, <16 x i16*> %ptrs, i32 8, <16 x i1> %mask) call void @llvm.masked.scatter.v16i16(<16 x i16> %vals, <16 x i16*> %ptrs, i32 8, <16 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v32i16(<32 x i16>* %a, <32 x i16*>* %b) #0 { define void @masked_scatter_v32i16(<32 x i16>* %a, <32 x i16*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_v32i16: ; VBITS_GE_2048-LABEL: masked_scatter_v32i16:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; VBITS_GE_2048-NEXT: cmpeq p0.h, p0/z, z0.h, #0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x i16>, <32 x i16>* %a %vals = load <32 x i16>, <32 x i16>* %a
%ptrs = load <32 x i16*>, <32 x i16*>* %b %ptrs = load <32 x i16*>, <32 x i16*>* %b
%mask = icmp eq <32 x i16> %vals, zeroinitializer %mask = icmp eq <32 x i16> %vals, zeroinitializer
call void @llvm.masked.scatter.v32i16(<32 x i16> %vals, <32 x i16*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32i16(<32 x i16> %vals, <32 x i16*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
; ;
; ST1W ; ST1W
; ;
define void @masked_scatter_v2i32(<2 x i32>* %a, <2 x i32*>* %b) #0 { define void @masked_scatter_v2i32(<2 x i32>* %a, <2 x i32*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v2i32: ; CHECK-LABEL: masked_scatter_v2i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q2, [x1] ; CHECK-NEXT: ldr q2, [x1]
; CHECK-NEXT: cmeq v1.2s, v0.2s, #0 ; CHECK-NEXT: cmeq v1.2s, v0.2s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0
; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z2.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <2 x i32>, <2 x i32>* %a %vals = load <2 x i32>, <2 x i32>* %a
%ptrs = load <2 x i32*>, <2 x i32*>* %b %ptrs = load <2 x i32*>, <2 x i32*>* %b
%mask = icmp eq <2 x i32> %vals, zeroinitializer %mask = icmp eq <2 x i32> %vals, zeroinitializer
call void @llvm.masked.scatter.v2i32(<2 x i32> %vals, <2 x i32*> %ptrs, i32 8, <2 x i1> %mask) call void @llvm.masked.scatter.v2i32(<2 x i32> %vals, <2 x i32*> %ptrs, i32 8, <2 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v4i32(<4 x i32>* %a, <4 x i32*>* %b) #0 { define void @masked_scatter_v4i32(<4 x i32>* %a, <4 x i32*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v4i32: ; CHECK-LABEL: masked_scatter_v4i32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: cmeq v2.4s, v0.4s, #0 ; CHECK-NEXT: cmeq v2.4s, v0.4s, #0
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: uunpklo z2.d, z2.s ; CHECK-NEXT: sunpklo z2.d, z2.s
; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0
; CHECK-NEXT: st1w { z0.d }, p0, [z1.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <4 x i32>, <4 x i32>* %a %vals = load <4 x i32>, <4 x i32>* %a
%ptrs = load <4 x i32*>, <4 x i32*>* %b %ptrs = load <4 x i32*>, <4 x i32*>* %b
%mask = icmp eq <4 x i32> %vals, zeroinitializer %mask = icmp eq <4 x i32> %vals, zeroinitializer
call void @llvm.masked.scatter.v4i32(<4 x i32> %vals, <4 x i32*> %ptrs, i32 8, <4 x i1> %mask) call void @llvm.masked.scatter.v4i32(<4 x i32> %vals, <4 x i32*> %ptrs, i32 8, <4 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v8i32(<8 x i32>* %a, <8 x i32*>* %b) #0 { define void @masked_scatter_v8i32(<8 x i32>* %a, <8 x i32*>* %b) #0 {
; Ensure sensible type legalisation. ; Ensure sensible type legalisation.
; VBITS_EQ_256-LABEL: masked_scatter_v8i32: ; VBITS_EQ_256-LABEL: masked_scatter_v8i32:
; VBITS_EQ_256: // %bb.0: ; VBITS_EQ_256: // %bb.0:
; VBITS_EQ_256-NEXT: ptrue p0.s, vl8 ; VBITS_EQ_256-NEXT: ptrue p0.s, vl8
; VBITS_EQ_256-NEXT: mov x8, #4 ; VBITS_EQ_256-NEXT: mov x8, #4
; VBITS_EQ_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_EQ_256-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_EQ_256-NEXT: ptrue p1.d, vl4 ; VBITS_EQ_256-NEXT: ptrue p1.d, vl4
; VBITS_EQ_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3] ; VBITS_EQ_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3]
; VBITS_EQ_256-NEXT: ld1d { z4.d }, p1/z, [x1] ; VBITS_EQ_256-NEXT: ld1d { z4.d }, p1/z, [x1]
; VBITS_EQ_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_EQ_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0
; VBITS_EQ_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_EQ_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_EQ_256-NEXT: uunpklo z3.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z3.d, z1.s
; VBITS_EQ_256-NEXT: ext z1.b, z1.b, z1.b, #16 ; VBITS_EQ_256-NEXT: ext z1.b, z1.b, z1.b, #16
; VBITS_EQ_256-NEXT: cmpne p0.d, p1/z, z3.d, #0 ; VBITS_EQ_256-NEXT: cmpne p0.d, p1/z, z3.d, #0
; VBITS_EQ_256-NEXT: uunpklo z3.d, z0.s ; VBITS_EQ_256-NEXT: uunpklo z3.d, z0.s
; VBITS_EQ_256-NEXT: uunpklo z1.d, z1.s ; VBITS_EQ_256-NEXT: sunpklo z1.d, z1.s
; VBITS_EQ_256-NEXT: ext z0.b, z0.b, z0.b, #16 ; VBITS_EQ_256-NEXT: ext z0.b, z0.b, z0.b, #16
; VBITS_EQ_256-NEXT: cmpne p1.d, p1/z, z1.d, #0 ; VBITS_EQ_256-NEXT: cmpne p1.d, p1/z, z1.d, #0
; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s ; VBITS_EQ_256-NEXT: uunpklo z0.d, z0.s
; VBITS_EQ_256-NEXT: st1w { z3.d }, p0, [z4.d] ; VBITS_EQ_256-NEXT: st1w { z3.d }, p0, [z4.d]
; VBITS_EQ_256-NEXT: st1w { z0.d }, p1, [z2.d] ; VBITS_EQ_256-NEXT: st1w { z0.d }, p1, [z2.d]
; VBITS_EQ_256-NEXT: ret ; VBITS_EQ_256-NEXT: ret
; ;
; VBITS_GE_512-LABEL: masked_scatter_v8i32: ; VBITS_GE_512-LABEL: masked_scatter_v8i32:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ptrue p1.d, vl8
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_GE_512-NEXT: cmpeq p0.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_512-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_512-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_512-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_512-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_512-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%vals = load <8 x i32>, <8 x i32>* %a %vals = load <8 x i32>, <8 x i32>* %a
%ptrs = load <8 x i32*>, <8 x i32*>* %b %ptrs = load <8 x i32*>, <8 x i32*>* %b
%mask = icmp eq <8 x i32> %vals, zeroinitializer %mask = icmp eq <8 x i32> %vals, zeroinitializer
call void @llvm.masked.scatter.v8i32(<8 x i32> %vals, <8 x i32*> %ptrs, i32 8, <8 x i1> %mask) call void @llvm.masked.scatter.v8i32(<8 x i32> %vals, <8 x i32*> %ptrs, i32 8, <8 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v16i32(<16 x i32>* %a, <16 x i32*>* %b) #0 { define void @masked_scatter_v16i32(<16 x i32>* %a, <16 x i32*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_scatter_v16i32: ; VBITS_GE_1024-LABEL: masked_scatter_v16i32:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.s, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.s, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_GE_1024-NEXT: cmpeq p0.s, p0/z, z0.s, #0
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_1024-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_1024-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_1024-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%vals = load <16 x i32>, <16 x i32>* %a %vals = load <16 x i32>, <16 x i32>* %a
%ptrs = load <16 x i32*>, <16 x i32*>* %b %ptrs = load <16 x i32*>, <16 x i32*>* %b
%mask = icmp eq <16 x i32> %vals, zeroinitializer %mask = icmp eq <16 x i32> %vals, zeroinitializer
call void @llvm.masked.scatter.v16i32(<16 x i32> %vals, <16 x i32*> %ptrs, i32 8, <16 x i1> %mask) call void @llvm.masked.scatter.v16i32(<16 x i32> %vals, <16 x i32*> %ptrs, i32 8, <16 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v32i32(<32 x i32>* %a, <32 x i32*>* %b) #0 { define void @masked_scatter_v32i32(<32 x i32>* %a, <32 x i32*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_v32i32: ; VBITS_GE_2048-LABEL: masked_scatter_v32i32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_GE_2048-NEXT: cmpeq p0.s, p0/z, z0.s, #0
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x i32>, <32 x i32>* %a %vals = load <32 x i32>, <32 x i32>* %a
%ptrs = load <32 x i32*>, <32 x i32*>* %b %ptrs = load <32 x i32*>, <32 x i32*>* %b
%mask = icmp eq <32 x i32> %vals, zeroinitializer %mask = icmp eq <32 x i32> %vals, zeroinitializer
call void @llvm.masked.scatter.v32i32(<32 x i32> %vals, <32 x i32*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32i32(<32 x i32> %vals, <32 x i32*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
▲ Show 20 Lines • Show All 139 Lines • ▼ Show 20 Lines
; CHECK-NEXT: sshr v2.2s, v2.2s, #16 ; CHECK-NEXT: sshr v2.2s, v2.2s, #16
; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: fmov w8, s2
; CHECK-NEXT: mov w9, v2.s[1] ; CHECK-NEXT: mov w9, v2.s[1]
; CHECK-NEXT: ldr q2, [x1] ; CHECK-NEXT: ldr q2, [x1]
; CHECK-NEXT: mov v0.h[0], w8 ; CHECK-NEXT: mov v0.h[0], w8
; CHECK-NEXT: mov v0.h[1], w9 ; CHECK-NEXT: mov v0.h[1], w9
; CHECK-NEXT: shl v0.4h, v0.4h, #15 ; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: sshr v0.4h, v0.4h, #15 ; CHECK-NEXT: sshr v0.4h, v0.4h, #15
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z0.d, z0.s
; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
; CHECK-NEXT: uunpklo z0.d, z1.s ; CHECK-NEXT: uunpklo z0.d, z1.s
; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] ; CHECK-NEXT: st1h { z0.d }, p0, [z2.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <2 x half>, <2 x half>* %a %vals = load <2 x half>, <2 x half>* %a
%ptrs = load <2 x half*>, <2 x half*>* %b %ptrs = load <2 x half*>, <2 x half*>* %b
%mask = fcmp oeq <2 x half> %vals, zeroinitializer %mask = fcmp oeq <2 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v2f16(<2 x half> %vals, <2 x half*> %ptrs, i32 8, <2 x i1> %mask) call void @llvm.masked.scatter.v2f16(<2 x half> %vals, <2 x half*> %ptrs, i32 8, <2 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v4f16(<4 x half>* %a, <4 x half*>* %b) #0 { define void @masked_scatter_v4f16(<4 x half>* %a, <4 x half*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v4f16: ; CHECK-LABEL: masked_scatter_v4f16:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: fcmeq v2.4h, v0.4h, #0.0 ; CHECK-NEXT: fcmeq v2.4h, v0.4h, #0.0
; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: uunpklo z2.s, z2.h ; CHECK-NEXT: sunpklo z2.s, z2.h
; CHECK-NEXT: uunpklo z2.d, z2.s ; CHECK-NEXT: sunpklo z2.d, z2.s
; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0
; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <4 x half>, <4 x half>* %a %vals = load <4 x half>, <4 x half>* %a
%ptrs = load <4 x half*>, <4 x half*>* %b %ptrs = load <4 x half*>, <4 x half*>* %b
%mask = fcmp oeq <4 x half> %vals, zeroinitializer %mask = fcmp oeq <4 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v4f16(<4 x half> %vals, <4 x half*> %ptrs, i32 8, <4 x i1> %mask) call void @llvm.masked.scatter.v4f16(<4 x half> %vals, <4 x half*> %ptrs, i32 8, <4 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v8f16(<8 x half>* %a, <8 x half*>* %b) #0 { define void @masked_scatter_v8f16(<8 x half>* %a, <8 x half*>* %b) #0 {
; VBITS_GE_512-LABEL: masked_scatter_v8f16: ; VBITS_GE_512-LABEL: masked_scatter_v8f16:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ldr q0, [x0] ; VBITS_GE_512-NEXT: ldr q0, [x0]
; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: fcmeq v2.8h, v0.8h, #0.0 ; VBITS_GE_512-NEXT: fcmeq v2.8h, v0.8h, #0.0
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_512-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_512-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z2.d, #0
; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%vals = load <8 x half>, <8 x half>* %a %vals = load <8 x half>, <8 x half>* %a
%ptrs = load <8 x half*>, <8 x half*>* %b %ptrs = load <8 x half*>, <8 x half*>* %b
%mask = fcmp oeq <8 x half> %vals, zeroinitializer %mask = fcmp oeq <8 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v8f16(<8 x half> %vals, <8 x half*> %ptrs, i32 8, <8 x i1> %mask) call void @llvm.masked.scatter.v8f16(<8 x half> %vals, <8 x half*> %ptrs, i32 8, <8 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v16f16(<16 x half>* %a, <16 x half*>* %b) #0 { define void @masked_scatter_v16f16(<16 x half>* %a, <16 x half*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_scatter_v16f16: ; VBITS_GE_1024-LABEL: masked_scatter_v16f16:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.h, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.h, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_1024-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_1024-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_1024-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_1024-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_1024-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_1024-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_1024-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_1024-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%vals = load <16 x half>, <16 x half>* %a %vals = load <16 x half>, <16 x half>* %a
%ptrs = load <16 x half*>, <16 x half*>* %b %ptrs = load <16 x half*>, <16 x half*>* %b
%mask = fcmp oeq <16 x half> %vals, zeroinitializer %mask = fcmp oeq <16 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v16f16(<16 x half> %vals, <16 x half*> %ptrs, i32 8, <16 x i1> %mask) call void @llvm.masked.scatter.v16f16(<16 x half> %vals, <16 x half*> %ptrs, i32 8, <16 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v32f16(<32 x half>* %a, <32 x half*>* %b) #0 { define void @masked_scatter_v32f16(<32 x half>* %a, <32 x half*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_v32f16: ; VBITS_GE_2048-LABEL: masked_scatter_v32f16:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x half>, <32 x half>* %a %vals = load <32 x half>, <32 x half>* %a
%ptrs = load <32 x half*>, <32 x half*>* %b %ptrs = load <32 x half*>, <32 x half*>* %b
%mask = fcmp oeq <32 x half> %vals, zeroinitializer %mask = fcmp oeq <32 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
; ;
; ST1W (float) ; ST1W (float)
; ;
define void @masked_scatter_v2f32(<2 x float>* %a, <2 x float*>* %b) #0 { define void @masked_scatter_v2f32(<2 x float>* %a, <2 x float*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v2f32: ; CHECK-LABEL: masked_scatter_v2f32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q2, [x1] ; CHECK-NEXT: ldr q2, [x1]
; CHECK-NEXT: fcmeq v1.2s, v0.2s, #0.0 ; CHECK-NEXT: fcmeq v1.2s, v0.2s, #0.0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0
; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z2.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <2 x float>, <2 x float>* %a %vals = load <2 x float>, <2 x float>* %a
%ptrs = load <2 x float*>, <2 x float*>* %b %ptrs = load <2 x float*>, <2 x float*>* %b
%mask = fcmp oeq <2 x float> %vals, zeroinitializer %mask = fcmp oeq <2 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v2f32(<2 x float> %vals, <2 x float*> %ptrs, i32 8, <2 x i1> %mask) call void @llvm.masked.scatter.v2f32(<2 x float> %vals, <2 x float*> %ptrs, i32 8, <2 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v4f32(<4 x float>* %a, <4 x float*>* %b) #0 { define void @masked_scatter_v4f32(<4 x float>* %a, <4 x float*>* %b) #0 {
; CHECK-LABEL: masked_scatter_v4f32: ; CHECK-LABEL: masked_scatter_v4f32:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: fcmeq v2.4s, v0.4s, #0.0 ; CHECK-NEXT: fcmeq v2.4s, v0.4s, #0.0
; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: uunpklo z2.d, z2.s ; CHECK-NEXT: sunpklo z2.d, z2.s
; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z2.d, #0
; CHECK-NEXT: st1w { z0.d }, p0, [z1.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%vals = load <4 x float>, <4 x float>* %a %vals = load <4 x float>, <4 x float>* %a
%ptrs = load <4 x float*>, <4 x float*>* %b %ptrs = load <4 x float*>, <4 x float*>* %b
%mask = fcmp oeq <4 x float> %vals, zeroinitializer %mask = fcmp oeq <4 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v4f32(<4 x float> %vals, <4 x float*> %ptrs, i32 8, <4 x i1> %mask) call void @llvm.masked.scatter.v4f32(<4 x float> %vals, <4 x float*> %ptrs, i32 8, <4 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v8f32(<8 x float>* %a, <8 x float*>* %b) #0 { define void @masked_scatter_v8f32(<8 x float>* %a, <8 x float*>* %b) #0 {
; VBITS_GE_512-LABEL: masked_scatter_v8f32: ; VBITS_GE_512-LABEL: masked_scatter_v8f32:
; VBITS_GE_512: // %bb.0: ; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ptrue p1.d, vl8
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_512-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_512-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_512-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_512-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_512-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_512-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_512-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_512-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_512-NEXT: ret ; VBITS_GE_512-NEXT: ret
%vals = load <8 x float>, <8 x float>* %a %vals = load <8 x float>, <8 x float>* %a
%ptrs = load <8 x float*>, <8 x float*>* %b %ptrs = load <8 x float*>, <8 x float*>* %b
%mask = fcmp oeq <8 x float> %vals, zeroinitializer %mask = fcmp oeq <8 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v8f32(<8 x float> %vals, <8 x float*> %ptrs, i32 8, <8 x i1> %mask) call void @llvm.masked.scatter.v8f32(<8 x float> %vals, <8 x float*> %ptrs, i32 8, <8 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v16f32(<16 x float>* %a, <16 x float*>* %b) #0 { define void @masked_scatter_v16f32(<16 x float>* %a, <16 x float*>* %b) #0 {
; VBITS_GE_1024-LABEL: masked_scatter_v16f32: ; VBITS_GE_1024-LABEL: masked_scatter_v16f32:
; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024: // %bb.0:
; VBITS_GE_1024-NEXT: ptrue p0.s, vl16 ; VBITS_GE_1024-NEXT: ptrue p0.s, vl16
; VBITS_GE_1024-NEXT: ptrue p1.d, vl16 ; VBITS_GE_1024-NEXT: ptrue p1.d, vl16
; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_1024-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_1024-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_1024-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_1024-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_1024-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_1024-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_1024-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_1024-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_1024-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_1024-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_1024-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_1024-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_1024-NEXT: ret ; VBITS_GE_1024-NEXT: ret
%vals = load <16 x float>, <16 x float>* %a %vals = load <16 x float>, <16 x float>* %a
%ptrs = load <16 x float*>, <16 x float*>* %b %ptrs = load <16 x float*>, <16 x float*>* %b
%mask = fcmp oeq <16 x float> %vals, zeroinitializer %mask = fcmp oeq <16 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v16f32(<16 x float> %vals, <16 x float*> %ptrs, i32 8, <16 x i1> %mask) call void @llvm.masked.scatter.v16f32(<16 x float> %vals, <16 x float*> %ptrs, i32 8, <16 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_v32f32(<32 x float>* %a, <32 x float*>* %b) #0 { define void @masked_scatter_v32f32(<32 x float>* %a, <32 x float*>* %b) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_v32f32: ; VBITS_GE_2048-LABEL: masked_scatter_v32f32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%ptrs = load <32 x float*>, <32 x float*>* %b %ptrs = load <32 x float*>, <32 x float*>* %b
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f32(<32 x float> %vals, <32 x float*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f32(<32 x float> %vals, <32 x float*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d, lsl #1] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d, lsl #1]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x half>, <32 x half>* %a %vals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = sext <32 x i32> %idxs to <32 x i64> %ext = sext <32 x i32> %idxs to <32 x i64>
%ptrs = getelementptr half, half* %base, <32 x i64> %ext %ptrs = getelementptr half, half* %base, <32 x i64> %ext
%mask = fcmp oeq <32 x half> %vals, zeroinitializer %mask = fcmp oeq <32 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
; NOTE: This produces an non-optimal addressing mode due to a temporary workaround ; NOTE: This produces an non-optimal addressing mode due to a temporary workaround
define void @masked_scatter_32b_scaled_sext_f32(<32 x float>* %a, <32 x i32>* %b, float* %base) #0 { define void @masked_scatter_32b_scaled_sext_f32(<32 x float>* %a, <32 x i32>* %b, float* %base) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_32b_scaled_sext_f32: ; VBITS_GE_2048-LABEL: masked_scatter_32b_scaled_sext_f32:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d, lsl #2] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d, lsl #2]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = sext <32 x i32> %idxs to <32 x i64> %ext = sext <32 x i32> %idxs to <32 x i64>
%ptrs = getelementptr float, float* %base, <32 x i64> %ext %ptrs = getelementptr float, float* %base, <32 x i64> %ext
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
Show All 27 Lines
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d, lsl #1] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d, lsl #1]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x half>, <32 x half>* %a %vals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = zext <32 x i32> %idxs to <32 x i64> %ext = zext <32 x i32> %idxs to <32 x i64>
%ptrs = getelementptr half, half* %base, <32 x i64> %ext %ptrs = getelementptr half, half* %base, <32 x i64> %ext
%mask = fcmp oeq <32 x half> %vals, zeroinitializer %mask = fcmp oeq <32 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
; NOTE: This produces an non-optimal addressing mode due to a temporary workaround ; NOTE: This produces an non-optimal addressing mode due to a temporary workaround
define void @masked_scatter_32b_unscaled_sext(<32 x half>* %a, <32 x i32>* %b, i8* %base) #0 { define void @masked_scatter_32b_unscaled_sext(<32 x half>* %a, <32 x i32>* %b, i8* %base) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_32b_unscaled_sext: ; VBITS_GE_2048-LABEL: masked_scatter_32b_unscaled_sext:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1sw { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x half>, <32 x half>* %a %vals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = sext <32 x i32> %idxs to <32 x i64> %ext = sext <32 x i32> %idxs to <32 x i64>
%byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %ext %byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %ext
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x half*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x half*>
Show All 9 Lines
; VBITS_GE_2048-NEXT: ptrue p0.h, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.h, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1h { z0.h }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1w { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_2048-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.s, z2.h ; VBITS_GE_2048-NEXT: sunpklo z2.s, z2.h
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d] ; VBITS_GE_2048-NEXT: st1h { z0.d }, p0, [x2, z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x half>, <32 x half>* %a %vals = load <32 x half>, <32 x half>* %a
%idxs = load <32 x i32>, <32 x i32>* %b %idxs = load <32 x i32>, <32 x i32>* %b
%ext = zext <32 x i32> %idxs to <32 x i64> %ext = zext <32 x i32> %idxs to <32 x i64>
%byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %ext %byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %ext
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x half*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x half*>
%mask = fcmp oeq <32 x half> %vals, zeroinitializer %mask = fcmp oeq <32 x half> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f16(<32 x half> %vals, <32 x half*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_64b_scaled(<32 x float>* %a, <32 x i64>* %b, float* %base) #0 { define void @masked_scatter_64b_scaled(<32 x float>* %a, <32 x i64>* %b, float* %base) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_64b_scaled: ; VBITS_GE_2048-LABEL: masked_scatter_64b_scaled:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d, lsl #2] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d, lsl #2]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i64>, <32 x i64>* %b %idxs = load <32 x i64>, <32 x i64>* %b
%ptrs = getelementptr float, float* %base, <32 x i64> %idxs %ptrs = getelementptr float, float* %base, <32 x i64> %idxs
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
call void @llvm.masked.scatter.v32f32(<32 x float> %vals, <32 x float*> %ptrs, i32 8, <32 x i1> %mask) call void @llvm.masked.scatter.v32f32(<32 x float> %vals, <32 x float*> %ptrs, i32 8, <32 x i1> %mask)
ret void ret void
} }
define void @masked_scatter_64b_unscaled(<32 x float>* %a, <32 x i64>* %b, i8* %base) #0 { define void @masked_scatter_64b_unscaled(<32 x float>* %a, <32 x i64>* %b, i8* %base) #0 {
; VBITS_GE_2048-LABEL: masked_scatter_64b_unscaled: ; VBITS_GE_2048-LABEL: masked_scatter_64b_unscaled:
; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048: // %bb.0:
; VBITS_GE_2048-NEXT: ptrue p0.s, vl32 ; VBITS_GE_2048-NEXT: ptrue p0.s, vl32
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z2.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z2.d, z2.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z2.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [x2, z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%idxs = load <32 x i64>, <32 x i64>* %b %idxs = load <32 x i64>, <32 x i64>* %b
%byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %idxs %byte_ptrs = getelementptr i8, i8* %base, <32 x i64> %idxs
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
Show All 9 Lines
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z2.d, x2 ; VBITS_GE_2048-NEXT: mov z2.d, x2
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d ; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d
; VBITS_GE_2048-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.d, z3.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z3.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%bases = load <32 x i8*>, <32 x i8*>* %b %bases = load <32 x i8*>, <32 x i8*>* %b
%byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
Show All 9 Lines
; VBITS_GE_2048-NEXT: ptrue p1.d, vl32 ; VBITS_GE_2048-NEXT: ptrue p1.d, vl32
; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1] ; VBITS_GE_2048-NEXT: ld1d { z1.d }, p1/z, [x1]
; VBITS_GE_2048-NEXT: mov z2.d, #4 // =0x4 ; VBITS_GE_2048-NEXT: mov z2.d, #4 // =0x4
; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_2048-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0
; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d ; VBITS_GE_2048-NEXT: add z1.d, p1/m, z1.d, z2.d
; VBITS_GE_2048-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_2048-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_2048-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT: uunpklo z2.d, z3.s ; VBITS_GE_2048-NEXT: sunpklo z2.d, z3.s
; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0 ; VBITS_GE_2048-NEXT: cmpne p0.d, p1/z, z2.d, #0
; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d] ; VBITS_GE_2048-NEXT: st1w { z0.d }, p0, [z1.d]
; VBITS_GE_2048-NEXT: ret ; VBITS_GE_2048-NEXT: ret
%vals = load <32 x float>, <32 x float>* %a %vals = load <32 x float>, <32 x float>* %a
%bases = load <32 x i8*>, <32 x i8*>* %b %bases = load <32 x i8*>, <32 x i8*>* %b
%byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4 %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4
%ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*> %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
%mask = fcmp oeq <32 x float> %vals, zeroinitializer %mask = fcmp oeq <32 x float> %vals, zeroinitializer
▲ Show 20 Lines • Show All 82 Lines • Show Last 20 Lines