This is an archive of the discontinued LLVM Phabricator instance.

Differential D97293

[ARM] Add a number of intrinsics for MVE lane interleaving
ClosedPublic

Authored by dmgreen on Feb 23 2021, 7:23 AM.

Details

Reviewers
SjoerdMeijer
simon_tatham
ostannard
efriedma
samtebbs
Commits
rGdd31b2c6e546: [ARM] Add a number of intrinsics for MVE lane interleaving
Summary

Add a number of intrinsics which natively lower to MVE operations to the lane interleaving pass, allowing it to efficiently interleave the lanes of chucks of operations containing these intrinsics.

Diff Detail

Event Timeline

dmgreen created this revision.Feb 23 2021, 7:23 AM
Herald added subscribers: danielkiss, hiraditya, kristof.beyls. · View Herald TranscriptFeb 23 2021, 7:23 AM
dmgreen requested review of this revision.Feb 23 2021, 7:23 AM
Herald added a project: Restricted Project. · View Herald TranscriptFeb 23 2021, 7:23 AM
dmgreen added a parent revision: D97292: [ARM] Add FP handling for MVE lane interleaving.Feb 23 2021, 7:23 AM
dmgreen updated this revision to Diff 336607.Apr 10 2021, 5:58 AM

Rebase

Harbormaster completed remote builds in B98102: Diff 336607.Apr 10 2021, 7:03 AM
SjoerdMeijer accepted this revision.Apr 12 2021, 12:38 AM

LGTM

This revision is now accepted and ready to land.Apr 12 2021, 12:38 AM
This revision was landed with ongoing or failed builds.Apr 12 2021, 9:23 AM
Closed by commit rGdd31b2c6e546: [ARM] Add a number of intrinsics for MVE lane interleaving (authored by dmgreen). · Explain Why
This revision was automatically updated to reflect the committed changes.
dmgreen added a commit: rGdd31b2c6e546: [ARM] Add a number of intrinsics for MVE lane interleaving.

Revision Contents

PathSize
llvm/
lib/
Target/
ARM/
34 lines
test/
CodeGen/
Thumb2/
139 lines

Diff 336861

llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp

Show First 20 Lines • Show All 187 Lines • ▼ Show 20 Lines while (!Worklist.empty()) {
case Instruction::FPExt: case Instruction::FPExt:
if (Exts.count(I)) if (Exts.count(I))
continue; continue;
for (auto *Use : I->users()) for (auto *Use : I->users())
Worklist.push_back(cast<Instruction>(Use)); Worklist.push_back(cast<Instruction>(Use));
Exts.insert(I); Exts.insert(I);
break; break;
case Instruction::Call: {
IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
if (!II)
return false;
switch (II->getIntrinsicID()) {
case Intrinsic::abs:
case Intrinsic::smin:
case Intrinsic::smax:
case Intrinsic::umin:
case Intrinsic::umax:
case Intrinsic::sadd_sat:
case Intrinsic::ssub_sat:
case Intrinsic::uadd_sat:
case Intrinsic::usub_sat:
case Intrinsic::minnum:
case Intrinsic::maxnum:
case Intrinsic::fabs:
case Intrinsic::fma:
case Intrinsic::ceil:
case Intrinsic::floor:
case Intrinsic::rint:
case Intrinsic::round:
case Intrinsic::trunc:
break;
default:
return false;
}
LLVM_FALLTHROUGH; // Fall through to treating these like an operator below.
}
// Binary/tertiary ops // Binary/tertiary ops
case Instruction::Add: case Instruction::Add:
case Instruction::Sub: case Instruction::Sub:
case Instruction::Mul: case Instruction::Mul:
case Instruction::AShr: case Instruction::AShr:
case Instruction::LShr: case Instruction::LShr:
case Instruction::Shl: case Instruction::Shl:
case Instruction::ICmp: case Instruction::ICmp:
case Instruction::FCmp: case Instruction::FCmp:
case Instruction::FAdd: case Instruction::FAdd:
case Instruction::FMul: case Instruction::FMul:
case Instruction::Select: case Instruction::Select:
if (Ops.count(I)) if (Ops.count(I))
continue; continue;
Ops.insert(I); Ops.insert(I);
for (Use &Op : I->operands()) { for (Use &Op : I->operands()) {
if (!isa<FixedVectorType>(Op->getType()))
continue;
if (isa<Instruction>(Op)) if (isa<Instruction>(Op))
Worklist.push_back(cast<Instruction>(&Op)); Worklist.push_back(cast<Instruction>(&Op));
else else
OtherLeafs.insert(&Op); OtherLeafs.insert(&Op);
} }
for (auto *Use : I->users()) for (auto *Use : I->users())
Worklist.push_back(cast<Instruction>(Use)); Worklist.push_back(cast<Instruction>(Use));
Show All 18 Lines LLVM_DEBUG({
dbgs() << "Found group:\n Exts:"; dbgs() << "Found group:\n Exts:";
for (auto *I : Exts) for (auto *I : Exts)
dbgs() << " " << *I << "\n"; dbgs() << " " << *I << "\n";
dbgs() << " Ops:"; dbgs() << " Ops:";
for (auto *I : Ops) for (auto *I : Ops)
dbgs() << " " << *I << "\n"; dbgs() << " " << *I << "\n";
dbgs() << " OtherLeafs:"; dbgs() << " OtherLeafs:";
for (auto *I : OtherLeafs) for (auto *I : OtherLeafs)
dbgs() << " " << *I << "\n"; dbgs() << " " << *I->get() << " of " << *I->getUser() << "\n";
dbgs() << "Truncs:"; dbgs() << "Truncs:";
for (auto *I : Truncs) for (auto *I : Truncs)
dbgs() << " " << *I << "\n"; dbgs() << " " << *I << "\n";
}); });
assert(!Truncs.empty() && "Expected some truncs"); assert(!Truncs.empty() && "Expected some truncs");
// Check types // Check types
▲ Show 20 Lines • Show All 103 Lines • Show Last 20 Lines

llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll

Show First 20 Lines • Show All 625 Lines • ▼ Show 20 Linesentry:
ret <16 x i8> %t ret <16 x i8> %t
} }
define arm_aapcs_vfpcc <8 x i16> @ext_intrinsics_trunc_i16(<8 x i16> %a, <8 x i16> %b) { define arm_aapcs_vfpcc <8 x i16> @ext_intrinsics_trunc_i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-LABEL: ext_intrinsics_trunc_i16: ; CHECK-LABEL: ext_intrinsics_trunc_i16:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .vsave {d8, d9} ; CHECK-NEXT: .vsave {d8, d9}
; CHECK-NEXT: vpush {d8, d9} ; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: vmov.u16 r0, q1[2] ; CHECK-NEXT: vmovlb.u16 q2, q1
; CHECK-NEXT: vmov.u16 r1, q1[0] ; CHECK-NEXT: vmovlb.s16 q3, q0
; CHECK-NEXT: vmov q2, q0 ; CHECK-NEXT: vqadd.s32 q4, q3, q2
; CHECK-NEXT: vmov q0[2], q0[0], r1, r0 ; CHECK-NEXT: vmovlt.u16 q1, q1
; CHECK-NEXT: vmov.u16 r0, q1[3] ; CHECK-NEXT: vqadd.u32 q4, q4, q2
; CHECK-NEXT: vmov.u16 r1, q1[1] ; CHECK-NEXT: vmovlt.s16 q0, q0
; CHECK-NEXT: vmov q0[3], q0[1], r1, r0
; CHECK-NEXT: vmov.u16 r0, q2[2]
; CHECK-NEXT: vmov.u16 r1, q2[0]
; CHECK-NEXT: vmovlb.u16 q0, q0
; CHECK-NEXT: vmov q3[2], q3[0], r1, r0
; CHECK-NEXT: vmov.u16 r0, q2[3]
; CHECK-NEXT: vmov.u16 r1, q2[1]
; CHECK-NEXT: vmov q3[3], q3[1], r1, r0
; CHECK-NEXT: vmov.u16 r1, q2[4]
; CHECK-NEXT: vmovlb.s16 q3, q3
; CHECK-NEXT: vqadd.s32 q4, q3, q0
; CHECK-NEXT: vqadd.u32 q4, q4, q0
; CHECK-NEXT: vqsub.s32 q4, q4, q3 ; CHECK-NEXT: vqsub.s32 q4, q4, q3
; CHECK-NEXT: vqsub.u32 q4, q4, q0 ; CHECK-NEXT: vqsub.u32 q4, q4, q2
; CHECK-NEXT: vabs.s32 q4, q4 ; CHECK-NEXT: vabs.s32 q4, q4
; CHECK-NEXT: vmin.s32 q4, q4, q3 ; CHECK-NEXT: vmin.s32 q4, q4, q3
; CHECK-NEXT: vmax.s32 q4, q4, q0 ; CHECK-NEXT: vmax.s32 q4, q4, q2
; CHECK-NEXT: vmin.u32 q3, q4, q3 ; CHECK-NEXT: vmin.u32 q3, q4, q3
; CHECK-NEXT: vmax.u32 q3, q3, q0 ; CHECK-NEXT: vqadd.s32 q4, q0, q1
; CHECK-NEXT: vmov r0, s12 ; CHECK-NEXT: vqadd.u32 q4, q4, q1
; CHECK-NEXT: vmov.16 q0[0], r0 ; CHECK-NEXT: vqsub.s32 q4, q4, q0
; CHECK-NEXT: vmov r0, s13 ; CHECK-NEXT: vqsub.u32 q4, q4, q1
; CHECK-NEXT: vmov.16 q0[1], r0 ; CHECK-NEXT: vabs.s32 q4, q4
; CHECK-NEXT: vmov r0, s14 ; CHECK-NEXT: vmin.s32 q4, q4, q0
; CHECK-NEXT: vmov.16 q0[2], r0 ; CHECK-NEXT: vmax.s32 q4, q4, q1
; CHECK-NEXT: vmov r0, s15 ; CHECK-NEXT: vmin.u32 q0, q4, q0
; CHECK-NEXT: vmov.16 q0[3], r0 ; CHECK-NEXT: vmax.u32 q1, q0, q1
; CHECK-NEXT: vmov.u16 r0, q2[6] ; CHECK-NEXT: vmax.u32 q0, q3, q2
; CHECK-NEXT: vmov q3[2], q3[0], r1, r0 ; CHECK-NEXT: vmovnt.i32 q0, q1
; CHECK-NEXT: vmov.u16 r0, q2[7]
; CHECK-NEXT: vmov.u16 r1, q2[5]
; CHECK-NEXT: vmov q3[3], q3[1], r1, r0
; CHECK-NEXT: vmov.u16 r0, q1[6]
; CHECK-NEXT: vmov.u16 r1, q1[4]
; CHECK-NEXT: vmov q2[2], q2[0], r1, r0
; CHECK-NEXT: vmov.u16 r0, q1[7]
; CHECK-NEXT: vmov.u16 r1, q1[5]
; CHECK-NEXT: vmov q2[3], q2[1], r1, r0
; CHECK-NEXT: vmovlb.u16 q1, q2
; CHECK-NEXT: vmovlb.s16 q2, q3
; CHECK-NEXT: vqadd.s32 q3, q2, q1
; CHECK-NEXT: vqadd.u32 q3, q3, q1
; CHECK-NEXT: vqsub.s32 q3, q3, q2
; CHECK-NEXT: vqsub.u32 q3, q3, q1
; CHECK-NEXT: vabs.s32 q3, q3
; CHECK-NEXT: vmin.s32 q3, q3, q2
; CHECK-NEXT: vmax.s32 q3, q3, q1
; CHECK-NEXT: vmin.u32 q2, q3, q2
; CHECK-NEXT: vmax.u32 q1, q2, q1
; CHECK-NEXT: vmov r0, s4
; CHECK-NEXT: vmov.16 q0[4], r0
; CHECK-NEXT: vmov r0, s5
; CHECK-NEXT: vmov.16 q0[5], r0
; CHECK-NEXT: vmov r0, s6
; CHECK-NEXT: vmov.16 q0[6], r0
; CHECK-NEXT: vmov r0, s7
; CHECK-NEXT: vmov.16 q0[7], r0
; CHECK-NEXT: vpop {d8, d9} ; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: bx lr ; CHECK-NEXT: bx lr
entry: entry:
%sa = sext <8 x i16> %a to <8 x i32> %sa = sext <8 x i16> %a to <8 x i32>
%sb = zext <8 x i16> %b to <8 x i32> %sb = zext <8 x i16> %b to <8 x i32>
%sadd = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %sa, <8 x i32> %sb) %sadd = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %sa, <8 x i32> %sb)
%uadd = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %sadd, <8 x i32> %sb) %uadd = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %sadd, <8 x i32> %sb)
%ssub = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %uadd, <8 x i32> %sa) %ssub = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %uadd, <8 x i32> %sa)
%usub = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %ssub, <8 x i32> %sb) %usub = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %ssub, <8 x i32> %sb)
%abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %usub, i1 true) %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %usub, i1 true)
%smin = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %abs, <8 x i32> %sa) %smin = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %abs, <8 x i32> %sa)
%smax = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %smin, <8 x i32> %sb) %smax = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %smin, <8 x i32> %sb)
%umin = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %smax, <8 x i32> %sa) %umin = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %smax, <8 x i32> %sa)
%umax = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %umin, <8 x i32> %sb) %umax = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %umin, <8 x i32> %sb)
%t = trunc <8 x i32> %umax to <8 x i16> %t = trunc <8 x i32> %umax to <8 x i16>
ret <8 x i16> %t ret <8 x i16> %t
} }
define arm_aapcs_vfpcc <8 x half> @ext_fpintrinsics_trunc_half(<8 x half> %a, <8 x half> %b) { define arm_aapcs_vfpcc <8 x half> @ext_fpintrinsics_trunc_half(<8 x half> %a, <8 x half> %b) {
; CHECK-LABEL: ext_fpintrinsics_trunc_half: ; CHECK-LABEL: ext_fpintrinsics_trunc_half:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .vsave {d8, d9} ; CHECK-NEXT: .vsave {d8, d9}
; CHECK-NEXT: vpush {d8, d9} ; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: vmov q2, q0 ; CHECK-NEXT: vcvtb.f32.f16 q2, q0
; CHECK-NEXT: vcvtt.f32.f16 s3, s5 ; CHECK-NEXT: vcvtb.f32.f16 q4, q1
; CHECK-NEXT: vcvtt.f32.f16 s15, s9 ; CHECK-NEXT: vabs.f32 q3, q2
; CHECK-NEXT: vcvtb.f32.f16 s2, s5 ; CHECK-NEXT: vcvtt.f32.f16 q0, q0
; CHECK-NEXT: vcvtb.f32.f16 s14, s9 ; CHECK-NEXT: vminnm.f32 q3, q3, q2
; CHECK-NEXT: vcvtt.f32.f16 s1, s4 ; CHECK-NEXT: vcvtt.f32.f16 q1, q1
; CHECK-NEXT: vcvtt.f32.f16 s13, s8 ; CHECK-NEXT: vmaxnm.f32 q3, q3, q4
; CHECK-NEXT: vcvtb.f32.f16 s0, s4 ; CHECK-NEXT: vfma.f32 q4, q3, q2
; CHECK-NEXT: vcvtb.f32.f16 s12, s8 ; CHECK-NEXT: vabs.f32 q3, q0
; CHECK-NEXT: vabs.f32 q4, q3 ; CHECK-NEXT: vminnm.f32 q3, q3, q0
; CHECK-NEXT: vminnm.f32 q4, q4, q3 ; CHECK-NEXT: vrintp.f32 q2, q4
; CHECK-NEXT: vmaxnm.f32 q4, q4, q0 ; CHECK-NEXT: vmaxnm.f32 q3, q3, q1
; CHECK-NEXT: vfma.f32 q0, q4, q3 ; CHECK-NEXT: vrintm.f32 q2, q2
; CHECK-NEXT: vcvtt.f32.f16 s19, s11 ; CHECK-NEXT: vfma.f32 q1, q3, q0
; CHECK-NEXT: vrintp.f32 q0, q0 ; CHECK-NEXT: vrintx.f32 q2, q2
; CHECK-NEXT: vcvtb.f32.f16 s18, s11 ; CHECK-NEXT: vrintp.f32 q0, q1
; CHECK-NEXT: vrinta.f32 q2, q2
; CHECK-NEXT: vrintm.f32 q0, q0 ; CHECK-NEXT: vrintm.f32 q0, q0
; CHECK-NEXT: vcvtt.f32.f16 s17, s10 ; CHECK-NEXT: vrintz.f32 q2, q2
; CHECK-NEXT: vrintx.f32 q0, q0 ; CHECK-NEXT: vrintx.f32 q0, q0
; CHECK-NEXT: vcvtb.f32.f16 s16, s10
; CHECK-NEXT: vrinta.f32 q0, q0 ; CHECK-NEXT: vrinta.f32 q0, q0
; CHECK-NEXT: vrintz.f32 q3, q0 ; CHECK-NEXT: vrintz.f32 q1, q0
; CHECK-NEXT: vcvtb.f16.f32 s0, s12 ; CHECK-NEXT: vcvtb.f16.f32 q0, q2
; CHECK-NEXT: vcvtt.f16.f32 s0, s13 ; CHECK-NEXT: vcvtt.f16.f32 q0, q1
; CHECK-NEXT: vcvtb.f16.f32 s1, s14
; CHECK-NEXT: vcvtt.f16.f32 s1, s15
; CHECK-NEXT: vcvtt.f32.f16 s15, s7
; CHECK-NEXT: vcvtb.f32.f16 s14, s7
; CHECK-NEXT: vcvtt.f32.f16 s13, s6
; CHECK-NEXT: vcvtb.f32.f16 s12, s6
; CHECK-NEXT: vabs.f32 q1, q4
; CHECK-NEXT: vminnm.f32 q1, q1, q4
; CHECK-NEXT: vmaxnm.f32 q1, q1, q3
; CHECK-NEXT: vfma.f32 q3, q1, q4
; CHECK-NEXT: vrintp.f32 q1, q3
; CHECK-NEXT: vrintm.f32 q1, q1
; CHECK-NEXT: vrintx.f32 q1, q1
; CHECK-NEXT: vrinta.f32 q1, q1
; CHECK-NEXT: vrintz.f32 q1, q1
; CHECK-NEXT: vcvtb.f16.f32 s2, s4
; CHECK-NEXT: vcvtt.f16.f32 s2, s5
; CHECK-NEXT: vcvtb.f16.f32 s3, s6
; CHECK-NEXT: vcvtt.f16.f32 s3, s7
; CHECK-NEXT: vpop {d8, d9} ; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: bx lr ; CHECK-NEXT: bx lr
entry: entry:
%sa = fpext <8 x half> %a to <8 x float> %sa = fpext <8 x half> %a to <8 x float>
%sb = fpext <8 x half> %b to <8 x float> %sb = fpext <8 x half> %b to <8 x float>
%abs = call <8 x float> @llvm.fabs.v8f32(<8 x float> %sa) %abs = call <8 x float> @llvm.fabs.v8f32(<8 x float> %sa)
%min = call <8 x float> @llvm.minnum.v8f32(<8 x float> %abs, <8 x float> %sa) %min = call <8 x float> @llvm.minnum.v8f32(<8 x float> %abs, <8 x float> %sa)
%max = call <8 x float> @llvm.maxnum.v8f32(<8 x float> %min, <8 x float> %sb) %max = call <8 x float> @llvm.maxnum.v8f32(<8 x float> %min, <8 x float> %sb)
Show All 28 Lines