This is an archive of the discontinued LLVM Phabricator instance.

Differential D78385

[ARM] Allow fma in tail predicated loops
ClosedPublic

Authored by dmgreen on Apr 17 2020, 9:59 AM.

Details

Reviewers
samparker
SjoerdMeijer
simon_tatham
ostannard
Commits
rG61b8af0375d7: [ARM] Allow fma in tail predicated loops
Summary

There are some intrinsics like this that currently block tail predication, but should be fine. This allows fma through, as the one that I ran into. There may be others that need the same treatment but I've only done this one here (I wanted smaller testcases for another patch!)

Diff Detail

Event Timeline

dmgreen created this revision.Apr 17 2020, 9:59 AM
Herald added subscribers: danielkiss, hiraditya, kristof.beyls. · View Herald TranscriptApr 17 2020, 9:59 AM
samparker accepted this revision.Apr 20 2020, 2:16 AM

Adding these cases as we run into them sounds good, thanks.

This revision is now accepted and ready to land.Apr 20 2020, 2:16 AM
Closed by commit rG61b8af0375d7: [ARM] Allow fma in tail predicated loops (authored by dmgreen). · Explain WhyApr 27 2020, 8:01 AM
This revision was automatically updated to reflect the committed changes.
Herald added a project: Restricted Project. · View Herald TranscriptApr 27 2020, 8:02 AM

Revision Contents

PathSize
llvm/
lib/
Target/
ARM/
2 lines
test/
CodeGen/
Thumb2/
338 lines

Diff 260327

llvm/lib/Target/ARM/MVETailPredication.cpp

Show First 20 Lines • Show All 349 Lines • ▼ Show 20 Lines for (auto &I : *BB) {
unsigned ElementWidth = VecTy->getScalarSizeInBits(); unsigned ElementWidth = VecTy->getScalarSizeInBits();
// MVE vectors are 128-bit, but don't support 128 x i1. // MVE vectors are 128-bit, but don't support 128 x i1.
// TODO: Can we support vectors larger than 128-bits? // TODO: Can we support vectors larger than 128-bits?
unsigned MaxWidth = TTI->getRegisterBitWidth(true); unsigned MaxWidth = TTI->getRegisterBitWidth(true);
if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth) if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth)
return false; return false;
MaskedInsts.push_back(cast<IntrinsicInst>(&I)); MaskedInsts.push_back(cast<IntrinsicInst>(&I));
} else if (auto *Int = dyn_cast<IntrinsicInst>(&I)) { } else if (auto *Int = dyn_cast<IntrinsicInst>(&I)) {
if (Int->getIntrinsicID() == Intrinsic::fma)
continue;
for (auto &U : Int->args()) { for (auto &U : Int->args()) {
if (isa<VectorType>(U->getType())) if (isa<VectorType>(U->getType()))
return false; return false;
} }
} }
} }
} }
▲ Show 20 Lines • Show All 348 Lines • Show Last 20 Lines

llvm/test/CodeGen/Thumb2/mve-fma-loops.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false %s -o - | FileCheck %s ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false %s -o - | FileCheck %s
define arm_aapcs_vfpcc void @fmas1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fmas1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fmas1: ; CHECK-LABEL: fmas1:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9, d10, d11} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB0_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: adr r4, .LCPI0_0 ; CHECK-NEXT: .LBB0_1: @ %vector.body
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vmov q3, q0
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vldrw.u32 q2, [r0], #16
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: vfma.f32 q3, q2, q1
; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vstrw.32 q3, [r2], #16
; CHECK-NEXT: vmov r3, s0 ; CHECK-NEXT: letp lr, .LBB0_1
; CHECK-NEXT: vdup.32 q2, r12 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: vdup.32 q0, r3 ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: dls lr, lr
; CHECK-NEXT: .LBB0_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vmov q5, q0
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
; CHECK-NEXT: vldrwt.u32 q4, [r0], #16
; CHECK-NEXT: vfma.f32 q5, q4, q3
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q5, [r2], #16
; CHECK-NEXT: le lr, .LBB0_2
; CHECK-NEXT: .LBB0_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9, d10, d11}
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI0_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
%trip.count.minus.1 = add i32 %n, -1 %trip.count.minus.1 = add i32 %n, -1
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines
for.cond.cleanup: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body, %entry
ret void ret void
} }
define arm_aapcs_vfpcc void @fma1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fma1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fma1: ; CHECK-LABEL: fma1:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB2_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: adr r4, .LCPI2_0 ; CHECK-NEXT: .LBB2_1: @ %vector.body
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vldrw.u32 q1, [r0], #16
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q2, [r1], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vfma.f32 q2, q1, q0
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: vstrw.32 q2, [r2], #16
; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: letp lr, .LBB2_1
; CHECK-NEXT: vmov r3, s0 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: vdup.32 q2, r12 ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: vdup.32 q0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: dls lr, lr
; CHECK-NEXT: .LBB2_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r0], #16
; CHECK-NEXT: vldrwt.u32 q4, [r1], #16
; CHECK-NEXT: vfma.f32 q4, q3, q0
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q4, [r2], #16
; CHECK-NEXT: le lr, .LBB2_2
; CHECK-NEXT: .LBB2_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI2_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
%trip.count.minus.1 = add i32 %n, -1 %trip.count.minus.1 = add i32 %n, -1
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines
for.cond.cleanup: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body, %entry
ret void ret void
} }
define arm_aapcs_vfpcc void @fmss1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fmss1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fmss1: ; CHECK-LABEL: fmss1:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r7, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, r5, r7, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9, d10, d11} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB4_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: eor r12, r12, #-2147483648
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: .LBB4_1: @ %vector.body
; CHECK-NEXT: adr r4, .LCPI4_0 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vmov q3, q0
; CHECK-NEXT: subs r5, r3, #1 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vldrw.u32 q2, [r0], #16
; CHECK-NEXT: vdup.32 q2, r5 ; CHECK-NEXT: vfma.f32 q3, q2, q1
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: vstrw.32 q3, [r2], #16
; CHECK-NEXT: vmov r12, s0 ; CHECK-NEXT: letp lr, .LBB4_1
; CHECK-NEXT: dls lr, lr ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: eor r3, r12, #-2147483648 ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: vdup.32 q0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: .LBB4_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vmov q5, q0
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
; CHECK-NEXT: vldrwt.u32 q4, [r0], #16
; CHECK-NEXT: vfma.f32 q5, q4, q3
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q5, [r2], #16
; CHECK-NEXT: le lr, .LBB4_2
; CHECK-NEXT: .LBB4_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9, d10, d11}
; CHECK-NEXT: pop {r4, r5, r7, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI4_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%fneg = fneg fast float %a %fneg = fneg fast float %a
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines
for.cond.cleanup: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body, %entry
ret void ret void
} }
define arm_aapcs_vfpcc void @fmss3(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fmss3(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fmss3: ; CHECK-LABEL: fmss3:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9, d10, d11} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB6_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: adr r4, .LCPI6_0 ; CHECK-NEXT: .LBB6_1: @ %vector.body
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vmov q3, q0
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vldrw.u32 q2, [r0], #16
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: vfms.f32 q3, q2, q1
; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vstrw.32 q3, [r2], #16
; CHECK-NEXT: vmov r3, s0 ; CHECK-NEXT: letp lr, .LBB6_1
; CHECK-NEXT: vdup.32 q2, r12 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: vdup.32 q0, r3 ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: dls lr, lr
; CHECK-NEXT: .LBB6_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vmov q5, q0
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
; CHECK-NEXT: vldrwt.u32 q4, [r0], #16
; CHECK-NEXT: vfms.f32 q5, q4, q3
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q5, [r2], #16
; CHECK-NEXT: le lr, .LBB6_2
; CHECK-NEXT: .LBB6_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9, d10, d11}
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI6_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
%trip.count.minus.1 = add i32 %n, -1 %trip.count.minus.1 = add i32 %n, -1
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines
for.cond.cleanup: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body, %entry
ret void ret void
} }
define arm_aapcs_vfpcc void @fms1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fms1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fms1: ; CHECK-LABEL: fms1:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r7, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, r5, r7, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB8_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: eor r12, r12, #-2147483648
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: .LBB8_1: @ %vector.body
; CHECK-NEXT: adr r4, .LCPI8_0 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q1, [r0], #16
; CHECK-NEXT: subs r5, r3, #1 ; CHECK-NEXT: vldrw.u32 q2, [r1], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vfma.f32 q2, q1, q0
; CHECK-NEXT: vdup.32 q2, r5 ; CHECK-NEXT: vstrw.32 q2, [r2], #16
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: letp lr, .LBB8_1
; CHECK-NEXT: vmov r12, s0 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: dls lr, lr ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: eor r3, r12, #-2147483648
; CHECK-NEXT: vdup.32 q0, r3
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: .LBB8_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r0], #16
; CHECK-NEXT: vldrwt.u32 q4, [r1], #16
; CHECK-NEXT: vfma.f32 q4, q3, q0
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q4, [r2], #16
; CHECK-NEXT: le lr, .LBB8_2
; CHECK-NEXT: .LBB8_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: pop {r4, r5, r7, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI8_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%fneg = fneg fast float %a %fneg = fneg fast float %a
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines
for.cond.cleanup: ; preds = %vector.body, %entry for.cond.cleanup: ; preds = %vector.body, %entry
ret void ret void
} }
define arm_aapcs_vfpcc void @fms3(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) { define arm_aapcs_vfpcc void @fms3(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
; CHECK-LABEL: fms3: ; CHECK-LABEL: fms3:
; CHECK: @ %bb.0: @ %entry ; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, lr} ; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: .vsave {d8, d9} ; CHECK-NEXT: cmp r3, #1
; CHECK-NEXT: vpush {d8, d9} ; CHECK-NEXT: it lt
; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: poplt {r7, pc}
; CHECK-NEXT: blt .LBB10_3 ; CHECK-NEXT: vmov r12, s0
; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: vdup.32 q0, r12
; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: dlstp.32 lr, r3
; CHECK-NEXT: adr r4, .LCPI10_0 ; CHECK-NEXT: .LBB10_1: @ %vector.body
; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q2, [r0], #16
; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: vneg.f32 q1, q1
; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: vfma.f32 q1, q2, q0
; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vstrw.32 q1, [r2], #16
; CHECK-NEXT: vmov r3, s0 ; CHECK-NEXT: letp lr, .LBB10_1
; CHECK-NEXT: vdup.32 q2, r12 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: vdup.32 q0, r3 ; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: dls lr, lr
; CHECK-NEXT: .LBB10_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vdup.32 q3, r3
; CHECK-NEXT: adds r3, #4
; CHECK-NEXT: vorr q3, q3, q1
; CHECK-NEXT: vptt.u32 cs, q2, q3
; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
; CHECK-NEXT: vldrwt.u32 q4, [r0], #16
; CHECK-NEXT: vneg.f32 q3, q3
; CHECK-NEXT: vfma.f32 q3, q4, q0
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q3, [r2], #16
; CHECK-NEXT: le lr, .LBB10_2
; CHECK-NEXT: .LBB10_3: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI10_0:
; CHECK-NEXT: .long 0 @ 0x0
; CHECK-NEXT: .long 1 @ 0x1
; CHECK-NEXT: .long 2 @ 0x2
; CHECK-NEXT: .long 3 @ 0x3
entry: entry:
%cmp8 = icmp sgt i32 %n, 0 %cmp8 = icmp sgt i32 %n, 0
br i1 %cmp8, label %vector.ph, label %for.cond.cleanup br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
vector.ph: ; preds = %entry vector.ph: ; preds = %entry
%n.rnd.up = add i32 %n, 3 %n.rnd.up = add i32 %n, 3
%n.vec = and i32 %n.rnd.up, -4 %n.vec = and i32 %n.rnd.up, -4
%trip.count.minus.1 = add i32 %n, -1 %trip.count.minus.1 = add i32 %n, -1
▲ Show 20 Lines • Show All 93 Lines • Show Last 20 Lines