[ARM] Allow rounding intrinsics to be tail predicated

[samtebbs]

Summary

This patch stops the trunc, rint, round, floor and ceil intrinsics from blocking tail predication.

[samtebbs]

Add missing floor and ceil tests

[dmgreen]

llvm/lib/Target/ARM/MVETailPredication.cpp

361

Um, you might be fixing my bug here, but can you make it so that the floating point instructions only tail predicate when Subtarget->hasMVEEFloatOps() is true? Otherwise in integer only MVE we could start trying to and tail predicate where it will end up expanding the instruction (which probably isn't a huge deal, but we should try and get it correct).

llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll

4

Can you also add a test for nearbyint too, to show that it _doesn't_ get tail predicated (I think it gets expanded to a multiple scalar instructions.

259

I think you can remove all this and the code below. It complains about stuff then use -mattr=+mve.fp in the run line.

You will probably have to remove the !tbaa info too.

[samtebbs]

Only tail predicate if the subtarget has float ops, add nearbyint test and clean up the test file.

[samtebbs]

llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll

259

I did try removing them before but ran into an abort, which https://reviews.llvm.org/D82292 has since fixed.

[SjoerdMeijer]

llvm/lib/Target/ARM/MVETailPredication.cpp

112

It would have my preference not to pass ST around all the time; just set it as a class member or get it in IsPredicatedVectorLoop.

[dmgreen]

llvm/lib/Target/ARM/MVETailPredication.cpp

345–346

fma can go into the float block too.

llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll

230

You can use the same test code as floor, for example, but with floor replaced by nearbyint. This won't test that the vecotrizer does or doesn't tail predicate it, but will test that we expand to something non-tail predicated still.

[samtebbs]

Fix up nearbyint test, make ST a class member and move fma case.

[samtebbs]

Fix types of nearbyint intrinsic

[dmgreen]

Thanks. LGTM with one nit.

llvm/lib/Target/ARM/MVETailPredication.cpp

345–346

I think you may need to add a LLVM_FALLTHROUGH (or a extra break) to stop a warning from cropping up.

[samtebbs]

Add LLVM_FALLTHROUGH

llvm/lib/Target/ARM/MVETailPredication.cpp

345–346

Thanks. Well spotted.

llvm/lib/Target/ARM/MVETailPredication.cpp

void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addPreserved<LoopInfoWrapperPass>();
     AU.setPreservesCFG();
   }
 
   bool runOnLoop(Loop *L, LPPassManager&) override;
 
 private:
   /// Perform the relevant checks on the loop and convert if possible.
   bool TryConvert(Value *TripCount);
 
   /// Return whether this is a vectorized loop, that contains masked
   /// load/stores.
   bool IsPredicatedVectorLoop();
 
   /// Perform checks on the arguments of @llvm.get.active.lane.mask
   /// intrinsic: check if the first is a loop induction variable, and for the
   /// the second check that no overflow can occur in the expression that use
static FixedVectorType *getVectorType(IntrinsicInst *I) {
   return VecTy;
 }
 
 bool MVETailPredication::IsPredicatedVectorLoop() {
   // Check that the loop contains at least one masked load/store intrinsic.
   // We only support 'normal' vector instructions - other than masked
   // load/stores.
   for (auto *BB : L->getBlocks()) {
     for (auto &I : *BB) {
       if (IsMasked(&I)) {
         FixedVectorType *VecTy = getVectorType(cast<IntrinsicInst>(&I));
         unsigned Lanes = VecTy->getNumElements();
         unsigned ElementWidth = VecTy->getScalarSizeInBits();
         // MVE vectors are 128-bit, but don't support 128 x i1.
         // TODO: Can we support vectors larger than 128-bits?
         unsigned MaxWidth = TTI->getRegisterBitWidth(true);
         if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth)
           return false;
         MaskedInsts.push_back(cast<IntrinsicInst>(&I));
       } else if (auto *Int = dyn_cast<IntrinsicInst>(&I)) {
         switch (Int->getIntrinsicID()) {
           case Intrinsic::fma:
           case Intrinsic::sadd_sat:
           case Intrinsic::uadd_sat:
+          case Intrinsic::trunc:
+          case Intrinsic::rint:
+          case Intrinsic::round:
+          case Intrinsic::floor:
+          case Intrinsic::ceil:
             continue;
           default:
             break;
         }
         for (auto &U : Int->args()) {
           if (isa<VectorType>(U->getType()))
             return false;
         }

llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
+define arm_aapcs_vfpcc void @round(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: round:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .setfp r7, sp
+; CHECK-NEXT:    mov r7, sp
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrinta.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef), !tbaa !3
+  %1 = call fast <4 x float> @llvm.round.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask), !tbaa !3
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @rint(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: rint:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .setfp r7, sp
+; CHECK-NEXT:    mov r7, sp
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintx.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef), !tbaa !3
+  %1 = call fast <4 x float> @llvm.rint.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask), !tbaa !3
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @trunc(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: trunc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .setfp r7, sp
+; CHECK-NEXT:    mov r7, sp
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintz.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef), !tbaa !3
+  %1 = call fast <4 x float> @llvm.trunc.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask), !tbaa !3
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @ceil(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: ceil:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .setfp r7, sp
+; CHECK-NEXT:    mov r7, sp
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintp.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef), !tbaa !3
+  %1 = call fast <4 x float> @llvm.ceil.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask), !tbaa !3
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @floor(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
+; CHECK-LABEL: floor:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .setfp r7, sp
+; CHECK-NEXT:    mov r7, sp
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    dlstp.32 lr, r2
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vrintm.f32 q0, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1], #16
+; CHECK-NEXT:    letp lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp5 = icmp eq i32 %n, 0
+  br i1 %cmp5, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %n, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %n, -1
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %next.gep = getelementptr float, float* %pSrcA, i32 %index
+  %next.gep14 = getelementptr float, float* %pDst, i32 %index
+  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
+  %0 = bitcast float* %next.gep to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %0, i32 4, <4 x i1> %active.lane.mask, <4 x float> undef), !tbaa !3
+  %1 = call fast <4 x float> @llvm.floor.v4f32(<4 x float> %wide.masked.load)
+  %2 = bitcast float* %next.gep14 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %1, <4 x float>* %2, i32 4, <4 x i1> %active.lane.mask), !tbaa !3
+  %index.next = add i32 %index, 4
+  %3 = icmp eq i32 %index.next, %n.vec
+  br i1 %3, label %for.cond.cleanup, label %vector.body, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #1
+
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>) #2
+
+declare <4 x float> @llvm.trunc.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.rint.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.round.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>) #3
+
+declare <4 x float> @llvm.floor.v4f32(<4 x float>) #3
+
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>) #4
+
+attributes #0 = { nofree nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-dotprod,-fp16fml,-hwdiv-arm,-i8mm,-sb,-sha2" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { nosync nounwind readnone willreturn }
+attributes #2 = { argmemonly nounwind readonly willreturn }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { argmemonly nounwind willreturn }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"min_enum_size", i32 4}
+!2 = !{!"clang version 11.0.0 ([e-mail]:llvm/llvm-project 39a4dade76da7f19ac9ae62dd02102141ff8141c)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"float", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = distinct !{!7, !8, !9}
+!8 = !{!"llvm.loop.unroll.disable"}
+!9 = !{!"llvm.loop.isvectorized", i32 1}