[ARM] Loop unrolling preferences for LOB cores

[samparker]

Summary

Perform loop unrolling differently for cores with low-overhead branches:

• Don't unroll the remainder loop, with the expectation that both the unrolled loop and the remainder will be converted into a loloop.
• Don't force unroll small loops as we now try to use the non-decrement form of LE for uncountable loops. We need to use CBN/Z for that optimisation, so reducing code size is important due to their limited range.

[SjoerdMeijer]

The added tests check exactly what's being changed/added here, so that's excellent.
I was just wondering if it would be good if we also add a more end-to-end/unit llc test, that shows actual loloop creation for these cases? That is, if these tests are not already there. I had only a very quick look, and am not sure, but you'll probably know.

test/Transforms/LoopUnroll/ARM/loop-unrolling.ll

1

nit: --check-prefixes=CHECK-UNROLL-A,CHECK is shorter

48

nit: this whole block looks exactly the same as CHECK-UNROLL-T2, so could be shared.

[samparker]

Cheers. I'm gonna put this patch on hold to do some more investigations into how hardware loops and loop unrolling interact... The current testing for this is in Transforms/HardwareLoops/ARM/structure.ll

lib/Target/ARM/ARMTargetTransformInfo.cpp

     }
   }
 
   LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
 
   UP.Partial = true;
   UP.Runtime = true;
   UP.UpperBound = true;
-  UP.UnrollRemainder = true;
+  // Hopefully we'll be able to generate a low-overhead loop for both the
+  // unrolled loop and the remainder loop.
+  UP.UnrollRemainder = ST->hasLOB() ? false : true;
   UP.DefaultUnrollRuntimeCount = 4;
   UP.UnrollAndJam = true;
   UP.UnrollAndJamInnerLoopThreshold = 60;
 
   // Force unrolling small loops can be very useful because of the branch
-  // taken cost of the backedge.
-  if (Cost < 12)
+  // taken cost of the backedge, but generally shouldn't be required if we
+  // have low-overhead branches.
+  if (Cost < 12 && !ST->hasLOB())
     UP.Force = true;
 }
 
 bool ARMTTIImpl::useReductionIntrinsic(unsigned Opcode, Type *Ty,
                                        TTI::ReductionFlags Flags) const {
   assert(isa<VectorType>(Ty) && "Expected Ty to be a vector type");
   unsigned ScalarBits = Ty->getScalarSizeInBits();
   if (!ST->hasMVEIntegerOps())

test/Transforms/LoopUnroll/ARM/loop-unrolling.ll

-; RUN: opt -mtriple=armv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
-; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
-; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a72 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
-; RUN: opt -mtriple=thumbv8m -mcpu=cortex-m23 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T1
-; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2
-; RUN: opt -mtriple=thumbv7em -mcpu=cortex-m7 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2
+; RUN: opt -mtriple=armv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a72 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv8m -mcpu=cortex-m23 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T1 --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2 --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv7em -mcpu=cortex-m7 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2 --check-prefix=CHECK
+; RUN: opt -mtriple=thumbv8.1m.main -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-LOB --check-prefix=CHECK
 
 ; CHECK-LABEL: partial
 define arm_aapcs_vfpcc void @partial(i32* nocapture %C, i32* nocapture readonly %A, i32* nocapture readonly %B) local_unnamed_addr #0 {
 entry:
   br label %for.body
 
 ; CHECK-LABEL: for.body
 for.body:

 ; CHECK-UNROLL-T2: [[IV12:%[a-z.0-9]+]] = add nuw nsw i32 [[IV11]], 1
 ; CHECK-UNROLL-T2: [[IV13:%[a-z.0-9]+]] = add nuw nsw i32 [[IV12]], 1
 ; CHECK-UNROLL-T2: [[IV14:%[a-z.0-9]+]] = add nuw nsw i32 [[IV13]], 1
 ; CHECK-UNROLL-T2: [[IV15:%[a-z.0-9]+]] = add nuw nsw i32 [[IV14]], 1
 ; CHECK-UNROLL-T2: [[IV16]] = add nuw nsw i32 [[IV15]], 1
 ; CHECK-UNROLL-T2: [[CMP:%[a-z.0-9]+]] = icmp eq i32 [[IV16]], 1024
 ; CHECK-UNROLL-T2: br i1 [[CMP]], label [[END:%[a-z.]+]], label %for.body
 
+; CHECK-UNROLL-LOB: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV16:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-LOB: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-LOB: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-LOB: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-LOB: [[IV4:%[a-z.0-9]+]] = add nuw nsw i32 [[IV3]], 1
+; CHECK-UNROLL-LOB: [[IV5:%[a-z.0-9]+]] = add nuw nsw i32 [[IV4]], 1
+; CHECK-UNROLL-LOB: [[IV6:%[a-z.0-9]+]] = add nuw nsw i32 [[IV5]], 1
+; CHECK-UNROLL-LOB: [[IV7:%[a-z.0-9]+]] = add nuw nsw i32 [[IV6]], 1
+; CHECK-UNROLL-LOB: [[IV8:%[a-z.0-9]+]] = add nuw nsw i32 [[IV7]], 1
+; CHECK-UNROLL-LOB: [[IV9:%[a-z.0-9]+]] = add nuw nsw i32 [[IV8]], 1
+; CHECK-UNROLL-LOB: [[IV10:%[a-z.0-9]+]] = add nuw nsw i32 [[IV9]], 1
+; CHECK-UNROLL-LOB: [[IV11:%[a-z.0-9]+]] = add nuw nsw i32 [[IV10]], 1
+; CHECK-UNROLL-LOB: [[IV12:%[a-z.0-9]+]] = add nuw nsw i32 [[IV11]], 1
+; CHECK-UNROLL-LOB: [[IV13:%[a-z.0-9]+]] = add nuw nsw i32 [[IV12]], 1
+; CHECK-UNROLL-LOB: [[IV14:%[a-z.0-9]+]] = add nuw nsw i32 [[IV13]], 1
+; CHECK-UNROLL-LOB: [[IV15:%[a-z.0-9]+]] = add nuw nsw i32 [[IV14]], 1
+; CHECK-UNROLL-LOB: [[IV16]] = add nuw nsw i32 [[IV15]], 1
+; CHECK-UNROLL-LOB: [[CMP:%[a-z.0-9]+]] = icmp eq i32 [[IV16]], 1024
+; CHECK-UNROLL-LOB: br i1 [[CMP]], label [[END:%[a-z.]+]], label %for.body
+
   %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.08
   %0 = load i32, i32* %arrayidx, align 4
   %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.08
   %1 = load i32, i32* %arrayidx1, align 4
   %mul = mul nsw i32 %1, %0
   %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.08
   store i32 %mul, i32* %arrayidx2, align 4

 ; CHECK-UNROLL-T2: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
 ; CHECK-UNROLL-T2: [[IV4]] = add nuw i32 [[IV3]], 1
 ; CHECK-UNROLL-T2: br
 
 ; CHECK-UNROLL-T2: for.body.epil:
 ; CHECK-UNROLL-T2: for.body.epil.1:
 ; CHECK-UNROLL-T2: for.body.epil.2:
 
+; CHECK-UNROLL-LOB: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z.0-9]+]] ], [ [[IV4:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-LOB: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-LOB: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-LOB: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-LOB: [[IV4]] = add nuw i32 [[IV3]], 1
+; CHECK-UNROLL-LOB: br
+
+; CHECK-UNROLL-LOB: for.body.epil:
+; CHECK-UNROLL-LOB-NOT: for.body.epil.1:
+
   %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.09
   %0 = load i32, i32* %arrayidx, align 4
   %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.09
   %1 = load i32, i32* %arrayidx1, align 4
   %mul = mul nsw i32 %1, %0
   %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.09
   store i32 %mul, i32* %arrayidx2, align 4

 ; CHECK-UNROLL-T2: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
 ; CHECK-UNROLL-T2: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
 ; CHECK-UNROLL-T2: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
 ; CHECK-UNROLL-T2: [[IV4]] = add nuw i32 [[IV3]], 1
 ; CHECK-UNROLL-T2: br
 ; CHECK-UNROLL-T2: for.body4.epil.1:
 ; CHECK-UNROLL-T2: for.body4.epil.2:
 
+; CHECK-UNROLL-LOB: for.body4.epil:
+; CHECK-UNROLL-LOB: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z0-9.]+]] ], [ [[IV4:%[a-z.0-9]+]], %for.body4 ]
+; CHECK-UNROLL-LOB: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-LOB: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-LOB: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-LOB: [[IV4]] = add nuw i32 [[IV3]], 1
+; CHECK-UNROLL-LOB: br
+; CHECK-UNROLL-LOB-NOT: for.body4.epil.1:
+
   %w.024 = phi i32 [ 0, %for.body4.lr.ph ], [ %inc, %for.body4 ]
   %add = add i32 %w.024, %mul
   %arrayidx = getelementptr inbounds i16, i16* %A, i32 %add
   %0 = load i16, i16* %arrayidx, align 2
   %conv = sext i16 %0 to i32
   %arrayidx5 = getelementptr inbounds i16, i16* %B, i32 %w.024
   %1 = load i16, i16* %arrayidx5, align 2
   %conv6 = sext i16 %1 to i32

 ; CHECK-UNROLL-A: br i1 %tobool
 ; CHECK-UNROLL-A-NOT: load
 
 ; CHECK-UNROLL-T1: %n.addr.04 = phi %struct.Node* [ %1, %while.body ], [ %n, %while.body.preheader ]
 ; CHECK-UNROLL-T1: %tobool = icmp eq %struct.Node* %1, null
 ; CHECK-UNROLL-T1: br i1 %tobool
 ; CHECK-UNROLL-T1-NOT: load
 
+; CHECK-UNROLL-LOB: %n.addr.04 = phi %struct.Node* [ %1, %while.body ], [ %n, %while.body.preheader ]
+; CHECK-UNROLL-LOB: %tobool = icmp eq %struct.Node* %1, null
+; CHECK-UNROLL-LOB: br i1 %tobool
+; CHECK-UNROLL-LOB-NOT: load
+
 ; CHECK-UNROLL-T2: [[CMP0:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR0:%[a-z.0-9]+]], null
 ; CHECK-UNROLL-T2: br i1 [[CMP0]], label [[END:%[a-z.0-9]+]]
 ; CHECK-UNROLL-T2: [[CMP1:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR1:%[a-z.0-9]+]], null
 ; CHECK-UNROLL-T2: br i1 [[CMP1]], label [[END]]
 ; CHECK-UNROLL-T2: [[CMP2:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR2:%[a-z.0-9]+]], null
 ; CHECK-UNROLL-T2: br i1 [[CMP2]], label [[END]]
 ; CHECK-UNROLL-T2: [[CMP3:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR3:%[a-z.0-9]+]], null
 ; CHECK-UNROLL-T2: br i1 [[CMP3]], label [[END]]