[ARM] Sink splat to ICmp

[dmgreen]

Summary

Nice and simple, this adds ICmp to the list of instructions that we sink a splat to in a loop, allowing the register forms of instructions to be selected more. I have not handled FCmp yet as the results look a little odd, trying to keep the register in an float reg and having to move it to a GPR.

[samparker]

LGTM

[dmgreen]

Thanks. Turned out this wasn't quite as simple as I was thinking. First it broke tail predication, the new version here should fix that, it's just a case of what is considered "loop invariant".

The other thing is I still had D66580 applied when making this patch, and the tests don't look as good with it applied. Still fine in comparison, but we should really get back to that one.

llvm/lib/Target/ARM/ARMISelLowering.cpp

bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
 
   if (!Subtarget->hasMVEIntegerOps())
     return false;
 
   auto IsSinker = [](Instruction *I, int Operand) {
     switch (I->getOpcode()) {
     case Instruction::Add:
     case Instruction::Mul:
+    case Instruction::ICmp:
       return true;
     case Instruction::Sub:
       return Operand == 1;
     default:
       return false;
     }
   };
 

llvm/lib/Target/ARM/MVETailPredication.cpp

bool MVETailPredication::isTailPredicate(Instruction *I, Value *NumElements) {
 
   CmpInst::Predicate Pred;
   Instruction *Shuffle = nullptr;
   Instruction *Induction = nullptr;
 
   // The vector icmp
   if (!match(I, m_ICmp(Pred, m_Instruction(Induction),
                        m_Instruction(Shuffle))) ||
-      Pred != ICmpInst::ICMP_ULE || !L->isLoopInvariant(Shuffle))
+      Pred != ICmpInst::ICMP_ULE)
     return false;
 
   // First find the stuff outside the loop which is setting up the limit
   // vector....
   // The invariant shuffle that broadcast the limit into a vector.
   Instruction *Insert = nullptr;
   if (!match(Shuffle, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
                                       m_Zero())))
     return false;
 
   // Insert the limit into a vector.
   Instruction *BECount = nullptr;
   if (!match(Insert, m_InsertElement(m_Undef(), m_Instruction(BECount),
                                      m_Zero())))
     return false;
 
   // The limit calculation, backedge count.
   Value *TripCount = nullptr;
   if (!match(BECount, m_Add(m_Value(TripCount), m_AllOnes())))
     return false;
 
-  if (TripCount != NumElements)
+  if (TripCount != NumElements || !L->isLoopInvariant(BECount))
     return false;
 
   // Now back to searching inside the loop body...
   // Find the add with takes the index iv and adds a constant vector to it. 
   Instruction *BroadcastSplat = nullptr;
   Constant *Const = nullptr;
   if (!match(Induction, m_Add(m_Instruction(BroadcastSplat),
                               m_Constant(Const))))

llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc void @thres_i32(i32* %data, i16 zeroext %N, i32 %T) {
 ; CHECK-LABEL: thres_i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #3
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #2
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.32 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vpt.s32 le, q0, q3
-; CHECK-NEXT:    vcmpt.s32 le, q3, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s32 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv, 2
   %cmp15 = icmp eq i16 %N, 0
   br i1 %cmp15, label %for.cond.cleanup, label %vector.ph

 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #7
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #3
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.16 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #3
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.16 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q3, [r0]
-; CHECK-NEXT:    vpt.s16 le, q0, q3
-; CHECK-NEXT:    vcmpt.s16 le, q3, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s16 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    vstrht.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv2 = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv2, 3
   %cmp22 = icmp eq i16 %N, 0
   br i1 %cmp22, label %for.cond.cleanup, label %vector.ph

 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #15
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #4
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.8 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #4
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.8 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u8 q3, [r0]
-; CHECK-NEXT:    vpt.s8 le, q0, q3
-; CHECK-NEXT:    vcmpt.s8 le, q3, q1
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s8 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q2, [r0], #16
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv2 = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv2, 4
   %cmp20 = icmp eq i16 %N, 0
   br i1 %cmp20, label %for.cond.cleanup, label %vector.ph

 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #3
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #2
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.32 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB5_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vpt.s32 le, q0, q3
-; CHECK-NEXT:    vcmpt.s32 le, q3, q1
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s32 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB5_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv, 2
   %cmp15 = icmp eq i16 %N, 0
   br i1 %cmp15, label %for.cond.cleanup, label %vector.ph

 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #7
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #3
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.16 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #3
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.16 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q3, [r0]
-; CHECK-NEXT:    vpt.s16 le, q0, q3
-; CHECK-NEXT:    vcmpt.s16 le, q3, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s16 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    vstrht.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv2 = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv2, 3
   %cmp22 = icmp eq i16 %N, 0
   br i1 %cmp22, label %for.cond.cleanup, label %vector.ph

 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:    mvn r3, #15
 ; CHECK-NEXT:    add.w r1, r3, r1, lsl #4
 ; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    vdup.8 q0, r2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r1, lsr #4
 ; CHECK-NEXT:    rsbs r1, r2, #0
-; CHECK-NEXT:    vdup.8 q1, r1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u8 q3, [r0]
-; CHECK-NEXT:    vpt.s8 le, q0, q3
-; CHECK-NEXT:    vcmpt.s8 le, q3, q1
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vcmpt.s8 le, q1, r1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q2, [r0], #16
+; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %conv2 = zext i16 %N to i32
   %mul = shl nuw nsw i32 %conv2, 4
   %cmp20 = icmp eq i16 %N, 0
   br i1 %cmp20, label %for.cond.cleanup, label %vector.ph