[ARM] Enable and/cmp0 folding

[chill]

Summary

The CodeGenPrepare pass can sink bitwise and used by compare to
zero into the basic blocks where the users are. This operation is
guarded by lowering hook, which is disabled for ARM. In the ARM
architecture versions from v7-M up these two operations can be folded
into tst rN, #imm instruction. Sinking of and can also enable
the cmov-to-bfi DAG combiner.

This patch fixes some benchmark regressions caused
by https://reviews.llvm.org/D129370 as well scoring slightly better overall.

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

21204

Can this be called with a typesize > 64bits?

[chill]

llvm/lib/Target/ARM/ARMISelLowering.cpp

21204

Good point, I guess any bit width may come this way.

[efriedma]

llvm/lib/Target/ARM/ARMISelLowering.cpp

21204

getSOImmVal() only applies in ARM mode, right? The Thumb equivalent is getT2SOImmVal().

[chill]

llvm/lib/Target/ARM/ARMISelLowering.cpp

21204

Thanks! Fixed.

[dmgreen]

Thanks. LGTM

llvm/lib/Target/ARM/ARMISelLowering.h

public:
     /// or null if the target does not support "fast" ISel.
     FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                              const TargetLibraryInfo *libInfo) const override;
 
     Sched::Preference getSchedulingPreference(SDNode *N) const override;
 
     bool preferZeroCompareBranch() const override { return true; }
 
+    bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
+
     bool
     isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 
     /// isFPImmLegal - Returns true if the target can instruction select the
     /// specified FP immediate natively. If false, the legalizer will
     /// materialize the FP immediate as a load from a constant pool.
     bool isFPImmLegal(const APFloat &Imm, EVT VT,

llvm/lib/Target/ARM/ARMISelLowering.cpp

 bool ARMTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
   return Subtarget->hasV6T2Ops();
 }
 
 bool ARMTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
   return Subtarget->hasV6T2Ops();
 }
 
+bool ARMTargetLowering::isMaskAndCmp0FoldingBeneficial(
+    const Instruction &AndI) const {
+  if (!Subtarget->hasV7Ops())
+    return false;
+
+  // Sink the `and` instruction only if the mask would fit into a modified
+  // immediate operand.
+  ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
+  if (!Mask || Mask->getValue().getBitWidth() > 32u)
+    return false;
+  return ARM_AM::getSOImmVal(unsigned(Mask->getValue().getZExtValue())) != -1;
+}
+
 bool ARMTargetLowering::shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
   return !Subtarget->hasMinSize() || Subtarget->isTargetWindows();
 }
 
 Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
                                          Value *Addr,
                                          AtomicOrdering Ord) const {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();

llvm/test/CodeGen/ARM/and-cmp0-sink.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv7m-eabi  %s -o - | FileCheck %s --check-prefix V7M
 ; RUN: llc -mtriple=armv7a-eabi  %s -o -   | FileCheck %s --check-prefix V7A
 ; RUN: llc -mtriple=thumbv7a-eabi  %s -o -   | FileCheck %s --check-prefix V7A-T
 ; RUN: llc -mtriple=armv6m-eabi  %s -o -   | FileCheck %s --check-prefix V6M
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define void @f(i32 %v, ptr noalias  %outp) {
 ; V7M-LABEL: f:
 ; V7M:       @ %bb.0: @ %entry
 ; V7M-NEXT:    movs r2, #0
-; V7M-NEXT:    and r12, r0, #14
 ; V7M-NEXT:    str r2, [r1]
-; V7M-NEXT:    and r3, r0, #4
-; V7M-NEXT:    and r2, r0, #2
-; V7M-NEXT:    lsls r0, r0, #31
-; V7M-NEXT:    bne .LBB0_2
+; V7M-NEXT:    lsls r2, r0, #31
+; V7M-NEXT:    bne .LBB0_3
 ; V7M-NEXT:  @ %bb.1: @ %if.then
-; V7M-NEXT:    cmp r2, #0
-; V7M-NEXT:    it ne
-; V7M-NEXT:    movne.w r2, #33024
-; V7M-NEXT:    cmp r3, #0
-; V7M-NEXT:    it ne
-; V7M-NEXT:    addne.w r2, r2, #16512
-; V7M-NEXT:    b .LBB0_3
-; V7M-NEXT:  .LBB0_2: @ %if.else
-; V7M-NEXT:    cmp r2, #0
-; V7M-NEXT:    it ne
-; V7M-NEXT:    movne.w r2, #8256
-; V7M-NEXT:    cmp r3, #0
-; V7M-NEXT:    it ne
-; V7M-NEXT:    addne.w r2, r2, #4128
+; V7M-NEXT:    tst.w r0, #14
+; V7M-NEXT:    beq .LBB0_6
+; V7M-NEXT:  @ %bb.2:
+; V7M-NEXT:    lsls r2, r0, #30
+; V7M-NEXT:    mov.w r3, #33024
+; V7M-NEXT:    and.w r2, r3, r2, asr #31
+; V7M-NEXT:    lsrs r0, r0, #2
+; V7M-NEXT:    bfi r2, r0, #7, #1
+; V7M-NEXT:    bfi r2, r0, #14, #1
+; V7M-NEXT:    b .LBB0_5
 ; V7M-NEXT:  .LBB0_3: @ %if.else
-; V7M-NEXT:    cmp.w r12, #0
-; V7M-NEXT:    it ne
-; V7M-NEXT:    strne r2, [r1]
+; V7M-NEXT:    tst.w r0, #14
+; V7M-NEXT:    it eq
+; V7M-NEXT:    bxeq lr
+; V7M-NEXT:  .LBB0_4:
+; V7M-NEXT:    lsls r2, r0, #30
+; V7M-NEXT:    mov.w r3, #8256
+; V7M-NEXT:    and.w r2, r3, r2, asr #31
+; V7M-NEXT:    lsrs r0, r0, #2
+; V7M-NEXT:    bfi r2, r0, #5, #1
+; V7M-NEXT:    bfi r2, r0, #12, #1
+; V7M-NEXT:  .LBB0_5: @ %if.end
+; V7M-NEXT:    str r2, [r1]
+; V7M-NEXT:  .LBB0_6: @ %exit
 ; V7M-NEXT:    bx lr
 ;
 ; V7A-LABEL: f:
 ; V7A:       @ %bb.0: @ %entry
 ; V7A-NEXT:    mov r2, #0
-; V7A-NEXT:    and r12, r0, #14
-; V7A-NEXT:    str r2, [r1]
-; V7A-NEXT:    and r3, r0, #4
-; V7A-NEXT:    and r2, r0, #2
 ; V7A-NEXT:    tst r0, #1
-; V7A-NEXT:    bne .LBB0_2
+; V7A-NEXT:    str r2, [r1]
+; V7A-NEXT:    bne .LBB0_3
 ; V7A-NEXT:  @ %bb.1: @ %if.then
-; V7A-NEXT:    cmp r2, #0
-; V7A-NEXT:    movw r0, #16512
-; V7A-NEXT:    movwne r2, #33024
-; V7A-NEXT:    b .LBB0_3
-; V7A-NEXT:  .LBB0_2: @ %if.else
-; V7A-NEXT:    cmp r2, #0
-; V7A-NEXT:    movw r0, #4128
-; V7A-NEXT:    movwne r2, #8256
+; V7A-NEXT:    tst r0, #14
+; V7A-NEXT:    beq .LBB0_6
+; V7A-NEXT:  @ %bb.2:
+; V7A-NEXT:    lsl r2, r0, #30
+; V7A-NEXT:    mov r3, #33024
+; V7A-NEXT:    and r2, r3, r2, asr #31
+; V7A-NEXT:    lsr r0, r0, #2
+; V7A-NEXT:    bfi r2, r0, #7, #1
+; V7A-NEXT:    bfi r2, r0, #14, #1
+; V7A-NEXT:    b .LBB0_5
 ; V7A-NEXT:  .LBB0_3: @ %if.else
-; V7A-NEXT:    cmp r3, #0
-; V7A-NEXT:    orrne r2, r2, r0
-; V7A-NEXT:    cmp r12, #0
-; V7A-NEXT:    strne r2, [r1]
+; V7A-NEXT:    tst r0, #14
+; V7A-NEXT:    bxeq lr
+; V7A-NEXT:  .LBB0_4:
+; V7A-NEXT:    lsl r2, r0, #30
+; V7A-NEXT:    mov r3, #8256
+; V7A-NEXT:    and r2, r3, r2, asr #31
+; V7A-NEXT:    lsr r0, r0, #2
+; V7A-NEXT:    bfi r2, r0, #5, #1
+; V7A-NEXT:    bfi r2, r0, #12, #1
+; V7A-NEXT:  .LBB0_5: @ %if.end
+; V7A-NEXT:    str r2, [r1]
+; V7A-NEXT:  .LBB0_6: @ %exit
 ; V7A-NEXT:    bx lr
 ;
 ; V7A-T-LABEL: f:
 ; V7A-T:       @ %bb.0: @ %entry
 ; V7A-T-NEXT:    movs r2, #0
-; V7A-T-NEXT:    and r12, r0, #14
 ; V7A-T-NEXT:    str r2, [r1]
-; V7A-T-NEXT:    and r3, r0, #4
-; V7A-T-NEXT:    and r2, r0, #2
-; V7A-T-NEXT:    lsls r0, r0, #31
-; V7A-T-NEXT:    bne .LBB0_2
+; V7A-T-NEXT:    lsls r2, r0, #31
+; V7A-T-NEXT:    bne .LBB0_3
 ; V7A-T-NEXT:  @ %bb.1: @ %if.then
-; V7A-T-NEXT:    cmp r2, #0
-; V7A-T-NEXT:    it ne
-; V7A-T-NEXT:    movne.w r2, #33024
-; V7A-T-NEXT:    cmp r3, #0
-; V7A-T-NEXT:    it ne
-; V7A-T-NEXT:    addne.w r2, r2, #16512
-; V7A-T-NEXT:    b .LBB0_3
-; V7A-T-NEXT:  .LBB0_2: @ %if.else
-; V7A-T-NEXT:    cmp r2, #0
-; V7A-T-NEXT:    it ne
-; V7A-T-NEXT:    movne.w r2, #8256
-; V7A-T-NEXT:    cmp r3, #0
-; V7A-T-NEXT:    it ne
-; V7A-T-NEXT:    addne.w r2, r2, #4128
+; V7A-T-NEXT:    tst.w r0, #14
+; V7A-T-NEXT:    beq .LBB0_6
+; V7A-T-NEXT:  @ %bb.2:
+; V7A-T-NEXT:    lsls r2, r0, #30
+; V7A-T-NEXT:    mov.w r3, #33024
+; V7A-T-NEXT:    and.w r2, r3, r2, asr #31
+; V7A-T-NEXT:    lsrs r0, r0, #2
+; V7A-T-NEXT:    bfi r2, r0, #7, #1
+; V7A-T-NEXT:    bfi r2, r0, #14, #1
+; V7A-T-NEXT:    b .LBB0_5
 ; V7A-T-NEXT:  .LBB0_3: @ %if.else
-; V7A-T-NEXT:    cmp.w r12, #0
-; V7A-T-NEXT:    it ne
-; V7A-T-NEXT:    strne r2, [r1]
+; V7A-T-NEXT:    tst.w r0, #14
+; V7A-T-NEXT:    it eq
+; V7A-T-NEXT:    bxeq lr
+; V7A-T-NEXT:  .LBB0_4:
+; V7A-T-NEXT:    lsls r2, r0, #30
+; V7A-T-NEXT:    mov.w r3, #8256
+; V7A-T-NEXT:    and.w r2, r3, r2, asr #31
+; V7A-T-NEXT:    lsrs r0, r0, #2
+; V7A-T-NEXT:    bfi r2, r0, #5, #1
+; V7A-T-NEXT:    bfi r2, r0, #12, #1
+; V7A-T-NEXT:  .LBB0_5: @ %if.end
+; V7A-T-NEXT:    str r2, [r1]
+; V7A-T-NEXT:  .LBB0_6: @ %exit
 ; V7A-T-NEXT:    bx lr
 ;
 ; V6M-LABEL: f:
 ; V6M:       @ %bb.0: @ %entry
 ; V6M-NEXT:    .save {r4, lr}
 ; V6M-NEXT:    push {r4, lr}
 ; V6M-NEXT:    movs r2, #0
 ; V6M-NEXT:    str r2, [r1]

llvm/test/CodeGen/Thumb2/mve-float16regloops.ll

 ; CHECK-LABEL: arm_biquad_cascade_df2T_f16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    ldrd r12, r6, [r0, #4]
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    ldrb r0, [r0]
-; CHECK-NEXT:    and r8, r3, #1
+; CHECK-NEXT:    ldrd r12, r6, [r0, #4]
+; CHECK-NEXT:    ldrb.w r9, [r0]
 ; CHECK-NEXT:    vldr.16 s0, .LCPI17_0
-; CHECK-NEXT:    lsr.w r9, r3, #1
+; CHECK-NEXT:    lsr.w r8, r3, #1
 ; CHECK-NEXT:    b .LBB17_3
 ; CHECK-NEXT:  .LBB17_1: @ %if.else
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
 ; CHECK-NEXT:    vmovx.f16 s5, s4
 ; CHECK-NEXT:    vstr.16 s4, [r12]
 ; CHECK-NEXT:  .LBB17_2: @ %if.end
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
 ; CHECK-NEXT:    vstr.16 s5, [r12, #2]
 ; CHECK-NEXT:    adds r6, #10
-; CHECK-NEXT:    subs r0, #1
+; CHECK-NEXT:    subs.w r9, r9, #1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    beq .LBB17_8
 ; CHECK-NEXT:  .LBB17_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB17_5 Depth 2
 ; CHECK-NEXT:    vldrh.u16 q2, [r6]
 ; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    vmov q4, q2
 ; CHECK-NEXT:    vshlc q4, r5, #16
 ; CHECK-NEXT:    vldrh.u16 q3, [r6, #4]
 ; CHECK-NEXT:    vmov q5, q3
 ; CHECK-NEXT:    vshlc q5, r5, #16
 ; CHECK-NEXT:    vldrh.u16 q1, [r12]
 ; CHECK-NEXT:    vmov.f32 s5, s1
 ; CHECK-NEXT:    mov r5, r2
-; CHECK-NEXT:    wls lr, r9, .LBB17_6
+; CHECK-NEXT:    wls lr, r8, .LBB17_6
 ; CHECK-NEXT:  @ %bb.4: @ %while.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:  .LBB17_5: @ %while.body
 ; CHECK-NEXT:    @ Parent Loop BB17_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrh r7, [r1], #4
-; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vfma.f16 q1, q2, r7
 ; CHECK-NEXT:    ldrh r4, [r1, #-2]
 ; CHECK-NEXT:    vmov.u16 r7, q1[0]
 ; CHECK-NEXT:    vfma.f16 q1, q3, r7
 ; CHECK-NEXT:    vins.f16 s5, s0
 ; CHECK-NEXT:    vfma.f16 q1, q4, r4
 ; CHECK-NEXT:    vmov.u16 r4, q1[1]
 ; CHECK-NEXT:    vfma.f16 q1, q5, r4
 ; CHECK-NEXT:    strh r4, [r5, #2]
 ; CHECK-NEXT:    vmov.f32 s4, s5
 ; CHECK-NEXT:    strh r7, [r5], #4
-; CHECK-NEXT:    vmov.16 q1[2], r3
+; CHECK-NEXT:    vmov.16 q1[2], r0
 ; CHECK-NEXT:    le lr, .LBB17_5
 ; CHECK-NEXT:  .LBB17_6: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    lsls r0, r3, #31
 ; CHECK-NEXT:    beq .LBB17_1
 ; CHECK-NEXT:  @ %bb.7: @ %if.then
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
-; CHECK-NEXT:    ldrh r1, [r1]
-; CHECK-NEXT:    vfma.f16 q1, q2, r1
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vfma.f16 q1, q3, r1
-; CHECK-NEXT:    strh r1, [r5]
+; CHECK-NEXT:    ldrh r0, [r1]
+; CHECK-NEXT:    vfma.f16 q1, q2, r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vfma.f16 q1, q3, r0
+; CHECK-NEXT:    strh r0, [r5]
 ; CHECK-NEXT:    vmovx.f16 s2, s4
 ; CHECK-NEXT:    vstr.16 s2, [r12]
 ; CHECK-NEXT:    b .LBB17_2
 ; CHECK-NEXT:  .LBB17_8: @ %do.end
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 ; CHECK-NEXT:    .p2align 1

llvm/test/CodeGen/Thumb2/mve-float32regloops.ll

 define void @arm_biquad_cascade_df2T_f32(%struct.arm_biquad_cascade_df2T_instance_f32* nocapture readonly %S, float* nocapture readonly %pSrc, float* nocapture %pDst, i32 %blockSize) {
 ; CHECK-LABEL: arm_biquad_cascade_df2T_f32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    ldrd r12, r6, [r0, #4]
-; CHECK-NEXT:    and r8, r3, #1
+; CHECK-NEXT:    lsr.w r8, r3, #1
 ; CHECK-NEXT:    ldrb r0, [r0]
-; CHECK-NEXT:    lsrs r3, r3, #1
 ; CHECK-NEXT:    vldr s0, .LCPI20_0
 ; CHECK-NEXT:    b .LBB20_3
 ; CHECK-NEXT:  .LBB20_1: @ %if.else
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    vmov.f32 s6, s5
 ; CHECK-NEXT:    vstr s4, [r12]
 ; CHECK-NEXT:  .LBB20_2: @ %if.end
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1

 ; CHECK-NEXT:    vshlc q4, r5, #32
 ; CHECK-NEXT:    vldrw.u32 q2, [r6, #8]
 ; CHECK-NEXT:    vmov q5, q2
 ; CHECK-NEXT:    vshlc q5, r5, #32
 ; CHECK-NEXT:    vldrw.u32 q1, [r12]
 ; CHECK-NEXT:    vmov.f32 s6, s0
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:    vmov.f32 s7, s0
-; CHECK-NEXT:    wls lr, r3, .LBB20_6
+; CHECK-NEXT:    wls lr, r8, .LBB20_6
 ; CHECK-NEXT:  @ %bb.4: @ %while.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    vmov q6, q1
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:  .LBB20_5: @ %while.body
 ; CHECK-NEXT:    @ Parent Loop BB20_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrd r7, r4, [r1], #8

 ; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vmov.f32 s4, s6
 ; CHECK-NEXT:    vmov.f32 s5, s7
 ; CHECK-NEXT:    vmov.f32 s6, s0
 ; CHECK-NEXT:    vmov q6, q1
 ; CHECK-NEXT:    le lr, .LBB20_5
 ; CHECK-NEXT:  .LBB20_6: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    lsls r7, r3, #31
 ; CHECK-NEXT:    beq .LBB20_1
 ; CHECK-NEXT:  @ %bb.7: @ %if.then
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    ldr r1, [r1]
 ; CHECK-NEXT:    vfma.f32 q1, q3, r1
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    vstr s4, [r5]
 ; CHECK-NEXT:    vfma.f32 q1, q2, r1