[ARM] Fold predicate_cast(load) into vldr p0

[dmgreen]

Summary

This adds a simple tablegen pattern for folding predicate_cast(load) into vldr p0, providing the alignment and offset are correct.

[SjoerdMeijer]

Sorry for kind of asking the usual testing question....but was curious if there's a negative test with a pattern where its condition isn't met, so alignment < 4, if that makes sense.

[dmgreen]

Oh yeah :(

[dmgreen]

align2 test now actually has an alignment of 2.

[dmgreen]

Ping

[simon_tatham]

llvm/lib/Target/ARM/ARMInstrMVE.td

4388

It seems surprising that there isn't a ready-made family of PatFrags like this one somewhere central where all targets can use it! But I had a quick look, didn't see any, and did see that several targets define their own, so there we go.

llvm/lib/Target/ARM/ARMInstrMVE.td

 }
 
 // Occasionally we need to cast between a i32 and a boolean vector, for
 // example when moving between rGPR and VPR.P0 as part of predicate vector
 // shuffles. We also sometimes need to cast between different predicate
 // vector types (v4i1<>v8i1, etc.) also as part of lowering vector shuffles.
 def predicate_cast : SDNode<"ARMISD::PREDICATE_CAST", SDTUnaryOp>;
 
+def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAlignment() >= 4;
+}]>;
+
 let Predicates = [HasMVEInt] in {
   foreach VT = [ v4i1, v8i1, v16i1 ] in {
     def : Pat<(i32 (predicate_cast (VT VCCR:$src))),
               (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>;
     def : Pat<(VT  (predicate_cast (i32 VCCR:$src))),
               (VT  (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>;
 
     foreach VT2 = [ v4i1, v8i1, v16i1 ] in
       def : Pat<(VT  (predicate_cast (VT2 VCCR:$src))),
                 (VT  (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>;
   }
 
+  // If we happen to be casting from a load we can convert that straight
+  // into a predicate load, so long as the load is of the correct type.
+  foreach VT = [ v4i1, v8i1, v16i1 ] in {
+    def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))),
+              (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>;
+  }
+
   // Here we match the specific SDNode type 'ARMVectorRegCastImpl'
   // rather than the more general 'ARMVectorRegCast' which would also
   // match some bitconverts. If we use the latter in cases where the
   // input and output types are the same, the bitconvert gets elided
   // and we end up generating a nonsense match of nothing.
 
   foreach VT = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in
     foreach VT2 = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in

llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll

; CHECK-BE-NEXT:    bx lr
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %lz)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %l = load i32, i32* %i, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <8 x i16> @load_predcast8(i32* %i, <8 x i16> %a) {
 ; CHECK-LE-LABEL: load_predcast8:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_predcast8:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0]
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vrev32.16 q0, q0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %l = load i32, i32* %i, align 4
   %c = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %l)
   %s = select <8 x i1> %c, <8 x i16> %a, <8 x i16> zeroinitializer
   ret <8 x i16> %s
 }
 
 define arm_aapcs_vfpcc <16 x i8> @load_predcast16(i32* %i, <16 x i8> %a) {
 ; CHECK-LE-LABEL: load_predcast16:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_predcast16:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0]
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vrev32.8 q0, q0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.8 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %l = load i32, i32* %i, align 4
   %c = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %l)
   %s = select <16 x i1> %c, <16 x i8> %a, <16 x i8> zeroinitializer
   ret <16 x i8> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_align2(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_align2:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_align2:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %l = load i32, i32* %i, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_offset(i16* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_offset:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr.w r0, [r0, #6]
+; CHECK-LE-NEXT:    adds r0, #6
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_offset:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr.w r0, [r0, #6]
+; CHECK-BE-NEXT:    adds r0, #6
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i16, i16* %i, i32 3
   %gb = bitcast i16* %g to i32*
   %l = load i32, i32* %gb, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_range4(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_range4:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr r0, [r0, #4]
+; CHECK-LE-NEXT:    vldr p0, [r0, #4]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_range4:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0, #4]
+; CHECK-BE-NEXT:    vldr p0, [r0, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i32, i32* %i, i32 1
   %l = load i32, i32* %g, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_range(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_range:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr.w r0, [r0, #508]
+; CHECK-LE-NEXT:    vldr p0, [r0, #508]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_range:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr.w r0, [r0, #508]
+; CHECK-BE-NEXT:    vldr p0, [r0, #508]
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i32, i32* %i, i32 127
   %l = load i32, i32* %g, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_range2(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_range2:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    movw r1, #65028
+; CHECK-LE-NEXT:    vldr p0, [r0, #-508]
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    movt r1, #65535
-; CHECK-LE-NEXT:    ldr r0, [r0, r1]
-; CHECK-LE-NEXT:    vmsr p0, r0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_range2:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    movw r1, #65028
+; CHECK-BE-NEXT:    vldr p0, [r0, #-508]
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movt r1, #65535
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    ldr r0, [r0, r1]
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i32, i32* %i, i32 -127
   %l = load i32, i32* %g, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_range3(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_range3:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    ldr.w r0, [r0, #512]
+; CHECK-LE-NEXT:    add.w r0, r0, #512
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_range3:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr.w r0, [r0, #512]
+; CHECK-BE-NEXT:    add.w r0, r0, #512
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i32, i32* %i, i32 128
   %l = load i32, i32* %g, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @load_bc4_range5(i32* %i, <4 x i32> %a) {
 ; CHECK-LE-LABEL: load_bc4_range5:
 ; CHECK-LE:       @ %bb.0:
-; CHECK-LE-NEXT:    movw r1, #65024
+; CHECK-LE-NEXT:    sub.w r0, r0, #512
 ; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-LE-NEXT:    movt r1, #65535
-; CHECK-LE-NEXT:    ldr r0, [r0, r1]
-; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vldr p0, [r0]
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: load_bc4_range5:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    movw r1, #65024
+; CHECK-BE-NEXT:    sub.w r0, r0, #512
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movt r1, #65535
+; CHECK-BE-NEXT:    vldr p0, [r0]
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    ldr r0, [r0, r1]
-; CHECK-BE-NEXT:    vmsr p0, r0
 ; CHECK-BE-NEXT:    vpsel q1, q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
   %g = getelementptr inbounds i32, i32* %i, i32 -128
   %l = load i32, i32* %g, align 4
   %c = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %l)
   %s = select <4 x i1> %c, <4 x i32> %a, <4 x i32> zeroinitializer
   ret <4 x i32> %s
 }
 
 declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
 declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
 declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)