[AArch64] Improve codegen of store lane instructions by avoiding GPR usage

[ab]

Summary

EDIT: I should note that all this only applies to base+offset addressing modes.

We used to generate stuff like

	umov.b	w8, v0[2]
	strb	w8, [x0, x1]

because the STR*ro* patterns were preferred to ST1*.
Instead, generate:

	add	x8, x0, x1
	st1.b	{ v0 }[2], [x8]

This patch increases the ST1* AddedComplexity to achieve that.

Thanks!
-Ahmed

[ab]

Ping

(Thanks for the reviews, Tim!)

-Ahmed

[ab]

Another ping for review!

Thanks,
-Ahmed

[jmolloy]

Hi Ahmed,

This patch would be better split into two. The first one seems fine to me.

For the second one; my local tinkering has suggested that you shouldn't need the "Non0" stuff at all.

You want to raise ST1Lane's AddedComplexity from 15 to 19, because the integer STRWro also has AddedComplexity 15 (and it matches more nodes in the DAG so has an overall higher complexity). This is fine, although it does expose AddedComplexity to be a very blunt hammer.

But if you do that you end up with suboptimal code for lane 0:

add	x8, x0, w1, sxtw #2
st1	{ v0.s }[0], [x8]

(which is different from the code snippet you pasted earlier. I get the above solely from changing the AddedComplexity from 15 to 19 on the ST1Lane patterns)

So surely all you need to do is make the AddedComplexity of your new Lane0 patterns "19". This is because your new patterns will match at least one more node than the ST1 patterns (because they match a RO addr mode).

This way, you're just telling SDAG "Here is a new pattern, it has the same AddedComplexity as the other one - choose at your will". Which is better than telling it it can never pick an ST1Lane for an index 0.

Does that sound sensible?

Cheers,

James

[ab]

Why yes.. yes indeed, it makes lots of sense!

I split the lane0 parts into D6772; both patches are much simpler now.

Thanks a lot for the review!
-Ahmed

[jmolloy]

This looks good to me, thanks for making the changes!

James

llvm/trunk/lib/Target/AArch64/AArch64InstrInfo.td

 defm LD4 : SIMDLdSt4SingleAliases<"ld4">;
 
 // Stores
 defm ST1 : SIMDStSingleB<0, 0b000,       "st1", VecListOneb, GPR64pi1>;
 defm ST1 : SIMDStSingleH<0, 0b010, 0,    "st1", VecListOneh, GPR64pi2>;
 defm ST1 : SIMDStSingleS<0, 0b100, 0b00, "st1", VecListOnes, GPR64pi4>;
 defm ST1 : SIMDStSingleD<0, 0b100, 0b01, "st1", VecListOned, GPR64pi8>;
 
-let AddedComplexity = 15 in
+let AddedComplexity = 19 in
 class St1Lane128Pat<SDPatternOperator scalar_store, Operand VecIndex,
                     ValueType VTy, ValueType STy, Instruction ST1>
   : Pat<(scalar_store
              (STy (vector_extract (VTy VecListOne128:$Vt), VecIndex:$idx)),
              GPR64sp:$Rn),
         (ST1 VecListOne128:$Vt, VecIndex:$idx, GPR64sp:$Rn)>;
 
 def : St1Lane128Pat<truncstorei8,  VectorIndexB, v16i8, i32, ST1i8>;
 def : St1Lane128Pat<truncstorei16, VectorIndexH, v8i16, i32, ST1i16>;
 def : St1Lane128Pat<store,         VectorIndexS, v4i32, i32, ST1i32>;
 def : St1Lane128Pat<store,         VectorIndexS, v4f32, f32, ST1i32>;
 def : St1Lane128Pat<store,         VectorIndexD, v2i64, i64, ST1i64>;
 def : St1Lane128Pat<store,         VectorIndexD, v2f64, f64, ST1i64>;
 def : St1Lane128Pat<store,         VectorIndexH, v8f16, f16, ST1i16>;
 
-let AddedComplexity = 15 in
+let AddedComplexity = 19 in
 class St1Lane64Pat<SDPatternOperator scalar_store, Operand VecIndex,
                    ValueType VTy, ValueType STy, Instruction ST1>
   : Pat<(scalar_store
              (STy (vector_extract (VTy VecListOne64:$Vt), VecIndex:$idx)),
              GPR64sp:$Rn),
         (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
              VecIndex:$idx, GPR64sp:$Rn)>;
 

llvm/trunk/test/CodeGen/AArch64/arm64-st1.ll

 ; RUN: llc < %s -march=arm64 -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s
 
 define void @st1lane_16b(<16 x i8> %A, i8* %D) {
 ; CHECK-LABEL: st1lane_16b
 ; CHECK: st1.b
   %tmp = extractelement <16 x i8> %A, i32 1
   store i8 %tmp, i8* %D
   ret void
 }
 
+define void @st1lane_ro_16b(<16 x i8> %A, i8* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_16b
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.b { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i8* %D, i64 %offset
+  %tmp = extractelement <16 x i8> %A, i32 1
+  store i8 %tmp, i8* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_16b(<16 x i8> %A, i8* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_16b
-; CHECK: umov.b w[[WREG:[0-9]+]], v0[0]
-; CHECK: strb   w[[WREG]], [x0, x1]
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.b { v0 }[0], [x[[XREG]]]
   %ptr = getelementptr i8* %D, i64 %offset
   %tmp = extractelement <16 x i8> %A, i32 0
   store i8 %tmp, i8* %ptr
   ret void
 }
 
 define void @st1lane_8h(<8 x i16> %A, i16* %D) {
 ; CHECK-LABEL: st1lane_8h
 ; CHECK: st1.h
   %tmp = extractelement <8 x i16> %A, i32 1
   store i16 %tmp, i16* %D
   ret void
 }
 
+define void @st1lane_ro_8h(<8 x i16> %A, i16* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_8h
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.h { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i16* %D, i64 %offset
+  %tmp = extractelement <8 x i16> %A, i32 1
+  store i16 %tmp, i16* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_8h(<8 x i16> %A, i16* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_8h
 ; CHECK: str h0, [x0, x1, lsl #1]
   %ptr = getelementptr i16* %D, i64 %offset
   %tmp = extractelement <8 x i16> %A, i32 0
   store i16 %tmp, i16* %ptr
   ret void
 }
 
 define void @st1lane_4s(<4 x i32> %A, i32* %D) {
 ; CHECK-LABEL: st1lane_4s
 ; CHECK: st1.s
   %tmp = extractelement <4 x i32> %A, i32 1
   store i32 %tmp, i32* %D
   ret void
 }
 
+define void @st1lane_ro_4s(<4 x i32> %A, i32* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_4s
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.s { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i32* %D, i64 %offset
+  %tmp = extractelement <4 x i32> %A, i32 1
+  store i32 %tmp, i32* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_4s(<4 x i32> %A, i32* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_4s
 ; CHECK: str s0, [x0, x1, lsl #2]
   %ptr = getelementptr i32* %D, i64 %offset
   %tmp = extractelement <4 x i32> %A, i32 0
   store i32 %tmp, i32* %ptr
   ret void
 }
 
 define void @st1lane_4s_float(<4 x float> %A, float* %D) {
 ; CHECK-LABEL: st1lane_4s_float
 ; CHECK: st1.s
   %tmp = extractelement <4 x float> %A, i32 1
   store float %tmp, float* %D
   ret void
 }
 
+define void @st1lane_ro_4s_float(<4 x float> %A, float* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_4s_float
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.s { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr float* %D, i64 %offset
+  %tmp = extractelement <4 x float> %A, i32 1
+  store float %tmp, float* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_4s_float(<4 x float> %A, float* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_4s_float
 ; CHECK: str s0, [x0, x1, lsl #2]
   %ptr = getelementptr float* %D, i64 %offset
   %tmp = extractelement <4 x float> %A, i32 0
   store float %tmp, float* %ptr
   ret void
 }
 
 define void @st1lane_2d(<2 x i64> %A, i64* %D) {
 ; CHECK-LABEL: st1lane_2d
 ; CHECK: st1.d
   %tmp = extractelement <2 x i64> %A, i32 1
   store i64 %tmp, i64* %D
   ret void
 }
 
+define void @st1lane_ro_2d(<2 x i64> %A, i64* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_2d
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.d { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i64* %D, i64 %offset
+  %tmp = extractelement <2 x i64> %A, i32 1
+  store i64 %tmp, i64* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_2d(<2 x i64> %A, i64* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_2d
 ; CHECK: str d0, [x0, x1, lsl #3]
   %ptr = getelementptr i64* %D, i64 %offset
   %tmp = extractelement <2 x i64> %A, i32 0
   store i64 %tmp, i64* %ptr
   ret void
 }
 
 define void @st1lane_2d_double(<2 x double> %A, double* %D) {
 ; CHECK-LABEL: st1lane_2d_double
 ; CHECK: st1.d
   %tmp = extractelement <2 x double> %A, i32 1
   store double %tmp, double* %D
   ret void
 }
 
+define void @st1lane_ro_2d_double(<2 x double> %A, double* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_2d_double
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.d { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr double* %D, i64 %offset
+  %tmp = extractelement <2 x double> %A, i32 1
+  store double %tmp, double* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_2d_double(<2 x double> %A, double* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_2d_double
 ; CHECK: str d0, [x0, x1, lsl #3]
   %ptr = getelementptr double* %D, i64 %offset
   %tmp = extractelement <2 x double> %A, i32 0
   store double %tmp, double* %ptr
   ret void
 }
 
 define void @st1lane_8b(<8 x i8> %A, i8* %D) {
 ; CHECK-LABEL: st1lane_8b
 ; CHECK: st1.b
   %tmp = extractelement <8 x i8> %A, i32 1
   store i8 %tmp, i8* %D
   ret void
 }
 
+define void @st1lane_ro_8b(<8 x i8> %A, i8* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_8b
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.b { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i8* %D, i64 %offset
+  %tmp = extractelement <8 x i8> %A, i32 1
+  store i8 %tmp, i8* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_8b(<8 x i8> %A, i8* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_8b
-; CHECK: umov.b w[[WREG:[0-9]+]], v0[0]
-; CHECK: strb   w[[WREG]], [x0, x1]
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.b { v0 }[0], [x[[XREG]]]
   %ptr = getelementptr i8* %D, i64 %offset
   %tmp = extractelement <8 x i8> %A, i32 0
   store i8 %tmp, i8* %ptr
   ret void
 }
 
 define void @st1lane_4h(<4 x i16> %A, i16* %D) {
 ; CHECK-LABEL: st1lane_4h
 ; CHECK: st1.h
   %tmp = extractelement <4 x i16> %A, i32 1
   store i16 %tmp, i16* %D
   ret void
 }
 
+define void @st1lane_ro_4h(<4 x i16> %A, i16* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_4h
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.h { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i16* %D, i64 %offset
+  %tmp = extractelement <4 x i16> %A, i32 1
+  store i16 %tmp, i16* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_4h(<4 x i16> %A, i16* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_4h
 ; CHECK: str h0, [x0, x1, lsl #1]
   %ptr = getelementptr i16* %D, i64 %offset
   %tmp = extractelement <4 x i16> %A, i32 0
   store i16 %tmp, i16* %ptr
   ret void
 }
 
 define void @st1lane_2s(<2 x i32> %A, i32* %D) {
 ; CHECK-LABEL: st1lane_2s
 ; CHECK: st1.s
   %tmp = extractelement <2 x i32> %A, i32 1
   store i32 %tmp, i32* %D
   ret void
 }
 
+define void @st1lane_ro_2s(<2 x i32> %A, i32* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_2s
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.s { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr i32* %D, i64 %offset
+  %tmp = extractelement <2 x i32> %A, i32 1
+  store i32 %tmp, i32* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_2s(<2 x i32> %A, i32* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_2s
 ; CHECK: str s0, [x0, x1, lsl #2]
   %ptr = getelementptr i32* %D, i64 %offset
   %tmp = extractelement <2 x i32> %A, i32 0
   store i32 %tmp, i32* %ptr
   ret void
 }
 
 define void @st1lane_2s_float(<2 x float> %A, float* %D) {
 ; CHECK-LABEL: st1lane_2s_float
 ; CHECK: st1.s
   %tmp = extractelement <2 x float> %A, i32 1
   store float %tmp, float* %D
   ret void
 }
 
+define void @st1lane_ro_2s_float(<2 x float> %A, float* %D, i64 %offset) {
+; CHECK-LABEL: st1lane_ro_2s_float
+; CHECK: add x[[XREG:[0-9]+]], x0, x1
+; CHECK: st1.s { v0 }[1], [x[[XREG]]]
+  %ptr = getelementptr float* %D, i64 %offset
+  %tmp = extractelement <2 x float> %A, i32 1
+  store float %tmp, float* %ptr
+  ret void
+}
+
 define void @st1lane0_ro_2s_float(<2 x float> %A, float* %D, i64 %offset) {
 ; CHECK-LABEL: st1lane0_ro_2s_float
 ; CHECK: str s0, [x0, x1, lsl #2]
   %ptr = getelementptr float* %D, i64 %offset
   %tmp = extractelement <2 x float> %A, i32 0
   store float %tmp, float* %ptr
   ret void
 }