[AMDGPU] Optimized indirect multi-VGPR addressing

[rampitec]

Summary

SelectMOVRELOffset prevents peeling of a constant from an index
if final base could be negative. isBaseWithConstantOffset() succeeds
if a value is an "add" or "or" operator. In case of "or" it shall
be an add-like "or" which never changes a sign of the sum given a
non-negative offset. I.e. we can safely allow peeling if operator is
an "or".

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1905–1907

Couldn't it change the sign for (or 0, 0x80000000)?

[rampitec]

Added check for positive index.

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1905–1907

Thanks. Added check.

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
   if (CurDAG->isBaseWithConstantOffset(Index)) {
     SDValue N0 = Index.getOperand(0);
     SDValue N1 = Index.getOperand(1);
     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
 
     // (add n0, c0)
     // Don't peel off the offset (c0) if doing so could possibly lead
     // the base (n0) to be negative.
-    if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) {
+    // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
+    if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
+        (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
       Base = N0;
       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
       return true;
     }
   }
 
   if (isa<ConstantSDNode>(Index))
     return false;

llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll

 ; GCN:     store_dword v[{{[0-9:]+}}], [[V7]]
 define amdgpu_kernel void @float8_extelt(float addrspace(1)* %out, i32 %sel) {
 entry:
   %ext = extractelement <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, i32 %sel
   store float %ext, float addrspace(1)* %out
   ret void
 }
 
-; TODO: Should be able to copy to m0 only once and increment base instead.
-
 ; GCN-LABEL: {{^}}double8_extelt:
+; GCN-NOT: buffer_
+; GCN-NOT: s_or_b32
 ; GCN-DAG: s_mov_b32 [[ZERO:s[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[BASE:v[0-9]+]], [[ZERO]]
-; GCN-DAG: s_mov_b32 m0, [[IND0:s[0-9]+]]
-; GCN-DAG: s_or_b32 [[IND1:s[0-9]+]], [[IND0]], 1
-; GCN-DAG: v_movrels_b32_e32 v[[RES_LO:[0-9]+]], [[BASE]]
-; GCN:     s_mov_b32 m0, [[IND1:s[0-9]+]]
-; GCN:     v_movrels_b32_e32 v[[RES_HI:[0-9]+]], [[BASE]]
+; GCN-DAG: v_mov_b32_e32 v[[#BASE:]], [[ZERO]]
+; GCN-DAG: s_mov_b32 m0, [[IND:s[0-9]+]]
+; GCN-DAG: v_movrels_b32_e32 v[[RES_LO:[0-9]+]], v[[#BASE]]
+; GCN-DAG: v_movrels_b32_e32 v[[RES_HI:[0-9]+]], v[[#BASE+1]]
 ; GCN:     store_dwordx2 v[{{[0-9:]+}}], v{{\[}}[[RES_LO]]:[[RES_HI]]]
 define amdgpu_kernel void @double8_extelt(double addrspace(1)* %out, i32 %sel) {
 entry:
   %ext = extractelement <8 x double> <double 1.0, double 2.0, double 3.0, double 4.0, double 5.0, double 6.0, double 7.0, double 8.0>, i32 %sel
   store double %ext, double addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}double7_extelt:
+; GCN-NOT: buffer_
+; GCN-NOT: s_or_b32
 ; GCN-DAG: s_mov_b32 [[ZERO:s[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[BASE:v[0-9]+]], [[ZERO]]
-; GCN-DAG: s_mov_b32 m0, [[IND0:s[0-9]+]]
-; GCN-DAG: s_or_b32 [[IND1:s[0-9]+]], [[IND0]], 1
-; GCN-DAG: v_movrels_b32_e32 v[[RES_LO:[0-9]+]], [[BASE]]
-; GCN:     s_mov_b32 m0, [[IND1:s[0-9]+]]
-; GCN:     v_movrels_b32_e32 v[[RES_HI:[0-9]+]], [[BASE]]
+; GCN-DAG: v_mov_b32_e32 v[[#BASE:]], [[ZERO]]
+; GCN-DAG: s_mov_b32 m0, [[IND:s[0-9]+]]
+; GCN-DAG: v_movrels_b32_e32 v[[RES_LO:[0-9]+]], v[[#BASE]]
+; GCN-DAG: v_movrels_b32_e32 v[[RES_HI:[0-9]+]], v[[#BASE+1]]
 ; GCN:     store_dwordx2 v[{{[0-9:]+}}], v{{\[}}[[RES_LO]]:[[RES_HI]]]
 define amdgpu_kernel void @double7_extelt(double addrspace(1)* %out, i32 %sel) {
 entry:
   %ext = extractelement <7 x double> <double 1.0, double 2.0, double 3.0, double 4.0, double 5.0, double 6.0, double 7.0>, i32 %sel
   store double %ext, double addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll

entry:
   %value = extractelement <16 x i32> %ld, i32 %offset
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}extractelement_v16i32_or_index:
 ; GCN: s_load_dword [[IDX_IN:s[0-9]+]]
 ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]]
-; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1
 
-; MOVREL: s_mov_b32 m0, [[IDX_FIN]]
+; MOVREL: s_mov_b32 m0, [[IDX_SHL]]
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 
-; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], gpr_idx(SRC0)
+; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], gpr_idx(SRC0)
 ; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE: s_set_gpr_idx_off
 define amdgpu_kernel void @extractelement_v16i32_or_index(i32 addrspace(1)* %out, <16 x i32> addrspace(1)* %in, i32 %idx.in) {
 entry:
   %ld = load volatile <16 x i32>, <16  x i32> addrspace(1)* %in
   %idx.shl = shl i32 %idx.in, 2
   %idx = or i32 %idx.shl, 1
   %value = extractelement <16 x i32> %ld, i32 %idx
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}insertelement_v16f32_or_index:
 ; GCN: s_load_dword [[IDX_IN:s[0-9]+]]
 ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]]
-; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1
 
-; MOVREL: s_mov_b32 m0, [[IDX_FIN]]
+; MOVREL: s_mov_b32 m0, [[IDX_SHL]]
 ; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 
-; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], gpr_idx(DST)
+; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], gpr_idx(DST)
 ; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE: s_set_gpr_idx_off
 define amdgpu_kernel void @insertelement_v16f32_or_index(<16 x float> addrspace(1)* %out, <16 x float> %a, i32 %idx.in) nounwind {
   %idx.shl = shl i32 %idx.in, 2
   %idx = or i32 %idx.shl, 1
   %vecins = insertelement <16 x float> %a, float 5.000000e+00, i32 %idx
   store <16 x float> %vecins, <16 x float> addrspace(1)* %out, align 64
   ret void

llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll

 ; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, 0, [[CC2]]
 define amdgpu_kernel void @double2_inselt(<2 x double> addrspace(1)* %out, <2 x double> %vec, i32 %sel) {
 entry:
   %v = insertelement <2 x double> %vec, double 1.000000e+00, i32 %sel
   store <2 x double> %v, <2 x double> addrspace(1)* %out
   ret void
 }
 
-; TODO: We should be able not to write to m0 twice and just increment base.
-
 ; GCN-LABEL: {{^}}double8_inselt:
 ; GCN-NOT: v_cndmask
 ; GCN-NOT: buffer_
-; GCN-DAG: s_or_b32 [[IND1:s[0-9]+]], [[IND0:s[0-9]+]], 1
-; GCN-DAG: s_mov_b32 m0, [[IND0]]
-; GCN-DAG: v_movreld_b32_e32 [[BASE:v[0-9]+]],
-; GCN:     s_mov_b32 m0, [[IND1]]
-; GCN:     v_movreld_b32_e32 [[BASE]]
+; GCN-NOT: s_or_b32
+; GCN-DAG: s_mov_b32 m0, [[IND:s[0-9]+]]
+; GCN-DAG: v_movreld_b32_e32 v[[#BASE:]], 0
+; GCN-NOT: s_mov_b32 m0
+; GCN:     v_movreld_b32_e32 v[[#BASE+1]],
 define amdgpu_kernel void @double8_inselt(<8 x double> addrspace(1)* %out, <8 x double> %vec, i32 %sel) {
 entry:
   %v = insertelement <8 x double> %vec, double 1.000000e+00, i32 %sel
   store <8 x double> %v, <8 x double> addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}double7_inselt:
 ; GCN-NOT: v_cndmask
 ; GCN-NOT: buffer_
-; GCN-DAG: s_or_b32 [[IND1:s[0-9]+]], [[IND0:s[0-9]+]], 1
-; GCN-DAG: s_mov_b32 m0, [[IND0]]
-; GCN-DAG: v_movreld_b32_e32 [[BASE:v[0-9]+]],
-; GCN:     s_mov_b32 m0, [[IND1]]
-; GCN:     v_movreld_b32_e32 [[BASE]]
+; GCN-NOT: s_or_b32
+; GCN-DAG: s_mov_b32 m0, [[IND:s[0-9]+]]
+; GCN-DAG: v_movreld_b32_e32 v[[#BASE]], 0
+; GCN-NOT: s_mov_b32 m0
+; GCN:     v_movreld_b32_e32 v[[#BASE+1]],
 define amdgpu_kernel void @double7_inselt(<7 x double> addrspace(1)* %out, <7 x double> %vec, i32 %sel) {
 entry:
   %v = insertelement <7 x double> %vec, double 1.000000e+00, i32 %sel
   store <7 x double> %v, <7 x double> addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}bit4_inselt:

llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll

 ; SI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x10
 ; SI-NEXT:    s_load_dword s4, s[4:5], 0x20
 ; SI-NEXT:    v_mov_b32_e32 v16, 0x40200000
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    s_lshl_b32 s4, s4, 1
-; SI-NEXT:    s_mov_b32 m0, s4
 ; SI-NEXT:    v_mov_b32_e32 v1, s9
 ; SI-NEXT:    v_mov_b32_e32 v2, s10
 ; SI-NEXT:    v_mov_b32_e32 v3, s11
 ; SI-NEXT:    v_mov_b32_e32 v4, s12
 ; SI-NEXT:    v_mov_b32_e32 v5, s13
 ; SI-NEXT:    v_mov_b32_e32 v6, s14
 ; SI-NEXT:    v_mov_b32_e32 v7, s15
 ; SI-NEXT:    v_mov_b32_e32 v8, s16
 ; SI-NEXT:    v_mov_b32_e32 v9, s17
 ; SI-NEXT:    v_mov_b32_e32 v10, s18
 ; SI-NEXT:    v_mov_b32_e32 v11, s19
 ; SI-NEXT:    v_mov_b32_e32 v12, s20
 ; SI-NEXT:    v_mov_b32_e32 v13, s21
 ; SI-NEXT:    v_mov_b32_e32 v14, s22
 ; SI-NEXT:    v_mov_b32_e32 v15, s23
-; SI-NEXT:    s_or_b32 s4, s4, 1
-; SI-NEXT:    v_movreld_b32_e32 v0, 0
 ; SI-NEXT:    s_mov_b32 m0, s4
-; SI-NEXT:    v_movreld_b32_e32 v0, v16
+; SI-NEXT:    v_movreld_b32_e32 v0, 0
+; SI-NEXT:    v_movreld_b32_e32 v1, v16
 ; SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
 ; SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
 ; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v8f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x80
 ; VI-NEXT:    v_mov_b32_e32 v16, 0x40200000
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    s_lshl_b32 s4, s4, 1
-; VI-NEXT:    s_mov_b32 m0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s9
 ; VI-NEXT:    v_mov_b32_e32 v2, s10
 ; VI-NEXT:    v_mov_b32_e32 v3, s11
 ; VI-NEXT:    v_mov_b32_e32 v4, s12
 ; VI-NEXT:    v_mov_b32_e32 v5, s13
 ; VI-NEXT:    v_mov_b32_e32 v6, s14
 ; VI-NEXT:    v_mov_b32_e32 v7, s15
 ; VI-NEXT:    v_mov_b32_e32 v8, s16
 ; VI-NEXT:    v_mov_b32_e32 v9, s17
 ; VI-NEXT:    v_mov_b32_e32 v10, s18
 ; VI-NEXT:    v_mov_b32_e32 v11, s19
 ; VI-NEXT:    v_mov_b32_e32 v12, s20
 ; VI-NEXT:    v_mov_b32_e32 v13, s21
 ; VI-NEXT:    v_mov_b32_e32 v14, s22
 ; VI-NEXT:    v_mov_b32_e32 v15, s23
-; VI-NEXT:    s_or_b32 s4, s4, 1
-; VI-NEXT:    v_movreld_b32_e32 v0, 0
 ; VI-NEXT:    s_mov_b32 m0, s4
-; VI-NEXT:    v_movreld_b32_e32 v0, v16
+; VI-NEXT:    v_movreld_b32_e32 v0, 0
+; VI-NEXT:    v_movreld_b32_e32 v1, v16
 ; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
 ; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
 ; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %vecins = insertelement <8 x double> %a, double 8.0, i32 %b
   store <8 x double> %vecins, <8 x double> addrspace(1)* %out, align 16
   ret void
 }
 
 declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }