[SelectionDAG] Teach SelectionDAGBuilder's getUniformBase for gather/scatter handling to accept GEPs with more than 2 operands if the middle operands are all 0s

[craig.topper]

Summary

Currently we can only get a uniform base from a simple GEP with 2 operands. This causes us to miss address folding opportunities for simple global array accesses as the test case shows.

This patch adds support for larger GEPs if the other indices are 0 since those don't require any additional computations to be inserted.

We may also want to handle constant splats of zero here, but I'm leaving that for future work when I have a real world example.

[delena]

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

3874 ↗

(On Diff #122479)

If !GEP->hasIndices() means that the GEP has only base - it is ok for the uniform base, but this GEP is not real, I think..

[craig.topper]

Remove the hasIndices check

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

 // In all other cases the function returns 'false'.
 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
                            SelectionDAGBuilder* SDB) {
   SelectionDAG& DAG = SDB->DAG;
   LLVMContext &Context = *DAG.getContext();
 
   assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
-  if (!GEP || GEP->getNumOperands() > 2)
+  if (!GEP)
     return false;
 
   const Value *GEPPtr = GEP->getPointerOperand();
   if (!GEPPtr->getType()->isVectorTy())
     Ptr = GEPPtr;
   else if (!(Ptr = getSplatValue(GEPPtr)))
     return false;
 
-  Value *IndexVal = GEP->getOperand(1);
+  unsigned FinalIndex = GEP->getNumOperands() - 1;
+  Value *IndexVal = GEP->getOperand(FinalIndex);
+
+  // Ensure all the other indices are 0.
+  for (unsigned i = 1; i < FinalIndex; ++i)
+    if (auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i)))
+      if (!C->isZero())
+        return false;
 
   // The operands of the GEP may be defined in another basic block.
   // In this case we'll not find nodes for the operands.
   if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
     return false;
 
   Base = SDB->getValue(Ptr);
   Index = SDB->getValue(IndexVal);

llvm/trunk/test/CodeGen/X86/masked_gather_scatter.ll

; SKX_32-NEXT:    retl
   %b = add <4 x i64> %a, %g3
   ret <4 x i64> %b
 }
 declare <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*>, i32, <4 x i1>, <4 x i64>)
 
 define <8 x i32> @test_global_array(<8 x i64> %indxs) {
 ; KNL_64-LABEL: test_global_array:
 ; KNL_64:       # BB#0:
-; KNL_64-NEXT:    vpsllq $2, %zmm0, %zmm0
 ; KNL_64-NEXT:    movl $glob_array, %eax
-; KNL_64-NEXT:    vpbroadcastq %rax, %zmm1
-; KNL_64-NEXT:    vpaddq %zmm0, %zmm1, %zmm1
 ; KNL_64-NEXT:    kxnorw %k0, %k0, %k1
-; KNL_64-NEXT:    vpgatherqd (,%zmm1), %ymm0 {%k1}
+; KNL_64-NEXT:    vpgatherqd (%rax,%zmm0,4), %ymm1 {%k1}
+; KNL_64-NEXT:    vmovdqa %ymm1, %ymm0
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test_global_array:
 ; KNL_32:       # BB#0:
-; KNL_32-NEXT:    vpmovqd %zmm0, %ymm0
-; KNL_32-NEXT:    vpslld $2, %ymm0, %ymm0
 ; KNL_32-NEXT:    movl $glob_array, %eax
-; KNL_32-NEXT:    vmovd %eax, %xmm1
-; KNL_32-NEXT:    vpbroadcastd %xmm1, %ymm1
-; KNL_32-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
-; KNL_32-NEXT:    vpmovsxdq %ymm0, %zmm1
 ; KNL_32-NEXT:    kxnorw %k0, %k0, %k1
-; KNL_32-NEXT:    vpgatherqd (,%zmm1), %ymm0 {%k1}
+; KNL_32-NEXT:    vpgatherqd (%eax,%zmm0,4), %ymm1 {%k1}
+; KNL_32-NEXT:    vmovdqa %ymm1, %ymm0
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test_global_array:
 ; SKX:       # BB#0:
-; SKX-NEXT:    vpsllq $2, %zmm0, %zmm0
 ; SKX-NEXT:    movl $glob_array, %eax
-; SKX-NEXT:    vpbroadcastq %rax, %zmm1
-; SKX-NEXT:    vpaddq %zmm0, %zmm1, %zmm1
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    vpgatherqd (,%zmm1), %ymm0 {%k1}
+; SKX-NEXT:    vpgatherqd (%rax,%zmm0,4), %ymm1 {%k1}
+; SKX-NEXT:    vmovdqa %ymm1, %ymm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test_global_array:
 ; SKX_32:       # BB#0:
 ; SKX_32-NEXT:    movl $glob_array, %eax
-; SKX_32-NEXT:    vpbroadcastd %eax, %ymm1
-; SKX_32-NEXT:    vpmovqd %zmm0, %ymm0
-; SKX_32-NEXT:    vpslld $2, %ymm0, %ymm0
-; SKX_32-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
-; SKX_32-NEXT:    vpgatherdd (,%ymm1), %ymm0 {%k1}
+; SKX_32-NEXT:    vpgatherqd (%eax,%zmm0,4), %ymm1 {%k1}
+; SKX_32-NEXT:    vmovdqa %ymm1, %ymm0
 ; SKX_32-NEXT:    retl
   %p = getelementptr inbounds [16 x i32], [16 x i32]* @glob_array, i64 0, <8 x i64> %indxs
   %g = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %p, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
   ret <8 x i32> %g
 }