[InstCombine] Correct the check for vector GEPs

[mssimpso]

Summary

Some of the GEP combines (e.g., descaling) can't handle vector GEPs. We have an existing check that attempts to bail out if given a vector GEP. However, the check only tests the GEP's pointer operand. A GEP results in a vector of pointers if at least one of its operands is vector-typed (i.e., its pointer operand could be a scalar but its index could be a vector). We should probably just check the type of the GEP itself. This should fix PR32414.

The added test case causes an assertion without this patch.

Reference: https://bugs.llvm.org/show_bug.cgi?id=32414

[efriedma]

LGTM.

[mssimpso]

Thanks Eli!

lib/Transforms/InstCombine/InstructionCombining.cpp

if (GEP.getOperand(1)->getType()->getScalarSizeInBits() ==
                            m_PtrToInt(m_Specific(GEP.getOperand(0)))))) {
           return CastInst::CreatePointerBitCastOrAddrSpaceCast(Y,
                                                                GEP.getType());
         }
       }
     }
   }
 
-  // Handle gep(bitcast x) and gep(gep x, 0, 0, 0).
-  Value *StrippedPtr = PtrOp->stripPointerCasts();
-  PointerType *StrippedPtrTy = dyn_cast<PointerType>(StrippedPtr->getType());
-
   // We do not handle pointer-vector geps here.
-  if (!StrippedPtrTy)
+  if (GEP.getType()->isVectorTy())
     return nullptr;
 
+  // Handle gep(bitcast x) and gep(gep x, 0, 0, 0).
+  Value *StrippedPtr = PtrOp->stripPointerCasts();
+  PointerType *StrippedPtrTy = cast<PointerType>(StrippedPtr->getType());
+
   if (StrippedPtr != PtrOp) {
     bool HasZeroPointerIndex = false;
     if (ConstantInt *C = dyn_cast<ConstantInt>(GEP.getOperand(1)))
       HasZeroPointerIndex = C->isZero();
 
     // Transform: GEP (bitcast [10 x i8]* X to [0 x i8]*), i32 0, ...
     // into     : GEP [10 x i8]* X, i32 0, ...
     //

test/Transforms/InstCombine/getelementptr.ll

 ; CHECK-NEXT: getelementptr [128 x i32]
 ; CHECK-NEXT: addrspacecast i32*
 ; CHECK-NEXT: ret i32 addrspace(1)*
   %gep = getelementptr [128 x i32], [128 x i32]* %p, i32 0, i32 0
   %x = addrspacecast i32* %gep to i32 addrspace(1)*
   ret i32 addrspace(1)* %x
 }
 
+define <2 x i32*> @PR32414(i32** %ptr) {
+; CHECK-LABEL: @PR32414(
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32** %ptr to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    ret <2 x i32*> [[TMP1]]
+;
+  %tmp0 = bitcast i32** %ptr to i32*
+  %tmp1 = getelementptr inbounds i32, i32* %tmp0, <2 x i64> <i64 0, i64 1>
+  ret <2 x i32*> %tmp1
+}
+
 ; CHECK: attributes [[NUW]] = { nounwind }