[LowerConstantIntrinsics] avoid crashing on alloca with unexpected operand type

[spatel]

Summary

The test here is reduced from the fuzzer-generated crasher in:
https://llvm.org/PR50023

I don't know if this is the best or complete solution, but the zext of the i42 type appears to match the behavior that I'm seeing if we run a weird type example like this through the IR optimizer with -O1.

[lebedev.ri]

LGTM

[RKSimon]

Awesome, thanks @spatel

llvm/lib/Analysis/MemoryBuiltins.cpp

 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) {
   if (!I.getAllocatedType()->isSized())
     return unknown();
 
   // must be a VLA
   assert(I.isArrayAllocation());
-  Value *ArraySize = I.getArraySize();
+
+  // If needed, adjust the alloca's operand size to match the pointer size.
+  // Subsequent math operations expect the types to match.
+  Value *ArraySize = Builder.CreateZExtOrTrunc(
+      I.getArraySize(), DL.getIntPtrType(I.getContext()));
+  assert(ArraySize->getType() == Zero->getType() &&
+         "Expected zero constant to have pointer type");
+
   Value *Size = ConstantInt::get(ArraySize->getType(),
                                  DL.getTypeAllocSize(I.getAllocatedType()));
   Size = Builder.CreateMul(Size, ArraySize);
   return std::make_pair(Size, Zero);
 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitCallBase(CallBase &CB) {
   Optional<AllocFnsTy> FnData = getAllocationSize(&CB, TLI);

llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll

 
 ; CHECK-LABEL: @test_objectsize_byref_arg(
 ; CHECK: ret i64 42
 define i64 @test_objectsize_byref_arg([42 x i8]* byref([42 x i8]) %ptr) {
   %cast = bitcast [42 x i8]* %ptr to i8*
   %size = tail call i64 @llvm.objectsize.i64(i8* %cast, i1 true, i1 false, i1 false)
   ret i64 %size
 }
+
+; https://llvm.org/PR50023
+; The alloca operand type may not match pointer type size.
+
+define i64 @vla_pointer_size_mismatch(i42 %x) {
+; CHECK-LABEL: @vla_pointer_size_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i42 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 1, [[TMP1]]
+; CHECK-NEXT:    [[A:%.*]] = alloca i8, i42 [[X]], align 1
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr i8, i8* [[A]], i8 17
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 17
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 17
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP6]])
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %A = alloca i8, i42 %x, align 1
+  %G1 = getelementptr i8, i8* %A, i8 17
+  %objsize = call i64 @llvm.objectsize.i64(i8* %G1, i1 false, i1 true, i1 true)
+  ret i64 %objsize
+}