[SROA] prevent crash on large memset length (PR50910)

[spatel]

Summary

I don't know much about this pass, but we need a stronger check on the memset length arg to avoid an assert. The current code was added with D59000.
The test is reduced from:
https://llvm.org/PR50910

[spatel]

Rebased - I found another bug ( https://llvm.org/PR50888 ) near here and patched it with f14495dc75d7 .

[spatel]

Ping.

[lebedev.ri]

I suppose we could instead clamp to the maximal remaining alloca size,
but it's UB so it doesn't seem worth spending too much time on it.
Seems fine to me.

llvm/lib/Transforms/Scalar/SROA.cpp

bool visitMemSetInst(MemSetInst &II) {
     Type *ScalarTy = AllocaTy->getScalarType();
 
     const bool CanContinue = [&]() {
       if (VecTy || IntTy)
         return true;
       if (BeginOffset > NewAllocaBeginOffset ||
           EndOffset < NewAllocaEndOffset)
         return false;
+      // Length must be in range for FixedVectorType.
       auto *C = cast<ConstantInt>(II.getLength());
-      if (C->getBitWidth() > 64)
+      const uint64_t Len = C->getLimitedValue();
+      if (Len > std::numeric_limits<unsigned>::max())
         return false;
-      const auto Len = C->getZExtValue();
       auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext());
       auto *SrcTy = FixedVectorType::get(Int8Ty, Len);
       return canConvertValue(DL, SrcTy, AllocaTy) &&
              DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedSize());
     }();
 
     // If this doesn't map cleanly onto the alloca type, and that type isn't
     // a single value type, just emit a memset.

llvm/test/Transforms/SROA/slice-width.ll

 ; CHECK-NEXT:    [[ARRAY:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 [[ARRAY]], i8 0, i64 ptrtoint (void ()* @PR50888 to i64), i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %array = alloca i8
   call void @llvm.memset.p0i8.i64(i8* align 16 %array, i8 0, i64 ptrtoint (void ()* @PR50888 to i64), i1 false)
   ret void
 }
+
+; Don't crash on out-of-bounds length.
+
+define void @PR50910() {
+; CHECK-LABEL: @PR50910(
+; CHECK-NEXT:    [[T1:%.*]] = alloca i8, i64 1, align 8
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[T1]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %t1 = alloca i8, i64 1, align 8
+  call void @llvm.memset.p0i8.i64(i8* align 8 %t1, i8 0, i64 4294967296, i1 false)
+  ret void
+}