Fix SROA in presence of incompatible vector types in the same partition

[serge-sans-paille]

Summary

This is a fix for https://bugs.llvm.org/show_bug.cgi?id=42585, triggered by a bitcast that enlarges the size of the vector type. Current codebase doesn't support that.

[serge-sans-paille]

Up?

[serge-sans-paille]

Updated test case.

[serge-sans-paille]

@chandlerc / @bkramer any thought on this fix?

[serge-sans-paille]

Obsoleted by https://reviews.llvm.org/rL372480, closing

llvm/lib/Transforms/Scalar/SROA.cpp

 /// don't want to do the rewrites unless we are confident that the result will
 /// be promotable, so we have an early test here.
 static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
   // Collect the candidate types for vector-based promotion. Also track whether
   // we have different element types.
   SmallVector<VectorType *, 4> CandidateTys;
   Type *CommonEltTy = nullptr;
   bool HaveCommonEltTy = true;
+  VectorType *CommonVecTy = nullptr;
   auto CheckCandidateType = [&](Type *Ty) {
     if (auto *VTy = dyn_cast<VectorType>(Ty)) {
       CandidateTys.push_back(VTy);
-      if (!CommonEltTy)
+      if (!CommonEltTy) {
         CommonEltTy = VTy->getElementType();
-      else if (CommonEltTy != VTy->getElementType())
+        CommonVecTy = VTy;
+      } else if (CommonEltTy != VTy->getElementType()) {
         HaveCommonEltTy = false;
+        CommonVecTy = nullptr;
+      } else if (CommonVecTy->getNumElements() != VTy->getNumElements()) {
+        CommonVecTy = nullptr;
+      }
     }
   };
   // Consider any loads or stores that are the exact size of the slice.
   for (const Slice &S : P)
     if (S.beginOffset() == P.beginOffset() &&
         S.endOffset() == P.endOffset()) {
       if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser()))
         CheckCandidateType(LI->getType());
auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
       assert(LHSTy->getElementType()->isIntegerTy() &&
              "All non-integer types eliminated!");
       return RHSTy->getNumElements() < LHSTy->getNumElements();
     };
     llvm::sort(CandidateTys, RankVectorTypes);
     CandidateTys.erase(
         std::unique(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes),
         CandidateTys.end());
+
+    // In presence of bitcast between different vector, we may end up with
+    // CandidateTys holding different vector types with the same element type
+    // and we can't cast between different vector types.
+  } else if (!CommonVecTy) {
+    return nullptr;
   } else {
+
 // The only way to have the same element type in every vector type is to
 // have the same vector type. Check that and remove all but one.
 #ifndef NDEBUG
     for (VectorType *VTy : CandidateTys) {
       assert(VTy->getElementType() == CommonEltTy &&
              "Unaccounted for element type!");
       assert(VTy == CandidateTys[0] &&
              "Different vector types with the same element type!");

llvm/test/Transforms/SROA/bitcast-vectype.ll

+; RUN: opt < %s -sroa -S | FileCheck %s
+; check that sroa does nothing on alloca followed by a bitcast
+target datalayout = "e-p:64:64:64-i8:8:8-i8:8:8-i86:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+%t = type { <1 x i8> }
+
+define <3 x i8> @vector_bitcast() {
+  ; CHECK-LABEL: @vector_bitcast
+    ; CHECK: %b = alloca [2 x %t]
+  ; CHECK: [[VAL0:%[^ ]+]] = bitcast [2 x %t]* %b to <2 x i8>*
+    ; CHECK: store <2 x i8> <i8 0, i8 1>, <2 x i8>* [[VAL0]]
+    ; CHECK: [[VAL1:%[^ ]+]] = bitcast [2 x %t]* %b to <3 x i8>*
+    ; CHECK: [[VAL2:%[^ ]+]] = load <3 x i8>, <3 x i8>* [[VAL1]]
+    ; CHECK: ret <3 x i8> [[VAL2]]
+  %b = alloca [2 x %t]
+  %a = bitcast [2 x %t]* %b to <2 x i8>*
+  store <2 x i8> <i8 0,i8 1>, <2 x i8>* %a
+  %cast = bitcast <2 x i8>* %a to <3 x i8>*
+  %vec = load <3 x i8>, <3 x i8>* %cast
+  ret <3 x i8> %vec
+}