[X86] Fix SmallVector sizes in constant pool shuffle decoding to avoid heap allocation

[craig.topper]

Summary

Some of the vectors are under sized to avoid heap allocation. In one case the vector was oversized.

Not sure if 64 uint64_ts is maybe too much stack space.

[RKSimon]

LGTM - I wonder if there's any useful (i.e. not unnecessarily noisy) that we could get the compiler to report cases where SmallVector has exceeded its 'stack' size?

lib/Target/X86/X86ShuffleDecodeConstantPool.cpp

void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
   Type *MaskTy = C->getType();
   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
   (void)MaskTySize;
   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
          "Unexpected vector size.");
 
   // The shuffle mask requires a byte vector.
   SmallBitVector UndefElts;
-  SmallVector<uint64_t, 32> RawMask;
+  SmallVector<uint64_t, 64> RawMask;
   if (!extractConstantMask(C, 8, UndefElts, RawMask))
     return;
 
   unsigned NumElts = RawMask.size();
   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
          "Unexpected number of vector elements.");
 
   for (unsigned i = 0; i != NumElts; ++i) {
void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
   (void)MaskTySize;
   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
          "Unexpected vector size.");
   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
 
   // The shuffle mask requires elements the same size as the target.
   SmallBitVector UndefElts;
-  SmallVector<uint64_t, 8> RawMask;
+  SmallVector<uint64_t, 16> RawMask;
   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
     return;
 
   unsigned NumElts = RawMask.size();
   unsigned NumEltsPerLane = 128 / ElSize;
   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
          "Unexpected number of vector elements.");
 

 }
 
 void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
   assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
          "Unexpected vector size.");
 
   // The shuffle mask requires a byte vector.
   SmallBitVector UndefElts;
-  SmallVector<uint64_t, 32> RawMask;
+  SmallVector<uint64_t, 16> RawMask;
   if (!extractConstantMask(C, 8, UndefElts, RawMask))
     return;
 
   unsigned NumElts = RawMask.size();
   assert(NumElts == 16 && "Unexpected number of vector elements.");
 
   for (unsigned i = 0; i != NumElts; ++i) {
     if (UndefElts[i]) {
void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
   (void)MaskTySize;
   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
          "Unexpected vector size.");
   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
          "Unexpected vector element size.");
 
   // The shuffle mask requires elements the same size as the target.
   SmallBitVector UndefElts;
-  SmallVector<uint64_t, 8> RawMask;
+  SmallVector<uint64_t, 64> RawMask;
   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
     return;
 
   unsigned NumElts = RawMask.size();
 
   for (unsigned i = 0; i != NumElts; ++i) {
     if (UndefElts[i]) {
       ShuffleMask.push_back(SM_SentinelUndef);
void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
   (void)MaskTySize;
   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
          "Unexpected vector size.");
   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
          "Unexpected vector element size.");
 
   // The shuffle mask requires elements the same size as the target.
   SmallBitVector UndefElts;
-  SmallVector<uint64_t, 8> RawMask;
+  SmallVector<uint64_t, 64> RawMask;
   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
     return;
 
   unsigned NumElts = RawMask.size();
 
   for (unsigned i = 0; i != NumElts; ++i) {
     if (UndefElts[i]) {
       ShuffleMask.push_back(SM_SentinelUndef);
       continue;
     }
     int Index = RawMask[i] & (NumElts*2 - 1);
     ShuffleMask.push_back(Index);
   }
 }
 } // llvm namespace