[X86][AVX] Extract SUBV_BROADCAST constant bits from just the lower subvector range (PR51281)

[RKSimon]

Summary

As reported on PR51281, an internal fuzz test encountered an issue when extracting constant bits from a SUBV_BROADCAST node from a constant pool source larger than the broadcasted subvector width.

The getTargetConstantBitsFromNode was assuming that the Constant would the same size as the subvector, resulting in the incorrect packing of the per-element bits data.

This patch attempts to solve this by using the SUBV_BROADCAST node to determine the subvector width, and then ensuring we extract only the lowest bits from Constant of that subvector bitsize.

[RKSimon]

ping?

[andreadb]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

if (const Constant *C = getTargetConstantFromBasePtr(Ptr)) {
       }
     }
   }
 
   // Extract constant bits from a subvector broadcast.
   if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
     auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
     SDValue Ptr = MemIntr->getBasePtr();
+    // The source constant may be larger than the subvector broadcast,
+    // ensure we extract the correct subvector constants.
     if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) {
       Type *CstTy = Cst->getType();
       unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
-      if (!CstTy->isVectorTy() || (SizeInBits % CstSizeInBits) != 0)
-        return false;
-      unsigned SubEltSizeInBits = CstTy->getScalarSizeInBits();
-      unsigned NumSubElts = CstSizeInBits / SubEltSizeInBits;
-      unsigned NumSubVecs = SizeInBits / CstSizeInBits;
+      unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits();
+      if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 ||
+          (SizeInBits % SubVecSizeInBits) != 0)
+        return false;
+      unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
+      unsigned NumSubElts = SubVecSizeInBits / CstEltSizeInBits;
+      unsigned NumSubVecs = SizeInBits / SubVecSizeInBits;
       APInt UndefSubElts(NumSubElts, 0);
       SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs,
-                                        APInt(SubEltSizeInBits, 0));
+                                        APInt(CstEltSizeInBits, 0));
       for (unsigned i = 0; i != NumSubElts; ++i) {
         if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i],
                                  UndefSubElts, i))
           return false;
         for (unsigned j = 1; j != NumSubVecs; ++j)
           SubEltBits[i + (j * NumSubElts)] = SubEltBits[i];
       }
       UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(),

llvm/test/CodeGen/X86/pr51281.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_mem_shuffle
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
 
 ; CHECK: .LCPI0_0:
 ; CHECK-NEXT: .long   0x00000000
 ; CHECK-NEXT: .zero   4
 ; CHECK-NEXT: .long   0x3eb5dbc6
 ; CHECK-NEXT: .long   0x3eb5dbc6
 ; CHECK-NEXT: .zero   4
 ; CHECK-NEXT: .long   0x3eb5dbc6
-; CHECK-NEXT: .zero   4
-; CHECK-NEXT: .zero   4
+; CHECK-NEXT: .long   0x3eb5dbc6
+; CHECK-NEXT: .long   0x3eb5dbc6
 
 ; CHECK: .LCPI0_1:
 ; CHECK-NEXT: .long   3
 ; CHECK-NEXT: .long   3
 ; CHECK-NEXT: .long   1
 ; CHECK-NEXT: .long   1
 ; CHECK-NEXT: .long   6
 ; CHECK-NEXT: .long   7