[X86][SSE] Add general 32-bit LOAD + VZEXT_MOVL support to EltsFromConsecutiveLoads

[RKSimon]

Summary

This patch adds support for consecutive (load/undef elements) 32-bit loads, followed by trailing undef/zero elements to be combined to a single MOVSS load.

Follow up to D16217

Note: I've been looking into correcting the domain for both the MOVSS/MOVD and the MOVSD/MOVQ load/stores but am concerned about the number of test changes - is this something that people think is worthwhile? I'd probably have to change many of the tests to ensure that they keep to the intended domain,

[RKSimon]

Add 512-bit vector support

[delena]

test/CodeGen/X86/merge-consecutive-loads-128.ll

334

I think that in these architectures we pay additional cycle for switching from INT to FP. Can we use movd?

test/CodeGen/X86/merge-consecutive-loads-256.ll

451

this instruction (movss) reads 4 bytes from memory. Does it require 4 bytes alignment?

[RKSimon]

test/CodeGen/X86/merge-consecutive-loads-128.ll

334

This was something I mentioned in the summary - adding domain support for MOVSS/MOVD is straightforward but has a knock on effect on a lot of tests, which would need some tests modifying to keep to the original domain and others we'd let switch. If you think its worthwhile I'll start looking at this more seriously?

test/CodeGen/X86/merge-consecutive-loads-256.ll

451

Not unless SSE/AVX alignment checks are enabled - AFAICT llvm assumes they aren't. We are using the alignment of the base pointer, so lowering of the consecutive load is being driven from that.

[RKSimon]

Converted loads to the integer domain - as Elena said, its the more sensible options for consecutive 32-bit loads.

[delena]

LGTM

lib/Target/X86/X86ISelLowering.cpp

5645

What happens here in 32-bit mode, where i64 is illegal?

5668

I think that all these checks are not necessary. (VT.getSizeInBits() >= 128) should be enough.

5672

What happens if you don't put hardcoded MVT::f32, but choose between f32 and i32 according to VT ?

lib/Target/X86/X86InstrAVX512.td

3054 ↗

(On Diff #46645)

Why v8i64 is not handled in this patterns?

[RKSimon]

Thanks Elena - I've addressed your comments in a couple of addition commits:

Added 32-bit target tests to make sure that the 64-bit integer loads still happen (this is the point of the VZEXT_LOAD op any way).

Added float/integer domain handling and removed the over-zealous IsTypeLegal tests.

Note: the v8i64 VZEXT_LOAD patterns are already handled in another part of X86InstrAVX512.td

lib/Target/X86/X86ISelLowering.cpp

if (LoadMask[i]) {
         break;
       }
     } else if (ZeroMask[i]) {
       IsConsecutiveLoad = false;
       break;
     }
   }
 
+  auto CreateLoad = [&DAG, &DL](EVT VT, LoadSDNode *LDBase) {
+    SDValue NewLd = DAG.getLoad(VT, DL, LDBase->getChain(),
+                                LDBase->getBasePtr(), LDBase->getPointerInfo(),
+                                LDBase->isVolatile(), LDBase->isNonTemporal(),
+                                LDBase->isInvariant(), LDBase->getAlignment());
+
+    if (LDBase->hasAnyUseOfValue(1)) {
+      SDValue NewChain =
+          DAG.getNode(ISD::TokenFactor, DL, MVT::Other, SDValue(LDBase, 1),
+                      SDValue(NewLd.getNode(), 1));
+      DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
+      DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
+                             SDValue(NewLd.getNode(), 1));
+    }
+
+    return NewLd;
+  };
+
   // LOAD - all consecutive load/undefs (must start/end with a load).
   // If we have found an entire vector of loads and undefs, then return a large
   // load of the entire vector width starting at the base pointer.
   if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
       LastLoadedElt == (int)(NumElems - 1) && ZeroMask.none()) {
     assert(LDBase && "Did not find base load for merging consecutive loads");
     EVT EltVT = LDBase->getValueType(0);
     // Ensure that the input vector size for the merged loads matches the
     // cumulative size of the input elements.
     if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems)
       return SDValue();
 
     if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
       return SDValue();
 
-    SDValue NewLd = SDValue();
-
-    NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
-                        LDBase->getPointerInfo(), LDBase->isVolatile(),
-                        LDBase->isNonTemporal(), LDBase->isInvariant(),
-                        LDBase->getAlignment());
-
-    if (LDBase->hasAnyUseOfValue(1)) {
-      SDValue NewChain =
-          DAG.getNode(ISD::TokenFactor, DL, MVT::Other, SDValue(LDBase, 1),
-                      SDValue(NewLd.getNode(), 1));
-      DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
-      DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
-                             SDValue(NewLd.getNode(), 1));
-    }
-
-    return NewLd;
+    return CreateLoad(VT, LDBase);
   }
 
   int LoadSize =
       (1 + LastLoadedElt - FirstLoadedElt) * LDBaseVT.getStoreSizeInBits();
 
   // VZEXT_LOAD - consecutive load/undefs followed by zeros/undefs.
   // TODO: The code below fires only for for loading the low 64-bits of a
   // of a 128-bit vector. It's probably worth generalizing more.
   if (IsConsecutiveLoad && FirstLoadedElt == 0 && LoadSize == 64 &&
       (VT.is128BitVector() && TLI.isTypeLegal(MVT::v2i64))) {
     SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
     SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
     SDValue ResNode =
         DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64,
                                 LDBase->getPointerInfo(),
                                 LDBase->getAlignment(),
                                 false/*isVolatile*/, true/*ReadMem*/,
                                 false/*WriteMem*/);
 
     // Make sure the newly-created LOAD is in the same position as LDBase in
     // terms of dependency. We create a TokenFactor for LDBase and ResNode, and
     // update uses of LDBase's output chain to use the TokenFactor.
     if (LDBase->hasAnyUseOfValue(1)) {
       SDValue NewChain =
           DAG.getNode(ISD::TokenFactor, DL, MVT::Other, SDValue(LDBase, 1),
                       SDValue(ResNode.getNode(), 1));
       DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
       DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
                              SDValue(ResNode.getNode(), 1));
     }
 
     return DAG.getBitcast(VT, ResNode);
   }
+
+  // VZEXT_MOVL - consecutive 32-bit load/undefs followed by zeros/undefs.
+  if (IsConsecutiveLoad && FirstLoadedElt == 0 && LoadSize == 32 &&
+      ((VT.is128BitVector() && TLI.isTypeLegal(MVT::v4f32)) ||
+       (VT.is256BitVector() && TLI.isTypeLegal(MVT::v8f32)))) {
+    MVT VecVT = VT.is256BitVector() ? MVT::v8f32 : MVT::v4f32;
+    SDValue V = CreateLoad(MVT::f32, LDBase);
+    V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, V);
+    V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, V);
+    return DAG.getBitcast(VT, V);
+  }
+
   return SDValue();
 }
 
 /// LowerVectorBroadcast - Attempt to use the vbroadcast instruction
 /// to generate a splat value for the following cases:
 /// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant.
 /// 2. A splat shuffle which uses a scalar_to_vector node which comes from
 /// a scalar load, or a constant.

test/CodeGen/X86/merge-consecutive-loads-128.ll

; AVX-NEXT:    retq
   %res5 = insertelement <8 x i16> %res4, i16 %val5, i32 5
   %res7 = insertelement <8 x i16> %res5, i16 %val7, i32 7
   ret <8 x i16> %res7
 }
 
 define <8 x i16> @merge_8i16_i16_34uuuuuu(i16* %ptr) nounwind uwtable noinline ssp {
 ; SSE-LABEL: merge_8i16_i16_34uuuuuu:
 ; SSE:       # BB#0:
-; SSE-NEXT:    pinsrw $0, 6(%rdi), %xmm0
-; SSE-NEXT:    pinsrw $1, 8(%rdi), %xmm0
+; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: merge_8i16_i16_34uuuuuu:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpinsrw $0, 6(%rdi), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $1, 8(%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 4
   %val0 = load i16, i16* %ptr0
   %val1 = load i16, i16* %ptr1
   %res0 = insertelement <8 x i16> undef, i16 %val0, i32 0
   %res1 = insertelement <8 x i16> %res0, i16 %val1, i32 1
   ret <8 x i16> %res1
; AVX-NEXT:    retq
   %resB = insertelement <16 x i8> %resA, i8 %valB, i32 11
   %resC = insertelement <16 x i8> %resB, i8 %valC, i32 12
   %resD = insertelement <16 x i8> %resC, i8 %valD, i32 13
   %resF = insertelement <16 x i8> %resD, i8 %valF, i32 15
   ret <16 x i8> %resF
 }
 
 define <16 x i8> @merge_16i8_i8_01u3uuzzuuuuuzzz(i8* %ptr) nounwind uwtable noinline ssp {
-; SSE2-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    movzbl (%rdi), %eax
-; SSE2-NEXT:    movzbl 1(%rdi), %ecx
-; SSE2-NEXT:    shll $8, %ecx
-; SSE2-NEXT:    orl %eax, %ecx
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pinsrw $0, %ecx, %xmm0
-; SSE2-NEXT:    movzbl 3(%rdi), %eax
-; SSE2-NEXT:    shll $8, %eax
-; SSE2-NEXT:    pinsrw $1, %eax, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
-; SSE41:       # BB#0:
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pinsrb $0, (%rdi), %xmm0
-; SSE41-NEXT:    pinsrb $1, 1(%rdi), %xmm0
-; SSE41-NEXT:    pinsrb $3, 3(%rdi), %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
+; SSE:       # BB#0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $0, (%rdi), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $1, 1(%rdi), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $3, 3(%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 3
   %val0 = load i8, i8* %ptr0
   %val1 = load i8, i8* %ptr1
   %val3 = load i8, i8* %ptr3
   %res0 = insertelement <16 x i8> undef, i8 %val0, i32 0

test/CodeGen/X86/merge-consecutive-loads-256.ll

; AVX2-NEXT:    retq
   %res2 = insertelement <8 x i32> %res0, i32 %val2, i32 2
   %res4 = insertelement <8 x i32> %res2, i32 %val4, i32 4
   %res5 = insertelement <8 x i32> %res4, i32     0, i32 5
   %res7 = insertelement <8 x i32> %res5, i32 %val7, i32 7
   ret <8 x i32> %res7
 }
 
 define <16 x i16> @merge_16i16_i16_89zzzuuuuuuuuuuuz(i16* %ptr) nounwind uwtable noinline ssp {
-; AVX1-LABEL: merge_16i16_i16_89zzzuuuuuuuuuuuz:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm1
-; AVX1-NEXT:    vpinsrw $1, 18(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: merge_16i16_i16_89zzzuuuuuuuuuuuz:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $0, 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT:    vpinsrw $1, 18(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: merge_16i16_i16_89zzzuuuuuuuuuuuz:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 8
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 9
   %val0 = load i16, i16* %ptr0
   %val1 = load i16, i16* %ptr1
   %res0 = insertelement <16 x i16> undef, i16 %val0, i16 0
   %res1 = insertelement <16 x i16> %res0, i16 %val1, i16 1
   %res2 = insertelement <16 x i16> %res1, i16     0, i16 2
   %res3 = insertelement <16 x i16> %res2, i16     0, i16 3
; AVX2-NEXT:    retq
   %resE = insertelement <16 x i16> %resD, i16 %valE, i16 14
   %resF = insertelement <16 x i16> %resE, i16 %valF, i16 15
   ret <16 x i16> %resF
 }
 
 define <32 x i8> @merge_32i8_i8_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu(i8* %ptr) nounwind uwtable noinline ssp {
 ; AVX-LABEL: merge_32i8_i8_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vpinsrb $0, 4(%rdi), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $1, 5(%rdi), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $3, 7(%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 4
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 5
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 7
   %val0 = load i8, i8* %ptr0
   %val1 = load i8, i8* %ptr1
   %val3 = load i8, i8* %ptr3
   %res0 = insertelement <32 x i8> undef, i8 %val0, i8 0
   %res1 = insertelement <32 x i8> %res0, i8 %val1, i8 1
   %res3 = insertelement <32 x i8> %res1, i8 %val3, i8 3
   ret <32 x i8> %res3
 }
 
 define <32 x i8> @merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu(i8* %ptr) nounwind uwtable noinline ssp {
-; AVX1-LABEL: merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrb $0, 4(%rdi), %xmm0, %xmm1
-; AVX1-NEXT:    vpinsrb $1, 5(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vpinsrb $3, 7(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrb $0, 4(%rdi), %xmm0, %xmm1
-; AVX2-NEXT:    vpinsrb $1, 5(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrb $3, 7(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu:
+; AVX:       # BB#0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 4
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 5
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 7
   %val0 = load i8, i8* %ptr0
   %val1 = load i8, i8* %ptr1
   %val3 = load i8, i8* %ptr3
   %res0 = insertelement <32 x i8> undef, i8 %val0, i8 0
   %res1 = insertelement <32 x i8> %res0, i8 %val1, i8 1
   %res3 = insertelement <32 x i8> %res1, i8 %val3, i8 3
   %resE = insertelement <32 x i8> %res3, i8     0, i8 14
   %resF = insertelement <32 x i8> %resE, i8     0, i8 15
   %resG = insertelement <32 x i8> %resF, i8     0, i8 16
   %resH = insertelement <32 x i8> %resG, i8     0, i8 17
   ret <32 x i8> %resH
 }