Prefer AVX512 memcpy when applicable

[davezarzycki]

Summary

When AVX512 is available and the preferred vector width is 512-bits or more, we should prefer AVX512 for memcpy().

https://bugs.llvm.org/show_bug.cgi?id=43240

[craig.topper]

Do we have a similar test for memset that should be updated?

llvm/lib/Target/X86/X86ISelLowering.cpp

2156 ↗

(On Diff #221161)

VLX shouldn’t be needed. That’s only for 128/256 bit vectors.

[davezarzycki]

The patch no longer depends on the AVX512 vector length extension.

[davezarzycki]

In D67874#1677892, @craig.topper wrote:

Do we have a similar test for memset that should be updated?

Hi @craig.topper – I can try and update a test of memset, but I don't see one that is trying to be comprehensive. For whatever it may be worth, the test suite passes (but that's not an accomplishment). Do you have a request?

[craig.topper]

I added some memset tests in r372494 and the RUN line changes here. Can you rebase?

[davezarzycki]

Hi @craig.topper – Thanks for writing the memset tests. I've expanded the non-zero one to also handle different preferred vector sizes.

[craig.topper]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

This doesn't look like it was generated by the update_llc_test_checks script.

[davezarzycki]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

How so? I tried running update_llc_test_checks against the test and nothing changed.

[craig.topper]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

I've never seen the script mix prefixes like this. As far as I know it always creates a block with the same prefix separated by a blank line from the other prefixes.

[RKSimon]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

Something's definitely gone wrong here, it might be that you've not set the --llc-binary args correctly? I'd recommend manually deleting the all the 'AVX512' checks from all these cases and then regenerating to see what happens.

[davezarzycki]

This patch has now been run through update_llc_test_checks.

[craig.topper]

LGTM

[davezarzycki]

r372540

lib/Target/X86/X86ISelLowering.cpp

 EVT X86TargetLowering::getOptimalMemOpType(
     uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
     bool ZeroMemset, bool MemcpyStrSrc,
     const AttributeList &FuncAttributes) const {
   if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
     if (Size >= 16 && (!Subtarget.isUnalignedMem16Slow() ||
                        ((DstAlign == 0 || DstAlign >= 16) &&
                         (SrcAlign == 0 || SrcAlign >= 16)))) {
+      // FIXME: Check if unaligned 64-byte accesses are slow.
+      if (Size >= 64 && Subtarget.hasAVX512() &&
+          (Subtarget.getPreferVectorWidth() >= 512)) {
+        return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
+      }
       // FIXME: Check if unaligned 32-byte accesses are slow.
       if (Size >= 32 && Subtarget.hasAVX() &&
           (Subtarget.getPreferVectorWidth() >= 256)) {
         // Although this isn't a well-supported type for AVX1, we'll let
         // legalization and shuffle lowering produce the optimal codegen. If we
         // choose an optimal type with a vector element larger than a byte,
         // getMemsetStores() may create an intermediate splat (using an integer
         // multiply) before we splat as a vector.

test/CodeGen/X86/memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin      -mcpu=core2   | FileCheck %s -check-prefix=DARWIN
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2   | FileCheck %s -check-prefix=LINUX
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake | FileCheck %s -check-prefix=LINUX-SKL
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx     | FileCheck %s -check-prefix=LINUX-SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=knl     | FileCheck %s -check-prefix=LINUX-KNL
+; RUN: llc < %s -mtriple=x86_64-apple-darwin      -mcpu=core2     | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2     | FileCheck %s -check-prefix=LINUX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake   | FileCheck %s -check-prefix=LINUX-SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx       | FileCheck %s -check-prefix=LINUX-SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=knl       | FileCheck %s -check-prefix=LINUX-KNL
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=avx512bw | FileCheck %s -check-prefix=LINUX-AVX512BW
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 addrspace(256)* nocapture, i64, i1) nounwind
 
 
 ; Variable memcpy's should lower to calls.
 define i8* @test1(i8* %a, i8* %b, i64 %n) nounwind {

 ; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
 ; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
 ; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
 ; LINUX-SKX-NEXT:    vzeroupper
 ; LINUX-SKX-NEXT:    retq
 ;
 ; LINUX-KNL-LABEL: test3:
 ; LINUX-KNL:       # %bb.0: # %entry
-; LINUX-KNL-NEXT:    vmovups (%rsi), %ymm0
-; LINUX-KNL-NEXT:    vmovups 32(%rsi), %ymm1
-; LINUX-KNL-NEXT:    vmovups %ymm1, 32(%rdi)
-; LINUX-KNL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-KNL-NEXT:    retq
 ;
 ; LINUX-AVX512BW-LABEL: test3:
 ; LINUX-AVX512BW:       # %bb.0: # %entry
 ; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
 ; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-AVX512BW-NEXT:    vzeroupper
 ; LINUX-AVX512BW-NEXT:    retq

 ; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
 ; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
 ; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
 ; LINUX-SKX-NEXT:    vzeroupper
 ; LINUX-SKX-NEXT:    retq
 ;
 ; LINUX-KNL-LABEL: test3_minsize:
 ; LINUX-KNL:       # %bb.0:
-; LINUX-KNL-NEXT:    vmovups (%rsi), %ymm0
-; LINUX-KNL-NEXT:    vmovups 32(%rsi), %ymm1
-; LINUX-KNL-NEXT:    vmovups %ymm1, 32(%rdi)
-; LINUX-KNL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-KNL-NEXT:    retq
 ;
 ; LINUX-AVX512BW-LABEL: test3_minsize:
 ; LINUX-AVX512BW:       # %bb.0:
 ; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
 ; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-AVX512BW-NEXT:    vzeroupper
 ; LINUX-AVX512BW-NEXT:    retq

 ; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
 ; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
 ; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
 ; LINUX-SKX-NEXT:    vzeroupper
 ; LINUX-SKX-NEXT:    retq
 ;
 ; LINUX-KNL-LABEL: test3_minsize_optsize:
 ; LINUX-KNL:       # %bb.0:
-; LINUX-KNL-NEXT:    vmovups (%rsi), %ymm0
-; LINUX-KNL-NEXT:    vmovups 32(%rsi), %ymm1
-; LINUX-KNL-NEXT:    vmovups %ymm1, 32(%rdi)
-; LINUX-KNL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-KNL-NEXT:    retq
 ;
 ; LINUX-AVX512BW-LABEL: test3_minsize_optsize:
 ; LINUX-AVX512BW:       # %bb.0:
 ; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
 ; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-AVX512BW-NEXT:    vzeroupper
 ; LINUX-AVX512BW-NEXT:    retq

 ; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
 ; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
 ; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
 ; LINUX-SKX-NEXT:    vzeroupper
 ; LINUX-SKX-NEXT:    retq
 ;
 ; LINUX-KNL-LABEL: test4:
 ; LINUX-KNL:       # %bb.0: # %entry
-; LINUX-KNL-NEXT:    vmovups (%rsi), %ymm0
-; LINUX-KNL-NEXT:    vmovups 32(%rsi), %ymm1
-; LINUX-KNL-NEXT:    vmovups %ymm1, 32(%rdi)
-; LINUX-KNL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-KNL-NEXT:    retq
 ;
 ; LINUX-AVX512BW-LABEL: test4:
 ; LINUX-AVX512BW:       # %bb.0: # %entry
 ; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
 ; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
 ; LINUX-AVX512BW-NEXT:    vzeroupper
 ; LINUX-AVX512BW-NEXT:    retq

test/CodeGen/X86/memset-nonzero.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse  | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2 | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2,-slow-unaligned-mem-16 | FileCheck %s --check-prefix=SSE2FAST
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx  | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512-ymm
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512-ymm
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512dq -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512-ymm
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ; https://llvm.org/bugs/show_bug.cgi?id=27100
 
 define void @memset_16_nonzero_bytes(i8* %x) {
 ; SSE-LABEL: memset_16_nonzero_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movabsq $3038287259199220266, %rax # imm = 0x2A2A2A2A2A2A2A2A
 ; SSE-NEXT:    movq %rax, 8(%rdi)

 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movaps {{.*#+}} xmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
 ; SSE2FAST-NEXT:    movups %xmm0, 48(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 32(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 16(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_64_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_64_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_64_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-ymm-LABEL: memset_64_nonzero_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512-ymm-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512NW-NEXT: retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 64, i64 -1)
   ret void
 }
 
 define void @memset_128_nonzero_bytes(i8* %x) {
 ; SSE-LABEL: memset_128_nonzero_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movabsq $3038287259199220266, %rax # imm = 0x2A2A2A2A2A2A2A2A

 ; SSE2FAST-NEXT:    movups %xmm0, 80(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 64(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 48(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 32(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 16(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_128_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_128_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_128_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-ymm-LABEL: memset_128_nonzero_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512-ymm-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 128, i64 -1)
   ret void
 }
 
 define void @memset_256_nonzero_bytes(i8* %x) {
 ; SSE-LABEL: memset_256_nonzero_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pushq %rax

 ; SSE2FAST-NEXT:    movups %xmm0, 80(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 64(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 48(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 32(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, 16(%rdi)
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_256_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 224(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 192(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 160(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 128(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_256_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_256_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-ymm-LABEL: memset_256_nonzero_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512-ymm-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovups %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 256, i64 -1)
   ret void
 }
 
 declare i8* @__memset_chk(i8*, i32, i64, i64)
 
 ; Repeat with a non-constant value for the stores.
 

 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovd %esi, %xmm0
 ; AVX2-NEXT:    vpbroadcastb %xmm0, %ymm0
 ; AVX2-NEXT:    vmovdqu %ymm0, 32(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: memset_64_nonconst_bytes:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd %esi, %xmm0
-; AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512-NEXT:    vmovdqu %ymm0, 32(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512-ymm-LABEL: memset_64_nonconst_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovd %esi, %xmm0
+; AVX512-ymm-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 64, i1 false)
   ret void
 }
 
 define void @memset_128_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE-LABEL: memset_128_nonconst_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    # kill: def $esi killed $esi def $rsi

 ; AVX2-NEXT:    vpbroadcastb %xmm0, %ymm0
 ; AVX2-NEXT:    vmovdqu %ymm0, 96(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, 64(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, 32(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: memset_128_nonconst_bytes:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd %esi, %xmm0
-; AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512-NEXT:    vmovdqu %ymm0, 96(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 64(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 32(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512-ymm-LABEL: memset_128_nonconst_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovd %esi, %xmm0
+; AVX512-ymm-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 96(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 64(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 128, i1 false)
   ret void
 }
 
 define void @memset_256_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE-LABEL: memset_256_nonconst_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl $256, %edx # imm = 0x100

 ; AVX2-NEXT:    vmovdqu %ymm0, 128(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, 96(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, 64(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, 32(%rdi)
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: memset_256_nonconst_bytes:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovd %esi, %xmm0
-; AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512-NEXT:    vmovdqu %ymm0, 224(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 192(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 160(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 128(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 96(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 64(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, 32(%rdi)
-; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX512-ymm-LABEL: memset_256_nonconst_bytes:
+; AVX512-ymm:       # %bb.0:
+; AVX512-ymm-NEXT:    vmovd %esi, %xmm0
+; AVX512-ymm-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 224(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 192(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 160(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 128(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 96(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 64(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, 32(%rdi)
+; AVX512-ymm-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX512-ymm-NEXT:    vzeroupper
+; AVX512-ymm-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 256, i1 false)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 

test/CodeGen/X86/memset-zero.ll

 ; SKYLAKE-NEXT:    vmovups %ymm0, 32(%rdi)
 ; SKYLAKE-NEXT:    vmovups %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
 ;
 ; KNL-LABEL: memset_64:
 ; KNL:       # %bb.0: # %entry
 ; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vmovups %ymm0, 32(%rdi)
-; KNL-NEXT:    vmovups %ymm0, (%rdi)
+; KNL-NEXT:    vmovups %zmm0, (%rdi)
 ; KNL-NEXT:    retq
 entry:
   call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i1 false)
   ret void
 }
 
 define void @memset_64_align64(i8* %a) nounwind  {
 ; X86-LABEL: memset_64_align64:

 ; SKYLAKE-NEXT:    vmovaps %ymm0, 32(%rdi)
 ; SKYLAKE-NEXT:    vmovaps %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
 ;
 ; KNL-LABEL: memset_64_align64:
 ; KNL:       # %bb.0: # %entry
 ; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; KNL-NEXT:    vmovaps %ymm0, 32(%rdi)
-; KNL-NEXT:    vmovaps %ymm0, (%rdi)
+; KNL-NEXT:    vmovaps %zmm0, (%rdi)
 ; KNL-NEXT:    retq
 entry:
   call void @llvm.memset.p0i8.i64(i8* align 64 %a, i8 0, i64 64, i1 false)
   ret void
 }