Prefer AVX512 memcpy when applicable

[davezarzycki]

Summary

When AVX512 is available and the preferred vector width is 512-bits or more, we should prefer AVX512 for memcpy().

https://bugs.llvm.org/show_bug.cgi?id=43240

[craig.topper]

Do we have a similar test for memset that should be updated?

llvm/lib/Target/X86/X86ISelLowering.cpp

2156

VLX shouldn’t be needed. That’s only for 128/256 bit vectors.

[davezarzycki]

The patch no longer depends on the AVX512 vector length extension.

[davezarzycki]

In D67874#1677892, @craig.topper wrote:

Do we have a similar test for memset that should be updated?

Hi @craig.topper – I can try and update a test of memset, but I don't see one that is trying to be comprehensive. For whatever it may be worth, the test suite passes (but that's not an accomplishment). Do you have a request?

[craig.topper]

I added some memset tests in r372494 and the RUN line changes here. Can you rebase?

[davezarzycki]

Hi @craig.topper – Thanks for writing the memset tests. I've expanded the non-zero one to also handle different preferred vector sizes.

[craig.topper]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

This doesn't look like it was generated by the update_llc_test_checks script.

[davezarzycki]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

How so? I tried running update_llc_test_checks against the test and nothing changed.

[craig.topper]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

I've never seen the script mix prefixes like this. As far as I know it always creates a block with the same prefix separated by a blank line from the other prefixes.

[RKSimon]

llvm/test/CodeGen/X86/memset-nonzero.ll

512 ↗

(On Diff #221217)

Something's definitely gone wrong here, it might be that you've not set the --llc-binary args correctly? I'd recommend manually deleting the all the 'AVX512' checks from all these cases and then regenerating to see what happens.

[davezarzycki]

This patch has now been run through update_llc_test_checks.

[craig.topper]

LGTM

[davezarzycki]

r372540

llvm/lib/Target/X86/X86ISelLowering.cpp

 EVT X86TargetLowering::getOptimalMemOpType(
     uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
     bool ZeroMemset, bool MemcpyStrSrc,
     const AttributeList &FuncAttributes) const {
   if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
     if (Size >= 16 && (!Subtarget.isUnalignedMem16Slow() ||
                        ((DstAlign == 0 || DstAlign >= 16) &&
                         (SrcAlign == 0 || SrcAlign >= 16)))) {
+      // FIXME: Check if unaligned 64-byte accesses are slow.
+      if (Size >= 64 && Subtarget.hasAVX512() &&
+          (Subtarget.getPreferVectorWidth() >= 512)) {
+        if (Subtarget.hasBWI() && Subtarget.hasVLX())
+          return MVT::v64i8;
+        return MVT::v16i32;
+      }
       // FIXME: Check if unaligned 32-byte accesses are slow.
       if (Size >= 32 && Subtarget.hasAVX() &&
           (Subtarget.getPreferVectorWidth() >= 256)) {
         // Although this isn't a well-supported type for AVX1, we'll let
         // legalization and shuffle lowering produce the optimal codegen. If we
         // choose an optimal type with a vector element larger than a byte,
         // getMemsetStores() may create an intermediate splat (using an integer
         // multiply) before we splat as a vector.

llvm/test/CodeGen/X86/memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 | FileCheck %s -check-prefix=LINUX
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-apple-darwin      -mcpu=core2   | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2   | FileCheck %s -check-prefix=LINUX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake | FileCheck %s -check-prefix=LINUX-SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx     | FileCheck %s -check-prefix=LINUX-SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=knl     | FileCheck %s -check-prefix=LINUX-KNL
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 addrspace(256)* nocapture, i64, i1) nounwind
 
 
 ; Variable memcpy's should lower to calls.
 define i8* @test1(i8* %a, i8* %b, i64 %n) nounwind {
 ; LINUX-LABEL: test1:

 ; hurting performance so it should just ignore optsize when expanding memcpy.
 ; rdar://8821501
 define void @test3(i8* nocapture %A, i8* nocapture %B) nounwind optsize noredzone {
 ; LINUX-LABEL: test3:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movl $64, %edx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
+; LINUX-SKL-LABEL: test3:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
 ; DARWIN-LABEL: test3:
 ; DARWIN:       ## %bb.0: ## %entry
 ; DARWIN-NEXT:    movq 56(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 56(%rdi)
 ; DARWIN-NEXT:    movq 48(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 48(%rdi)
 ; DARWIN-NEXT:    movq 40(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 40(%rdi)

 
 define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize noredzone {
 ; LINUX-LABEL: test3_minsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
+; LINUX-SKL-LABEL: test3_minsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
 ; DARWIN-LABEL: test3_minsize:
 ; DARWIN:       ## %bb.0:
 ; DARWIN-NEXT:    pushq $64
 ; DARWIN-NEXT:    popq %rdx
 ; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind optsize minsize noredzone {
 ; LINUX-LABEL: test3_minsize_optsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
+; LINUX-SKL-LABEL: test3_minsize_optsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize_optsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize_optsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
 ; DARWIN-LABEL: test3_minsize_optsize:
 ; DARWIN:       ## %bb.0:
 ; DARWIN-NEXT:    pushq $64
 ; DARWIN-NEXT:    popq %rdx
 ; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }

 ; LINUX-NEXT:    movq 16(%rsi), %rax
 ; LINUX-NEXT:    movq %rax, 16(%rdi)
 ; LINUX-NEXT:    movq (%rsi), %rax
 ; LINUX-NEXT:    movq 8(%rsi), %rcx
 ; LINUX-NEXT:    movq %rcx, 8(%rdi)
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
+; LINUX-SKL-LABEL: test4:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test4:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test4:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
 ; DARWIN-LABEL: test4:
 ; DARWIN:       ## %bb.0: ## %entry
 ; DARWIN-NEXT:    movq 56(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 56(%rdi)
 ; DARWIN-NEXT:    movq 48(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 48(%rdi)
 ; DARWIN-NEXT:    movq 40(%rsi), %rax
 ; DARWIN-NEXT:    movq %rax, 40(%rdi)