[X86] Fix 48/96 byte memcmp code gen

[davezarzycki]

Summary

Detect scalar ISD::ZERO_EXTEND generated by memcmp lowering and convert it to ISD::INSERT_SUBVECTOR.

[craig.topper]

LGTM

[davezarzycki]

657e4240b15ffb8a24c5a704a927a7848f3f40ee

lib/Target/X86/X86ISelLowering.cpp

if ((OpSize == 128 && Subtarget.hasSSE2()) ||
 
     EVT VecVT = MVT::v16i8;
     EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;
     if (OpSize == 256) {
       VecVT = MVT::v32i8;
       CmpVT = PreferKOT ? MVT::v32i1 : VecVT;
     }
     EVT CastVT = VecVT;
+    bool NeedsAVX512FCast = false;
     if (OpSize == 512 || NeedZExt) {
       if (Subtarget.hasBWI()) {
         VecVT = MVT::v64i8;
         CmpVT = MVT::v64i1;
         if (OpSize == 512)
           CastVT = VecVT;
       } else {
         VecVT = MVT::v16i32;
         CmpVT = MVT::v16i1;
         CastVT = OpSize == 512 ? VecVT :
                  OpSize == 256 ? MVT::v8i32 : MVT::v4i32;
+        NeedsAVX512FCast = true;
       }
     }
 
     auto ScalarToVector = [&](SDValue X) -> SDValue {
-      X = DAG.getBitcast(CastVT, X);
-      if (!NeedZExt)
+      bool TmpZext = false;
+      EVT TmpCastVT = CastVT;
+      if (X.getOpcode() == ISD::ZERO_EXTEND) {
+        SDValue OrigX = X.getOperand(0);
+        unsigned OrigSize = OrigX.getScalarValueSizeInBits();
+        if (OrigSize < OpSize) {
+          if (OrigSize == 128) {
+            TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;
+            X = OrigX;
+            TmpZext = true;
+          } else if (OrigSize == 256) {
+            TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;
+            X = OrigX;
+            TmpZext = true;
+          }
+        }
+      }
+      X = DAG.getBitcast(TmpCastVT, X);
+      if (!NeedZExt && !TmpZext)
         return X;
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       MVT VecIdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
       return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
                          DAG.getConstant(0, DL, VecVT), X,
                          DAG.getConstant(0, DL, VecIdxVT));
     };
 

test/CodeGen/X86/memcmp.ll

 ; X64-AVX1-NEXT:    callq memcmp
 ; X64-AVX1-NEXT:    testl %eax, %eax
 ; X64-AVX1-NEXT:    sete %al
 ; X64-AVX1-NEXT:    popq %rcx
 ; X64-AVX1-NEXT:    retq
 ;
 ; X64-AVX2-LABEL: length48_eq:
 ; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    movq 32(%rdi), %rcx
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $8, %edx
-; X64-AVX2-NEXT:    vmovd %ecx, %xmm0
-; X64-AVX2-NEXT:    vpinsrb $1, %edx, %xmm0, %xmm0
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $16, %edx
-; X64-AVX2-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $24, %edx
-; X64-AVX2-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $32, %rdx
-; X64-AVX2-NEXT:    vpinsrb $4, %edx, %xmm0, %xmm0
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $40, %rcx
-; X64-AVX2-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm1
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    movq 40(%rdi), %rcx
-; X64-AVX2-NEXT:    shrq $48, %rdx
-; X64-AVX2-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $56, %rdx
-; X64-AVX2-NEXT:    shrq $56, %rax
-; X64-AVX2-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $8, %eax
-; X64-AVX2-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $16, %eax
-; X64-AVX2-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $24, %eax
-; X64-AVX2-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    shrq $32, %rax
-; X64-AVX2-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    shrq $48, %rax
-; X64-AVX2-NEXT:    shrq $40, %rcx
-; X64-AVX2-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movq 32(%rsi), %rcx
-; X64-AVX2-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $8, %edx
-; X64-AVX2-NEXT:    vmovd %ecx, %xmm2
-; X64-AVX2-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $16, %edx
-; X64-AVX2-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movl %ecx, %edx
-; X64-AVX2-NEXT:    shrl $24, %edx
-; X64-AVX2-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $32, %rdx
-; X64-AVX2-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $40, %rcx
-; X64-AVX2-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq 40(%rsi), %rcx
-; X64-AVX2-NEXT:    shrq $48, %rdx
-; X64-AVX2-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq %rcx, %rdx
-; X64-AVX2-NEXT:    shrq $56, %rax
-; X64-AVX2-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $8, %eax
-; X64-AVX2-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $16, %eax
-; X64-AVX2-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movl %ecx, %eax
-; X64-AVX2-NEXT:    shrl $24, %eax
-; X64-AVX2-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    shrq $32, %rax
-; X64-AVX2-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    movq %rcx, %rax
-; X64-AVX2-NEXT:    shrq $40, %rcx
-; X64-AVX2-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
-; X64-AVX2-NEXT:    shrq $48, %rax
-; X64-AVX2-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
-; X64-AVX2-NEXT:    shrq $56, %rdx
-; X64-AVX2-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX2-NEXT:    vmovdqu 32(%rsi), %xmm2
 ; X64-AVX2-NEXT:    vpxor (%rsi), %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
 ; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    sete %al
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
 ;
 ; X64-AVX512-LABEL: length48_eq:
 ; X64-AVX512:       # %bb.0:
-; X64-AVX512-NEXT:    movq 32(%rdi), %rcx
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $8, %edx
-; X64-AVX512-NEXT:    vmovd %ecx, %xmm0
-; X64-AVX512-NEXT:    vpinsrb $1, %edx, %xmm0, %xmm0
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $16, %edx
-; X64-AVX512-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $24, %edx
-; X64-AVX512-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $32, %rdx
-; X64-AVX512-NEXT:    vpinsrb $4, %edx, %xmm0, %xmm0
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $40, %rcx
-; X64-AVX512-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm1
 ; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX512-NEXT:    movq 40(%rdi), %rcx
-; X64-AVX512-NEXT:    shrq $48, %rdx
-; X64-AVX512-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $56, %rdx
-; X64-AVX512-NEXT:    shrq $56, %rax
-; X64-AVX512-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $8, %eax
-; X64-AVX512-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $16, %eax
-; X64-AVX512-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $24, %eax
-; X64-AVX512-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    shrq $32, %rax
-; X64-AVX512-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    shrq $48, %rax
-; X64-AVX512-NEXT:    shrq $40, %rcx
-; X64-AVX512-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movq 32(%rsi), %rcx
-; X64-AVX512-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $8, %edx
-; X64-AVX512-NEXT:    vmovd %ecx, %xmm2
-; X64-AVX512-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $16, %edx
-; X64-AVX512-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movl %ecx, %edx
-; X64-AVX512-NEXT:    shrl $24, %edx
-; X64-AVX512-NEXT:    vpinsrb $3, %edx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $32, %rdx
-; X64-AVX512-NEXT:    vpinsrb $4, %edx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $40, %rcx
-; X64-AVX512-NEXT:    vpinsrb $5, %ecx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq 40(%rsi), %rcx
-; X64-AVX512-NEXT:    shrq $48, %rdx
-; X64-AVX512-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq %rcx, %rdx
-; X64-AVX512-NEXT:    shrq $56, %rax
-; X64-AVX512-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $8, %eax
-; X64-AVX512-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $16, %eax
-; X64-AVX512-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movl %ecx, %eax
-; X64-AVX512-NEXT:    shrl $24, %eax
-; X64-AVX512-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    shrq $32, %rax
-; X64-AVX512-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    movq %rcx, %rax
-; X64-AVX512-NEXT:    shrq $40, %rcx
-; X64-AVX512-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
-; X64-AVX512-NEXT:    shrq $48, %rax
-; X64-AVX512-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
-; X64-AVX512-NEXT:    shrq $56, %rdx
-; X64-AVX512-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm2
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-AVX512-NEXT:    vmovdqu 32(%rsi), %xmm2
 ; X64-AVX512-NEXT:    vpxor (%rsi), %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vpxor %ymm2, %ymm1, %ymm1
 ; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX512-NEXT:    sete %al
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
 ;
 ; X64-MIC-AVX-LABEL: length48_eq:
 ; X64-MIC-AVX:       # %bb.0:
 ; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
 ; X64-MIC-AVX-NEXT:    vmovdqu (%rsi), %ymm1
-; X64-MIC-AVX-NEXT:    movq 40(%rdi), %rax
-; X64-MIC-AVX-NEXT:    movq 32(%rdi), %rcx
-; X64-MIC-AVX-NEXT:    vmovd %ecx, %xmm2
-; X64-MIC-AVX-NEXT:    shrq $32, %rcx
-; X64-MIC-AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; X64-MIC-AVX-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
-; X64-MIC-AVX-NEXT:    shrq $32, %rax
-; X64-MIC-AVX-NEXT:    movq 40(%rsi), %rcx
-; X64-MIC-AVX-NEXT:    movq 32(%rsi), %rdx
-; X64-MIC-AVX-NEXT:    vmovd %edx, %xmm3
-; X64-MIC-AVX-NEXT:    shrq $32, %rdx
-; X64-MIC-AVX-NEXT:    vpinsrd $1, %edx, %xmm3, %xmm3
-; X64-MIC-AVX-NEXT:    vpinsrd $2, %ecx, %xmm3, %xmm3
-; X64-MIC-AVX-NEXT:    shrq $32, %rcx
-; X64-MIC-AVX-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm2
-; X64-MIC-AVX-NEXT:    vpinsrd $3, %ecx, %xmm3, %xmm3
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm2
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rsi), %xmm3
 ; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm3, %zmm2, %k0
 ; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
 ; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
 ; X64-MIC-AVX-NEXT:    sete %al
 ; X64-MIC-AVX-NEXT:    vzeroupper
 ; X64-MIC-AVX-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 48) nounwind
   %cmp = icmp eq i32 %call, 0

 ; X64-AVX1-NEXT:    callq memcmp
 ; X64-AVX1-NEXT:    testl %eax, %eax
 ; X64-AVX1-NEXT:    setne %al
 ; X64-AVX1-NEXT:    popq %rcx
 ; X64-AVX1-NEXT:    retq
 ;
 ; X64-AVX2-LABEL: length48_eq_const:
 ; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    pushq %rbp
-; X64-AVX2-NEXT:    pushq %r15
-; X64-AVX2-NEXT:    pushq %r14
-; X64-AVX2-NEXT:    pushq %r12
-; X64-AVX2-NEXT:    pushq %rbx
 ; X64-AVX2-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX2-NEXT:    movq 40(%rdi), %rcx
-; X64-AVX2-NEXT:    movq %rcx, %r8
-; X64-AVX2-NEXT:    shrq $56, %r8
-; X64-AVX2-NEXT:    movq %rcx, %r9
-; X64-AVX2-NEXT:    shrq $48, %r9
-; X64-AVX2-NEXT:    movq %rcx, %r10
-; X64-AVX2-NEXT:    shrq $32, %r10
-; X64-AVX2-NEXT:    movl %ecx, %r11d
-; X64-AVX2-NEXT:    shrl $24, %r11d
-; X64-AVX2-NEXT:    movl %ecx, %r14d
-; X64-AVX2-NEXT:    shrl $16, %r14d
-; X64-AVX2-NEXT:    movl %ecx, %r15d
-; X64-AVX2-NEXT:    shrl $8, %r15d
-; X64-AVX2-NEXT:    movq 32(%rdi), %rdi
-; X64-AVX2-NEXT:    movq %rdi, %r12
-; X64-AVX2-NEXT:    shrq $56, %r12
-; X64-AVX2-NEXT:    movq %rdi, %rbx
-; X64-AVX2-NEXT:    shrq $48, %rbx
-; X64-AVX2-NEXT:    movq %rdi, %rdx
-; X64-AVX2-NEXT:    shrq $32, %rdx
-; X64-AVX2-NEXT:    movl %edi, %ebp
-; X64-AVX2-NEXT:    shrl $24, %ebp
-; X64-AVX2-NEXT:    movl %edi, %esi
-; X64-AVX2-NEXT:    shrl $16, %esi
-; X64-AVX2-NEXT:    vmovd %edi, %xmm1
-; X64-AVX2-NEXT:    movl %edi, %eax
-; X64-AVX2-NEXT:    shrl $8, %eax
-; X64-AVX2-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $2, %esi, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $3, %ebp, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    shrq $40, %rdi
-; X64-AVX2-NEXT:    vpinsrb $5, %edi, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $6, %ebx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $7, %r12d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $9, %r15d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $11, %r11d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $12, %r10d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    shrq $40, %rcx
-; X64-AVX2-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $14, %r9d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpinsrb $15, %r8d, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX2-NEXT:    vmovdqu 32(%rdi), %xmm1
 ; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
 ; X64-AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX2-NEXT:    setne %al
-; X64-AVX2-NEXT:    popq %rbx
-; X64-AVX2-NEXT:    popq %r12
-; X64-AVX2-NEXT:    popq %r14
-; X64-AVX2-NEXT:    popq %r15
-; X64-AVX2-NEXT:    popq %rbp
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
 ;
 ; X64-AVX512-LABEL: length48_eq_const:
 ; X64-AVX512:       # %bb.0:
-; X64-AVX512-NEXT:    pushq %rbp
-; X64-AVX512-NEXT:    pushq %r15
-; X64-AVX512-NEXT:    pushq %r14
-; X64-AVX512-NEXT:    pushq %r12
-; X64-AVX512-NEXT:    pushq %rbx
 ; X64-AVX512-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-AVX512-NEXT:    movq 40(%rdi), %rcx
-; X64-AVX512-NEXT:    movq %rcx, %r8
-; X64-AVX512-NEXT:    shrq $56, %r8
-; X64-AVX512-NEXT:    movq %rcx, %r9
-; X64-AVX512-NEXT:    shrq $48, %r9
-; X64-AVX512-NEXT:    movq %rcx, %r10
-; X64-AVX512-NEXT:    shrq $32, %r10
-; X64-AVX512-NEXT:    movl %ecx, %r11d
-; X64-AVX512-NEXT:    shrl $24, %r11d
-; X64-AVX512-NEXT:    movl %ecx, %r14d
-; X64-AVX512-NEXT:    shrl $16, %r14d
-; X64-AVX512-NEXT:    movl %ecx, %r15d
-; X64-AVX512-NEXT:    shrl $8, %r15d
-; X64-AVX512-NEXT:    movq 32(%rdi), %rdi
-; X64-AVX512-NEXT:    movq %rdi, %r12
-; X64-AVX512-NEXT:    shrq $56, %r12
-; X64-AVX512-NEXT:    movq %rdi, %rbx
-; X64-AVX512-NEXT:    shrq $48, %rbx
-; X64-AVX512-NEXT:    movq %rdi, %rdx
-; X64-AVX512-NEXT:    shrq $32, %rdx
-; X64-AVX512-NEXT:    movl %edi, %ebp
-; X64-AVX512-NEXT:    shrl $24, %ebp
-; X64-AVX512-NEXT:    movl %edi, %esi
-; X64-AVX512-NEXT:    shrl $16, %esi
-; X64-AVX512-NEXT:    vmovd %edi, %xmm1
-; X64-AVX512-NEXT:    movl %edi, %eax
-; X64-AVX512-NEXT:    shrl $8, %eax
-; X64-AVX512-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $2, %esi, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $3, %ebp, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    shrq $40, %rdi
-; X64-AVX512-NEXT:    vpinsrb $5, %edi, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $6, %ebx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $7, %r12d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $9, %r15d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $10, %r14d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $11, %r11d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $12, %r10d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    shrq $40, %rcx
-; X64-AVX512-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $14, %r9d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpinsrb $15, %r8d, %xmm1, %xmm1
-; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
+; X64-AVX512-NEXT:    vmovdqu 32(%rdi), %xmm1
 ; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %ymm1, %ymm1
 ; X64-AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vptest %ymm0, %ymm0
 ; X64-AVX512-NEXT:    setne %al
-; X64-AVX512-NEXT:    popq %rbx
-; X64-AVX512-NEXT:    popq %r12
-; X64-AVX512-NEXT:    popq %r14
-; X64-AVX512-NEXT:    popq %r15
-; X64-AVX512-NEXT:    popq %rbp
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
 ;
 ; X64-MIC-AVX-LABEL: length48_eq_const:
 ; X64-MIC-AVX:       # %bb.0:
 ; X64-MIC-AVX-NEXT:    vmovdqu (%rdi), %ymm0
-; X64-MIC-AVX-NEXT:    movq 40(%rdi), %rax
-; X64-MIC-AVX-NEXT:    movq 32(%rdi), %rcx
-; X64-MIC-AVX-NEXT:    vmovd %ecx, %xmm1
-; X64-MIC-AVX-NEXT:    shrq $32, %rcx
-; X64-MIC-AVX-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; X64-MIC-AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; X64-MIC-AVX-NEXT:    shrq $32, %rax
-; X64-MIC-AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
-; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm0, %k0
-; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [892613426,959985462,858927408,926299444,0,0,0,0]
-; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm0, %zmm1, %k1
-; X64-MIC-AVX-NEXT:    kortestw %k1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqu 32(%rdi), %xmm1
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm2 = [892613426,959985462,858927408,926299444,0,0,0,0]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm2, %zmm1, %k0
+; X64-MIC-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [858927408,926299444,825243960,892613426,959985462,858927408,926299444,825243960]
+; X64-MIC-AVX-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX-NEXT:    kortestw %k0, %k1
 ; X64-MIC-AVX-NEXT:    setne %al
 ; X64-MIC-AVX-NEXT:    vzeroupper
 ; X64-MIC-AVX-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 48) nounwind
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
 

 ; X64-AVX2-NEXT:    callq memcmp
 ; X64-AVX2-NEXT:    testl %eax, %eax
 ; X64-AVX2-NEXT:    setne %al
 ; X64-AVX2-NEXT:    popq %rcx
 ; X64-AVX2-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: length96_eq:
 ; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    movq 80(%rdi), %rcx
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vmovd %ecx, %xmm0
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $5, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq 88(%rdi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $8, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $8, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $16, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $24, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $32, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq 64(%rdi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vmovd %ecx, %xmm1
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $5, %ecx, %xmm1, %xmm2
-; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm1
-; X64-AVX512BW-NEXT:    movq 72(%rdi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $8, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $8, %ecx, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $16, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $24, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $32, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $48, %rax
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $13, %ecx, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movq 80(%rsi), %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm2, %xmm2
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vmovd %ecx, %xmm3
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $5, %ecx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq 88(%rsi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $8, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $8, %ecx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $16, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $24, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $32, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $13, %ecx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq 64(%rsi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm3, %xmm3
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vmovd %ecx, %xmm4
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm4, %xmm4
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm4, %xmm4
-; X64-AVX512BW-NEXT:    movl %ecx, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm4, %xmm4
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm4, %xmm4
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $5, %ecx, %xmm4, %xmm4
-; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm1, %k0
-; X64-AVX512BW-NEXT:    movq 72(%rsi), %rcx
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm4, %xmm1
-; X64-AVX512BW-NEXT:    movq %rcx, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $7, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $8, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $8, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    vpinsrb $9, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $16, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $10, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %ecx, %eax
-; X64-AVX512BW-NEXT:    shrl $24, %eax
-; X64-AVX512BW-NEXT:    vpinsrb $11, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $32, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $12, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rcx, %rax
-; X64-AVX512BW-NEXT:    shrq $40, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    shrq $48, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $14, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
-; X64-AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; X64-AVX512BW-NEXT:    vpcmpneqb %zmm1, %zmm0, %k1
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512BW-NEXT:    vpcmpneqb (%rsi), %zmm0, %k0
+; X64-AVX512BW-NEXT:    vpcmpneqb %zmm2, %zmm1, %k1
 ; X64-AVX512BW-NEXT:    kortestq %k1, %k0
 ; X64-AVX512BW-NEXT:    setne %al
 ; X64-AVX512BW-NEXT:    vzeroupper
 ; X64-AVX512BW-NEXT:    retq
 ;
 ; X64-AVX512F-LABEL: length96_eq:
 ; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    movq 80(%rdi), %rax
-; X64-AVX512F-NEXT:    vmovd %eax, %xmm0
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-AVX512F-NEXT:    movq 88(%rdi), %rax
-; X64-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; X64-AVX512F-NEXT:    movq 64(%rdi), %rax
-; X64-AVX512F-NEXT:    vmovd %eax, %xmm1
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm2
-; X64-AVX512F-NEXT:    movq 72(%rdi), %rax
-; X64-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    movq 80(%rsi), %rax
-; X64-AVX512F-NEXT:    vmovd %eax, %xmm3
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm3, %xmm3
-; X64-AVX512F-NEXT:    movq 88(%rsi), %rax
-; X64-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm3
-; X64-AVX512F-NEXT:    movq 64(%rsi), %rax
-; X64-AVX512F-NEXT:    vmovd %eax, %xmm4
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm4, %xmm4
-; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm2, %k0
-; X64-AVX512F-NEXT:    movq 72(%rsi), %rax
-; X64-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm2
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm2
-; X64-AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; X64-AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1
-; X64-AVX512F-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
 ; X64-AVX512F-NEXT:    kortestw %k1, %k0
 ; X64-AVX512F-NEXT:    setne %al
 ; X64-AVX512F-NEXT:    vzeroupper
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-MIC-AVX2-LABEL: length96_eq:
 ; X64-MIC-AVX2:       # %bb.0:
 ; X64-MIC-AVX2-NEXT:    pushq %rax
 ; X64-MIC-AVX2-NEXT:    movl $96, %edx
 ; X64-MIC-AVX2-NEXT:    callq memcmp
 ; X64-MIC-AVX2-NEXT:    testl %eax, %eax
 ; X64-MIC-AVX2-NEXT:    setne %al
 ; X64-MIC-AVX2-NEXT:    popq %rcx
 ; X64-MIC-AVX2-NEXT:    retq
 ;
 ; X64-MIC-AVX512F-LABEL: length96_eq:
 ; X64-MIC-AVX512F:       # %bb.0:
-; X64-MIC-AVX512F-NEXT:    movq 80(%rdi), %rax
-; X64-MIC-AVX512F-NEXT:    vmovd %eax, %xmm0
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-MIC-AVX512F-NEXT:    movq 88(%rdi), %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; X64-MIC-AVX512F-NEXT:    movq 64(%rdi), %rax
-; X64-MIC-AVX512F-NEXT:    vmovd %eax, %xmm1
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm2
-; X64-MIC-AVX512F-NEXT:    movq 72(%rdi), %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    movq 80(%rsi), %rax
-; X64-MIC-AVX512F-NEXT:    vmovd %eax, %xmm3
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm3, %xmm3
-; X64-MIC-AVX512F-NEXT:    movq 88(%rsi), %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm3
-; X64-MIC-AVX512F-NEXT:    movq 64(%rsi), %rax
-; X64-MIC-AVX512F-NEXT:    vmovd %eax, %xmm4
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %eax, %xmm4, %xmm4
-; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm2, %k0
-; X64-MIC-AVX512F-NEXT:    movq 72(%rsi), %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm4, %xmm2
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm2
-; X64-MIC-AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; X64-MIC-AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1
-; X64-MIC-AVX512F-NEXT:    vpcmpneqd %zmm1, %zmm0, %k1
+; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rsi), %ymm2
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd (%rsi), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd %zmm2, %zmm1, %k1
 ; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
 ; X64-MIC-AVX512F-NEXT:    setne %al
 ; X64-MIC-AVX512F-NEXT:    vzeroupper
 ; X64-MIC-AVX512F-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 96) nounwind
   %cmp = icmp ne i32 %call, 0
   ret i1 %cmp
 }

 ; X64-AVX2-NEXT:    callq memcmp
 ; X64-AVX2-NEXT:    testl %eax, %eax
 ; X64-AVX2-NEXT:    sete %al
 ; X64-AVX2-NEXT:    popq %rcx
 ; X64-AVX2-NEXT:    retq
 ;
 ; X64-AVX512BW-LABEL: length96_eq_const:
 ; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    movq 80(%rdi), %rax
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    vmovd %eax, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq 88(%rdi), %rax
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $7, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $8, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $16, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $24, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    shrq $32, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $12, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    shrq $40, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq 64(%rdi), %rax
-; X64-AVX512BW-NEXT:    shrq $48, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $14, %ecx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm0, %xmm0
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $8, %edx
-; X64-AVX512BW-NEXT:    vmovd %eax, %xmm1
-; X64-AVX512BW-NEXT:    vpinsrb $1, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $16, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $2, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %eax, %edx
-; X64-AVX512BW-NEXT:    shrl $24, %edx
-; X64-AVX512BW-NEXT:    vpinsrb $3, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $32, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $4, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $40, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $5, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm2
-; X64-AVX512BW-NEXT:    movq 72(%rdi), %rax
-; X64-AVX512BW-NEXT:    shrq $48, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rax, %rdx
-; X64-AVX512BW-NEXT:    shrq $56, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $8, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $8, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    vpinsrb $9, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $16, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $10, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movl %eax, %ecx
-; X64-AVX512BW-NEXT:    shrl $24, %ecx
-; X64-AVX512BW-NEXT:    vpinsrb $11, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    shrq $32, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $12, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    movq %rax, %rcx
-; X64-AVX512BW-NEXT:    shrq $40, %rax
-; X64-AVX512BW-NEXT:    vpinsrb $13, %eax, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    shrq $48, %rcx
-; X64-AVX512BW-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    shrq $56, %rdx
-; X64-AVX512BW-NEXT:    vpinsrb $15, %edx, %xmm1, %xmm1
-; X64-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; X64-AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm0
+; X64-AVX512BW-NEXT:    vmovdqu 64(%rdi), %ymm1
 ; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm0, %k0
-; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm2, %k1
-; X64-AVX512BW-NEXT:    kortestq %k0, %k1
+; X64-AVX512BW-NEXT:    vpcmpneqb {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512BW-NEXT:    kortestq %k1, %k0
 ; X64-AVX512BW-NEXT:    sete %al
 ; X64-AVX512BW-NEXT:    vzeroupper
 ; X64-AVX512BW-NEXT:    retq
 ;
 ; X64-AVX512F-LABEL: length96_eq_const:
 ; X64-AVX512F:       # %bb.0:
 ; X64-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
-; X64-AVX512F-NEXT:    movq 72(%rdi), %rax
-; X64-AVX512F-NEXT:    movq 64(%rdi), %rcx
-; X64-AVX512F-NEXT:    vmovd %ecx, %xmm1
-; X64-AVX512F-NEXT:    shrq $32, %rcx
-; X64-AVX512F-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    shrq $32, %rax
-; X64-AVX512F-NEXT:    movq 88(%rdi), %rcx
-; X64-AVX512F-NEXT:    movq 80(%rdi), %rdx
-; X64-AVX512F-NEXT:    vmovd %edx, %xmm2
-; X64-AVX512F-NEXT:    shrq $32, %rdx
-; X64-AVX512F-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
-; X64-AVX512F-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
-; X64-AVX512F-NEXT:    shrq $32, %rcx
-; X64-AVX512F-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm2
-; X64-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X64-AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k0
-; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
-; X64-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-AVX512F-NEXT:    kortestw %k1, %k0
 ; X64-AVX512F-NEXT:    sete %al
 ; X64-AVX512F-NEXT:    vzeroupper
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X64-MIC-AVX2-LABEL: length96_eq_const:
 ; X64-MIC-AVX2:       # %bb.0:
 ; X64-MIC-AVX2-NEXT:    pushq %rax
 ; X64-MIC-AVX2-NEXT:    movl $.L.str, %esi
 ; X64-MIC-AVX2-NEXT:    movl $96, %edx
 ; X64-MIC-AVX2-NEXT:    callq memcmp
 ; X64-MIC-AVX2-NEXT:    testl %eax, %eax
 ; X64-MIC-AVX2-NEXT:    sete %al
 ; X64-MIC-AVX2-NEXT:    popq %rcx
 ; X64-MIC-AVX2-NEXT:    retq
 ;
 ; X64-MIC-AVX512F-LABEL: length96_eq_const:
 ; X64-MIC-AVX512F:       # %bb.0:
 ; X64-MIC-AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0
-; X64-MIC-AVX512F-NEXT:    movq 72(%rdi), %rax
-; X64-MIC-AVX512F-NEXT:    movq 64(%rdi), %rcx
-; X64-MIC-AVX512F-NEXT:    vmovd %ecx, %xmm1
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rcx
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rax
-; X64-MIC-AVX512F-NEXT:    movq 88(%rdi), %rcx
-; X64-MIC-AVX512F-NEXT:    movq 80(%rdi), %rdx
-; X64-MIC-AVX512F-NEXT:    vmovd %edx, %xmm2
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rdx
-; X64-MIC-AVX512F-NEXT:    vpinsrd $1, %edx, %xmm2, %xmm2
-; X64-MIC-AVX512F-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
-; X64-MIC-AVX512F-NEXT:    shrq $32, %rcx
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm2
-; X64-MIC-AVX512F-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X64-MIC-AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k0
-; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k1
-; X64-MIC-AVX512F-NEXT:    kortestw %k0, %k1
+; X64-MIC-AVX512F-NEXT:    vmovdqu 64(%rdi), %ymm1
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm0, %k0
+; X64-MIC-AVX512F-NEXT:    vpcmpneqd {{.*}}(%rip), %zmm1, %k1
+; X64-MIC-AVX512F-NEXT:    kortestw %k1, %k0
 ; X64-MIC-AVX512F-NEXT:    sete %al
 ; X64-MIC-AVX512F-NEXT:    vzeroupper
 ; X64-MIC-AVX512F-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 0), i64 96) nounwind
   %c = icmp eq i32 %m, 0
   ret i1 %c
 }