X86: Load and ZeroExtend i1 value

[delena]

Summary

Optimized loading (zextload) of i1 value from memory.
This patch is a partial revert of https://llvm.org/svn/llvm-project/llvm/trunk@237793.
Extra "and" causes performance degradation.

We assume that i1 is stored in zero-extended form. And store operation is responsible for zeroing upper bits.
I described this assumption in the documentation.

[qcolombet]

../docs/LangRef.rst

1756

I don’t think this part of the patch makes sense.
This may be the right thing to do for x86 but maybe not for all the targets. Ih other words, I don’t think we want to make this a requirement for all targets, unless this was already a requirement we forgot to document so far?

../lib/Target/X86/X86InstrCompiler.td

1142

Add the comment regarding i1 being zero extended before store here.

[delena]

I moved the comment regarding i1 memory form from the common LangRef to X86InstrCompiler.td file.

[qcolombet]

Hi Elena,

LGTM.

Thanks,
-Quentin

[DavidKreitzer]

Thanks for doing this, Elena. LGTM also.

../docs/LangRef.rst

 -  ``f16:16:16`` - half is 16-bit aligned
 -  ``f32:32:32`` - float is 32-bit aligned
 -  ``f64:64:64`` - double is 64-bit aligned
 -  ``f128:128:128`` - quad is 128-bit aligned
 -  ``v64:64:64`` - 64-bit vector is 64-bit aligned
 -  ``v128:128:128`` - 128-bit vector is 128-bit aligned
 -  ``a:0:64`` - aggregates are 64-bit aligned
 
+i1 is a 1-bit integer type stored in 8-bit (byte) in zero-extended form.
+Upper bits cleanup should be executed before ``store``.
+
 When LLVM is determining the alignment for a given type, it uses the
 following rules:
 
 #. If the type sought is an exact match for one of the specifications,
    that specification is used.
 #. If no match is found, and the type sought is an integer type, then
    the smallest integer type that is larger than the bitwidth of the
    sought type is used. If none of the specifications are larger than

../lib/Target/X86/X86InstrAVX512.td

def : Pat<(v32i1 (scalar_to_vector VK1:$src)),
           (COPY_TO_REGCLASS VK1:$src, VK32)>;
 def : Pat<(v64i1 (scalar_to_vector VK1:$src)),
           (COPY_TO_REGCLASS VK1:$src, VK64)>;
 
 def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
 def : Pat<(store (i1  1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
 def : Pat<(store (i1  0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
 
-def truncstorei1 : PatFrag<(ops node:$val, node:$ptr),
-                           (truncstore node:$val, node:$ptr), [{
-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1;
-}]>;
-
-def : Pat<(truncstorei1 GR8:$src, addr:$dst),
-          (MOV8mr addr:$dst, GR8:$src)>;
-
 // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
 let Predicates = [HasAVX512, NoDQI] in {
   // GR from/to 8-bit mask without native support
   def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
             (COPY_TO_REGCLASS
              (KMOVWkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), VK8)>;
   def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
             (EXTRACT_SUBREG

../lib/Target/X86/X86InstrCompiler.td

 defm : CMOVmr<X86_COND_P , CMOVNP16rm, CMOVNP32rm, CMOVNP64rm>;
 defm : CMOVmr<X86_COND_NP, CMOVP16rm , CMOVP32rm , CMOVP64rm>;
 defm : CMOVmr<X86_COND_S , CMOVNS16rm, CMOVNS32rm, CMOVNS64rm>;
 defm : CMOVmr<X86_COND_NS, CMOVS16rm , CMOVS32rm , CMOVS64rm>;
 defm : CMOVmr<X86_COND_O , CMOVNO16rm, CMOVNO32rm, CMOVNO64rm>;
 defm : CMOVmr<X86_COND_NO, CMOVO16rm , CMOVO32rm , CMOVO64rm>;
 
 // zextload bool -> zextload byte
-def : Pat<(zextloadi8i1  addr:$src), (AND8ri (MOV8rm addr:$src), (i8 1))>;
-def : Pat<(zextloadi16i1 addr:$src), (AND16ri8 (MOVZX16rm8 addr:$src), (i16 1))>;
-def : Pat<(zextloadi32i1 addr:$src), (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1))>;
+def : Pat<(zextloadi8i1  addr:$src), (MOV8rm addr:$src)>;
+def : Pat<(zextloadi16i1 addr:$src), (MOVZX16rm8 addr:$src)>;
+def : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>;
 def : Pat<(zextloadi64i1 addr:$src),
-          (SUBREG_TO_REG (i64 0),
-           (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1)), sub_32bit)>;
+          (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>;
 
 // extload bool -> extload byte
 // When extloading from 16-bit and smaller memory locations into 64-bit
 // registers, use zero-extending loads so that the entire 64-bit register is
 // defined, avoiding partial-register updates.
 
 def : Pat<(extloadi8i1 addr:$src),   (MOV8rm      addr:$src)>;
 def : Pat<(extloadi16i1 addr:$src),  (MOVZX16rm8  addr:$src)>;

../test/CodeGen/X86/and-encoding.ll

 
 define void @f1() {
 ; CHECK-LABEL: f1:
 ; CHECK: andq    $-32, %rsp              # encoding: [0x48,0x83,0xe4,0xe0]
   %foo = alloca i8, align 32
   ret void
 }
 
-define void @f2(i1 *%x, i16 *%y) {
+define void @f2(i16 %x, i1 *%y) {
 ; CHECK-LABEL: f2:
-; CHECK: andl $1, %eax                # encoding: [0x83,0xe0,0x01]
-  %a = load i1, i1* %x
-  %b = zext i1 %a to i16
-  store i16 %b, i16* %y
+; CHECK: andl $1, %edi                # encoding: [0x83,0xe7,0x01]
+  %c = trunc i16 %x to i1
+  store i1 %c, i1* %y
   ret void
 }
 
-define i32 @f3(i1 *%x) {
+define void @f3(i32 %x, i1 *%y) {
 ; CHECK-LABEL: f3:
-; CHECK: andl $1, %eax                # encoding: [0x83,0xe0,0x01]
-  %a = load i1, i1* %x
-  %b = zext i1 %a to i32
-  ret i32 %b
-}
-
-define i64 @f4(i1 *%x) {
-; CHECK-LABEL: f4:
-; CHECK: andl $1, %eax                # encoding: [0x83,0xe0,0x01]
-  %a = load i1, i1* %x
-  %b = zext i1 %a to i64
-  ret i64 %b
+; CHECK: andl $1, %edi                # encoding: [0x83,0xe7,0x01]
+  %c = trunc i32 %x to i1
+  store i1 %c, i1* %y
+  ret void
 }

../test/CodeGen/X86/avx512-insert-extract.ll

define i64 @test14(<8 x i64>%a, <8 x i64>%b, i64 %a1, i64 %b1) {
   %cmpvector_func.i = icmp slt <8 x i64> %a, %b
   %extract24vector_func.i = extractelement <8 x i1> %cmpvector_func.i, i32 4
   %res = select i1 %extract24vector_func.i, i64 %a1, i64 %b1
   ret i64 %res
 }
 
 ;CHECK-LABEL: test15
 ;CHECK: movb (%rdi), %al
-;CHECK: andb $1, %al
 ;CHECK: movw    $-1, %ax
 ;CHECK: cmovew
 define i16 @test15(i1 *%addr) {
   %x = load i1 , i1 * %addr, align 1
   %x1 = insertelement <16 x i1> undef, i1 %x, i32 10
   %x2 = bitcast <16 x i1>%x1 to i16
   ret i16 %x2
 }
 
 ;CHECK-LABEL: test16
 ;CHECK: movb (%rdi), %al
-;CHECK: andw $1, %ax
 ;CHECK: kmovw
 ;CHECK: kshiftlw        $10
 ;CHECK: korw
 ;CHECK: ret
 define i16 @test16(i1 *%addr, i16 %a) {
   %x = load i1 , i1 * %addr, align 128
   %a1 = bitcast i16 %a to <16 x i1>
   %x1 = insertelement <16 x i1> %a1, i1 %x, i32 10

../test/CodeGen/X86/avx512-mask-op.ll

 ;}
 
 @f1.v = internal unnamed_addr global i1 false, align 4
 
 define void @f1(i32 %c) {
 ; KNL-LABEL: f1:
 ; KNL:       ## BB#0: ## %entry
 ; KNL-NEXT:    movzbl {{.*}}(%rip), %edi
-; KNL-NEXT:    andl $1, %edi
 ; KNL-NEXT:    movl %edi, %eax
 ; KNL-NEXT:    andl $1, %eax
 ; KNL-NEXT:    kmovw %eax, %k0
 ; KNL-NEXT:    kxnorw %k0, %k0, %k1
 ; KNL-NEXT:    kshiftrw $15, %k1, %k1
 ; KNL-NEXT:    kxorw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, {{.*}}(%rip)
 ; KNL-NEXT:    xorl $1, %edi
 ; KNL-NEXT:    jmp _f2 ## TAILCALL
 ;
 ; SKX-LABEL: f1:
 ; SKX:       ## BB#0: ## %entry
 ; SKX-NEXT:    movzbl {{.*}}(%rip), %edi
-; SKX-NEXT:    andl $1, %edi
 ; SKX-NEXT:    movl %edi, %eax
 ; SKX-NEXT:    andl $1, %eax
 ; SKX-NEXT:    kmovw %eax, %k0
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX-NEXT:    kshiftrw $15, %k1, %k1
 ; SKX-NEXT:    kxorw %k1, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, {{.*}}(%rip)
 ; SKX-NEXT:    xorl $1, %edi
 ; SKX-NEXT:    jmp _f2 ## TAILCALL
 entry:
   %.b1 = load i1, i1* @f1.v, align 4
   %not..b1 = xor i1 %.b1, true
   store i1 %not..b1, i1* @f1.v, align 4
   %0 = zext i1 %not..b1 to i32
   tail call void @f2(i32 %0) #2
   ret void
 }
 
 declare void @f2(i32) #1
 
+define void @store_i16_i1(i16 %x, i1 *%y) {
+; CHECK-LABEL: store_i16_i1:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    movb %dil, (%rsi)
+; CHECK-NEXT:    retq
+  %c = trunc i16 %x to i1
+  store i1 %c, i1* %y
+  ret void
+}
+
+define void @store_i8_i1(i8 %x, i1 *%y) {
+; CHECK-LABEL: store_i8_i1:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    movb %dil, (%rsi)
+; CHECK-NEXT:    retq
+  %c = trunc i8 %x to i1
+  store i1 %c, i1* %y
+  ret void
+}

../test/CodeGen/X86/masked_gather_scatter.ll

 ; SKX-NEXT:    vpslld $31, %xmm2, %xmm2
 ; SKX-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; SKX-NEXT:    kmovb %k1, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    vpmovsxdq %xmm1, %ymm1
 ; SKX-NEXT:    vpsllq $2, %ymm1, %ymm1
 ; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm1
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; SKX-NEXT:    # implicit-def: %XMM0
-; SKX-NEXT:    andb $1, %al
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    je .LBB29_2
 ; SKX-NEXT:  # BB#1: # %cond.load
 ; SKX-NEXT:    vmovq %xmm1, %rax
 ; SKX-NEXT:    vmovd (%rax), %xmm0
 ; SKX-NEXT:  .LBB29_2: # %else
 ; SKX-NEXT:    kmovb %k1, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SKX-NEXT:    andb $1, %al
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    je .LBB29_4
 ; SKX-NEXT:  # BB#3: # %cond.load1
 ; SKX-NEXT:    vpextrq $1, %xmm1, %rax
 ; SKX-NEXT:    vpinsrd $1, (%rax), %xmm0, %xmm0
 ; SKX-NEXT:  .LBB29_4: # %else2
 ; SKX-NEXT:    kmovb %k1, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
-; SKX-NEXT:    andb $1, %al
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    je .LBB29_6
 ; SKX-NEXT:  # BB#5: # %cond.load4
 ; SKX-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; SKX-NEXT:    vmovq %xmm1, %rax
 ; SKX-NEXT:    vpinsrd $2, (%rax), %xmm0, %xmm0
 ; SKX-NEXT:  .LBB29_6: # %else5
 ; SKX-NEXT:    vmovdqa32 %xmm0, %xmm3 {%k1}
 ; SKX-NEXT:    vmovaps %zmm3, %zmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test30:
 ; SKX_32:       # BB#0:
 ; SKX_32-NEXT:    subl $12, %esp
 ; SKX_32-NEXT:  .Ltmp0:
 ; SKX_32-NEXT:    .cfi_def_cfa_offset 16
 ; SKX_32-NEXT:    vpslld $31, %xmm2, %xmm2
 ; SKX_32-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; SKX_32-NEXT:    kmovb %k1, {{[0-9]+}}(%esp)
 ; SKX_32-NEXT:    vpslld $2, %xmm1, %xmm1
 ; SKX_32-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; SKX_32-NEXT:    # implicit-def: %XMM1
-; SKX_32-NEXT:    andb $1, %al
+; SKX_32-NEXT:    testb %al, %al
 ; SKX_32-NEXT:    je .LBB29_2
 ; SKX_32-NEXT:  # BB#1: # %cond.load
 ; SKX_32-NEXT:    vmovd %xmm2, %eax
 ; SKX_32-NEXT:    vmovd (%eax), %xmm1
 ; SKX_32-NEXT:  .LBB29_2: # %else
 ; SKX_32-NEXT:    kmovb %k1, {{[0-9]+}}(%esp)
 ; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; SKX_32-NEXT:    andb $1, %al
+; SKX_32-NEXT:    testb %al, %al
 ; SKX_32-NEXT:    je .LBB29_4
 ; SKX_32-NEXT:  # BB#3: # %cond.load1
 ; SKX_32-NEXT:    vpextrd $1, %xmm2, %eax
 ; SKX_32-NEXT:    vpinsrd $1, (%eax), %xmm1, %xmm1
 ; SKX_32-NEXT:  .LBB29_4: # %else2
 ; SKX_32-NEXT:    vmovdqa32 {{[0-9]+}}(%esp), %xmm0
 ; SKX_32-NEXT:    kmovb %k1, (%esp)
 ; SKX_32-NEXT:    movb (%esp), %al
-; SKX_32-NEXT:    andb $1, %al
+; SKX_32-NEXT:    testb %al, %al
 ; SKX_32-NEXT:    je .LBB29_6
 ; SKX_32-NEXT:  # BB#5: # %cond.load4
 ; SKX_32-NEXT:    vpextrd $2, %xmm2, %eax
 ; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm1, %xmm1
 ; SKX_32-NEXT:  .LBB29_6: # %else5
 ; SKX_32-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
 ; SKX_32-NEXT:    addl $12, %esp
 ; SKX_32-NEXT:    retl

../test/CodeGen/X86/x86-shrink-wrapping.ll

 ; is disabled.
 ; DISABLE: pushq
 ; The stack adjustment can use SUB instr because we do not need to
 ; preserve the EFLAGS at this point.
 ; DISABLE-NEXT: subq $16, %rsp
 ;
 ; Load the value of b.
 ; CHECK: movb _b(%rip), [[BOOL:%cl]]
-; Extract i1 from the loaded value.
-; CHECK-NEXT: andb $1, [[BOOL]]
 ; Create the zero value for the select assignment.
 ; CHECK-NEXT: xorl [[CMOVE_VAL:%eax]], [[CMOVE_VAL]]
 ; CHECK-NEXT: testb [[BOOL]], [[BOOL]]
 ; CHECK-NEXT: jne [[STOREC_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: movb $48, [[CMOVE_VAL:%al]]
 ;
 ; CHECK: [[STOREC_LABEL]]: