[X86] Avoid generating inc/dec when slow for x.atomic_store(1 + x.atomic_load())

[morisset]

Summary

I had forgotten to check for NotSlowIncDec in the patterns that can generate
inc/dec for the above pattern (added in D4796).

[jfb]

lgtm

Please edit the message to point out that right now this affects Atom Silvermont, KNL, and SKX.

[morisset]

Closed by commit rL219336 (authored by @morisset).

lib/Target/X86/X86InstrCompiler.td

def NAME#64m : I<0, Pseudo, (outs), (ins i64mem:$dst),
         "#RELEASE_UNOP PSEUDO!",
         [(atomic_store_64 addr:$dst, dag64)]>;
 }
 
 defm RELEASE_INC : RELEASE_UNOP<
     (add (atomic_load_8  addr:$dst), (i8 1)),
     (add (atomic_load_16 addr:$dst), (i16 1)),
     (add (atomic_load_32 addr:$dst), (i32 1)),
-    (add (atomic_load_64 addr:$dst), (i64 1))>;
+    (add (atomic_load_64 addr:$dst), (i64 1))>, Requires<[NotSlowIncDec]>;
 defm RELEASE_DEC : RELEASE_UNOP<
     (add (atomic_load_8  addr:$dst), (i8 -1)),
     (add (atomic_load_16 addr:$dst), (i16 -1)),
     (add (atomic_load_32 addr:$dst), (i32 -1)),
-    (add (atomic_load_64 addr:$dst), (i64 -1))>;
+    (add (atomic_load_64 addr:$dst), (i64 -1))>, Requires<[NotSlowIncDec]>;
 /*
 TODO: These don't work because the type inference of TableGen fails.
 TODO: find a way to fix it.
 defm RELEASE_NEG : RELEASE_UNOP<
     (ineg (atomic_load_8  addr:$dst)),
     (ineg (atomic_load_16 addr:$dst)),
     (ineg (atomic_load_32 addr:$dst)),
     (ineg (atomic_load_64 addr:$dst))>;

test/CodeGen/X86/atomic_mi.ll

 ; RUN: llc < %s -march=x86-64 -verify-machineinstrs | FileCheck %s --check-prefix X64
 ; RUN: llc < %s -march=x86 -verify-machineinstrs | FileCheck %s --check-prefix X32
+; RUN: llc < %s -march=x86-64 -mattr=slow-incdec -verify-machineinstrs | FileCheck %s --check-prefix SLOW_INC
 
 ; This file checks that atomic (non-seq_cst) stores of immediate values are
 ; done in one mov instruction and not 2. More precisely, it makes sure that the
 ; immediate is not first copied uselessly into a register.
 
 ; Similarily, it checks that a binary operation of an immediate with an atomic
 ; variable that is stored back in that variable is done as a single instruction.
 ; For example: x.store(42 + x.load(memory_order_acquire), memory_order_release)

 ; X64-LABEL: inc_8
 ; X64-NOT: lock
 ; X64: incb
 ; X64-NOT: movb
 ; X32-LABEL: inc_8
 ; X32-NOT: lock
 ; X32: incb
 ; X32-NOT: movb
+; SLOW_INC-LABEL: inc_8
+; SLOW_INC-NOT: incb
+; SLOW_INC-NOT: movb
   %1 = load atomic i8* %p seq_cst, align 1
   %2 = add i8 %1, 1
   store atomic i8 %2, i8* %p release, align 1
   ret void
 }
 
 define void @inc_16(i16* %p) {
 ;   Currently the transformation is not done on 16 bit accesses, as the backend
 ;   treat 16 bit arithmetic as expensive on X86/X86_64.
 ; X64-LABEL: inc_16
 ; X64-NOT: incw
 ; X32-LABEL: inc_16
 ; X32-NOT: incw
+; SLOW_INC-LABEL: inc_16
+; SLOW_INC-NOT: incw
   %1 = load atomic i16* %p acquire, align 2
   %2 = add i16 %1, 1
   store atomic i16 %2, i16* %p release, align 2
   ret void
 }
 
 define void @inc_32(i32* %p) {
 ; X64-LABEL: inc_32
 ; X64-NOT: lock
 ; X64: incl
 ; X64-NOT: movl
 ; X32-LABEL: inc_32
 ; X32-NOT: lock
 ; X32: incl
 ; X32-NOT: movl
+; SLOW_INC-LABEL: inc_32
+; SLOW_INC-NOT: incl
+; SLOW_INC-NOT: movl
   %1 = load atomic i32* %p acquire, align 4
   %2 = add i32 %1, 1
   store atomic i32 %2, i32* %p monotonic, align 4
   ret void
 }
 
 define void @inc_64(i64* %p) {
 ; X64-LABEL: inc_64
 ; X64-NOT: lock
 ; X64: incq
 ; X64-NOT: movq
 ;   We do not check X86-32 as it cannot do 'incq'.
 ; X32-LABEL: inc_64
+; SLOW_INC-LABEL: inc_64
+; SLOW_INC-NOT: incq
+; SLOW_INC-NOT: movq
   %1 = load atomic i64* %p acquire, align 8
   %2 = add i64 %1, 1
   store atomic i64 %2, i64* %p release, align 8
   ret void
 }
 
 define void @inc_32_seq_cst(i32* %p) {
 ; X64-LABEL: inc_32_seq_cst

 ; X64-LABEL: dec_8
 ; X64-NOT: lock
 ; X64: decb
 ; X64-NOT: movb
 ; X32-LABEL: dec_8
 ; X32-NOT: lock
 ; X32: decb
 ; X32-NOT: movb
+; SLOW_INC-LABEL: dec_8
+; SLOW_INC-NOT: decb
+; SLOW_INC-NOT: movb
   %1 = load atomic i8* %p seq_cst, align 1
   %2 = sub i8 %1, 1
   store atomic i8 %2, i8* %p release, align 1
   ret void
 }
 
 define void @dec_16(i16* %p) {
 ;   Currently the transformation is not done on 16 bit accesses, as the backend
 ;   treat 16 bit arithmetic as expensive on X86/X86_64.
 ; X64-LABEL: dec_16
 ; X64-NOT: decw
 ; X32-LABEL: dec_16
 ; X32-NOT: decw
+; SLOW_INC-LABEL: dec_16
+; SLOW_INC-NOT: decw
   %1 = load atomic i16* %p acquire, align 2
   %2 = sub i16 %1, 1
   store atomic i16 %2, i16* %p release, align 2
   ret void
 }
 
 define void @dec_32(i32* %p) {
 ; X64-LABEL: dec_32
 ; X64-NOT: lock
 ; X64: decl
 ; X64-NOT: movl
 ; X32-LABEL: dec_32
 ; X32-NOT: lock
 ; X32: decl
 ; X32-NOT: movl
+; SLOW_INC-LABEL: dec_32
+; SLOW_INC-NOT: decl
+; SLOW_INC-NOT: movl
   %1 = load atomic i32* %p acquire, align 4
   %2 = sub i32 %1, 1
   store atomic i32 %2, i32* %p monotonic, align 4
   ret void
 }
 
 define void @dec_64(i64* %p) {
 ; X64-LABEL: dec_64
 ; X64-NOT: lock
 ; X64: decq
 ; X64-NOT: movq
 ;   We do not check X86-32 as it cannot do 'decq'.
 ; X32-LABEL: dec_64
+; SLOW_INC-LABEL: dec_64
+; SLOW_INC-NOT: decq
+; SLOW_INC-NOT: movq
   %1 = load atomic i64* %p acquire, align 8
   %2 = sub i64 %1, 1
   store atomic i64 %2, i64* %p release, align 8
   ret void
 }
 
 define void @dec_32_seq_cst(i32* %p) {
 ; X64-LABEL: dec_32_seq_cst
 ; X64: xchgl
 ; X32-LABEL: dec_32_seq_cst
 ; X32: xchgl
   %1 = load atomic i32* %p monotonic, align 4
   %2 = sub i32 %1, 1
   store atomic i32 %2, i32* %p seq_cst, align 4
   ret void
 }