[X86] Add patterns for using movss/movsd for atomic load/store of f32/64. Remove atomic fadd pseudos use isel patterns instead.

[craig.topper]

Summary

This patch adds patterns for turning bitcasted atomic load/store into movss/sd.

It also removes the pseudo instructions for atomic RMW fadd. Instead just adding isel patterns for folding an atomic load into addss/sd. And relying on the new movss/sd store pattern to handle the write part.

This also makes the fadd patterns use VEX and EVEX instructions when AVX or AVX512F are enabled.

[RKSimon]

llvm/test/CodeGen/X86/atomic-fp.ll

9

Maybe add SSE1 only tests?

197

Can we improve the i686 i64/f64 codegen at all?

[craig.topper]

llvm/test/CodeGen/X86/atomic-fp.ll

197

Yeah we need to add something like a v2i64 VEXTRACT_STORE node and turn the atomic store into that then select that to the MOVQ store. Maybe some more work depending on how the (v2i64 (scalar_to_vector (i64 (bitcast f64)))) that implies gets legalized and cleaned up

This only work if the atomic store isn't seq_cst. seq_cst either requires the locked cmpxchg8b that we have now or we need an mfence after the movq. gcc and icc both use the movq+mfence(or a locked or to stack if mfence isn't available) I think.

[craig.topper]

Add SSE1 test RUN lines.
Remove some FIXMEs and fix a comment X86InstrCompiler.td

[craig.topper]

Rebase after pre-commiting the SSE1 command lines

[RKSimon]

LGTM

[danilaml]

I wonder if instead of matching bitcasts, implementing TODOs (hooks) form here https://reviews.llvm.org/D15471 was considered.

llvm/lib/Target/X86/X86ISelLowering.cpp

X86TargetLowering::EmitLoweredSelect(MachineInstr &MI,
 
   // Now remove the CMOV(s).
   ThisMBB->erase(MIItBegin, MIItEnd);
 
   return SinkMBB;
 }
 
 MachineBasicBlock *
-X86TargetLowering::EmitLoweredAtomicFP(MachineInstr &MI,
-                                       MachineBasicBlock *BB) const {
-  // Combine the following atomic floating-point modification pattern:
-  //   a.store(reg OP a.load(acquire), release)
-  // Transform them into:
-  //   OPss (%gpr), %xmm
-  //   movss %xmm, (%gpr)
-  // Or sd equivalent for 64-bit operations.
-  unsigned MOp, FOp;
-  switch (MI.getOpcode()) {
-  default: llvm_unreachable("unexpected instr type for EmitLoweredAtomicFP");
-  case X86::RELEASE_FADD32mr:
-    FOp = X86::ADDSSrm;
-    MOp = X86::MOVSSmr;
-    break;
-  case X86::RELEASE_FADD64mr:
-    FOp = X86::ADDSDrm;
-    MOp = X86::MOVSDmr;
-    break;
-  }
-  const X86InstrInfo *TII = Subtarget.getInstrInfo();
-  DebugLoc DL = MI.getDebugLoc();
-  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
-  unsigned ValOpIdx = X86::AddrNumOperands;
-  unsigned VSrc = MI.getOperand(ValOpIdx).getReg();
-  MachineInstrBuilder MIB =
-      BuildMI(*BB, MI, DL, TII->get(FOp),
-              MRI.createVirtualRegister(MRI.getRegClass(VSrc)))
-          .addReg(VSrc);
-  for (int i = 0; i < X86::AddrNumOperands; ++i) {
-    MachineOperand &Operand = MI.getOperand(i);
-    // Clear any kill flags on register operands as we'll create a second
-    // instruction using the same address operands.
-    if (Operand.isReg())
-      Operand.setIsKill(false);
-    MIB.add(Operand);
-  }
-  MachineInstr *FOpMI = MIB;
-  MIB = BuildMI(*BB, MI, DL, TII->get(MOp));
-  for (int i = 0; i < X86::AddrNumOperands; ++i)
-    MIB.add(MI.getOperand(i));
-  MIB.addReg(FOpMI->getOperand(0).getReg(), RegState::Kill);
-  MI.eraseFromParent(); // The pseudo instruction is gone now.
-  return BB;
-}
-
-MachineBasicBlock *
 X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
                                         MachineBasicBlock *BB) const {
   MachineFunction *MF = BB->getParent();
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL = MI.getDebugLoc();
   const BasicBlock *LLVM_BB = BB->getBasicBlock();
 
   assert(MF->shouldSplitStack());
unsigned PopF =
         MI.getOpcode() == X86::WRFLAGS32 ? X86::POPF32 : X86::POPF64;
     BuildMI(*BB, MI, DL, TII->get(Push)).addReg(MI.getOperand(0).getReg());
     BuildMI(*BB, MI, DL, TII->get(PopF));
 
     MI.eraseFromParent(); // The pseudo is gone now.
     return BB;
   }
 
-  case X86::RELEASE_FADD32mr:
-  case X86::RELEASE_FADD64mr:
-    return EmitLoweredAtomicFP(MI, BB);
-
   case X86::FP32_TO_INT16_IN_MEM:
   case X86::FP32_TO_INT32_IN_MEM:
   case X86::FP32_TO_INT64_IN_MEM:
   case X86::FP64_TO_INT16_IN_MEM:
   case X86::FP64_TO_INT32_IN_MEM:
   case X86::FP64_TO_INT64_IN_MEM:
   case X86::FP80_TO_INT16_IN_MEM:
   case X86::FP80_TO_INT32_IN_MEM:

llvm/lib/Target/X86/X86InstrCompiler.td

def : Pat<(atomic_store_64 addr:$dst,
             (!cast<Instruction>(Name#"64mr") addr:$dst, GR64:$src)>;
 }
 defm : RELEASE_BINOP_MI<"ADD", add>;
 defm : RELEASE_BINOP_MI<"AND", and>;
 defm : RELEASE_BINOP_MI<"OR",  or>;
 defm : RELEASE_BINOP_MI<"XOR", xor>;
 defm : RELEASE_BINOP_MI<"SUB", sub>;
 
-// Same as above, but for floating-point.
-// FIXME: imm version.
-// FIXME: Version that doesn't clobber $src, using AVX's VADDSS.
+// Atomic load + floating point patterns.
 // FIXME: This could also handle SIMD operations with *ps and *pd instructions.
-let usesCustomInserter = 1, SchedRW = [WriteMicrocoded] in {
-multiclass RELEASE_FP_BINOP_MI<SDNode op> {
-    def NAME#32mr : I<0, Pseudo, (outs), (ins i32mem:$dst, FR32:$src),
-        "#BINOP "#NAME#"32mr PSEUDO!",
-        [(atomic_store_32 addr:$dst,
-           (i32 (bitconvert (op
-             (f32 (bitconvert (i32 (atomic_load_32 addr:$dst)))),
-          FR32:$src))))]>, Requires<[HasSSE1]>;
-    def NAME#64mr : I<0, Pseudo, (outs), (ins i64mem:$dst, FR64:$src),
-        "#BINOP "#NAME#"64mr PSEUDO!",
-        [(atomic_store_64 addr:$dst,
-           (i64 (bitconvert (op
-             (f64 (bitconvert (i64 (atomic_load_64 addr:$dst)))),
-          FR64:$src))))]>, Requires<[HasSSE2]>;
+multiclass ATOMIC_LOAD_FP_BINOP_MI<string Name, SDNode op> {
+  def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))),
+            (!cast<Instruction>(Name#"SSrm") FR32:$src1, addr:$src2)>,
+            Requires<[UseSSE1]>;
+  def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))),
+            (!cast<Instruction>("V"#Name#"SSrm") FR32:$src1, addr:$src2)>,
+            Requires<[UseAVX]>;
+  def : Pat<(op FR32X:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))),
+            (!cast<Instruction>("V"#Name#"SSZrm") FR32X:$src1, addr:$src2)>,
+            Requires<[HasAVX512]>;
+
+  def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))),
+            (!cast<Instruction>(Name#"SDrm") FR64:$src1, addr:$src2)>,
+            Requires<[UseSSE1]>;
+  def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))),
+            (!cast<Instruction>("V"#Name#"SDrm") FR64:$src1, addr:$src2)>,
+            Requires<[UseAVX]>;
+  def : Pat<(op FR64X:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))),
+            (!cast<Instruction>("V"#Name#"SDZrm") FR64X:$src1, addr:$src2)>,
+            Requires<[HasAVX512]>;
 }
-defm RELEASE_FADD : RELEASE_FP_BINOP_MI<fadd>;
+defm : ATOMIC_LOAD_FP_BINOP_MI<"ADD", fadd>;
 // FIXME: Add fsub, fmul, fdiv, ...
-}
 
 multiclass RELEASE_UNOP<string Name, dag dag8, dag dag16, dag dag32,
                         dag dag64> {
   def : Pat<(atomic_store_8 addr:$dst, dag8),
             (!cast<Instruction>(Name#8m) addr:$dst)>;
   def : Pat<(atomic_store_16 addr:$dst, dag16),
             (!cast<Instruction>(Name#16m) addr:$dst)>;
   def : Pat<(atomic_store_32 addr:$dst, dag32),

 def : Pat<(atomic_store_64 addr:$dst, GR64:$src),
           (MOV64mr addr:$dst, GR64:$src)>;
 
 def : Pat<(i8  (atomic_load_8 addr:$src)),  (MOV8rm addr:$src)>;
 def : Pat<(i16 (atomic_load_16 addr:$src)), (MOV16rm addr:$src)>;
 def : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>;
 def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>;
 
+// Floating point loads/stores.
+def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+          (MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>;
+def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+          (VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>;
+def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+          (VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>;
+
+def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+          (MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>;
+def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+          (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>;
+def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+          (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>;
+
+def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))),
+          (MOVSSrm addr:$src)>, Requires<[UseSSE1]>;
+def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))),
+          (VMOVSSrm addr:$src)>, Requires<[UseAVX]>;
+def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))),
+          (VMOVSSZrm addr:$src)>, Requires<[HasAVX512]>;
+
+def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))),
+          (MOVSDrm addr:$src)>, Requires<[UseSSE2]>;
+def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))),
+          (VMOVSDrm addr:$src)>, Requires<[UseAVX]>;
+def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))),
+          (VMOVSDZrm addr:$src)>, Requires<[HasAVX512]>;
+
 //===----------------------------------------------------------------------===//
 // DAG Pattern Matching Rules
 //===----------------------------------------------------------------------===//
 
 // Use AND/OR to store 0/-1 in memory when optimizing for minsize. This saves
 // binary size compared to a regular MOV, but it introduces an unnecessary
 // load, so is not suitable for regular or optsize functions.
 let Predicates = [OptForMinSize] in {

llvm/test/CodeGen/X86/atomic-fp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix X86 --check-prefix X86-NOSSE
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=sse -verify-machineinstrs | FileCheck %s --check-prefix X86 --check-prefix X86-SSE --check-prefix X86-SSE1
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=sse2 -verify-machineinstrs | FileCheck %s --check-prefix X86 --check-prefix X86-SSE --check-prefix X86-SSE2
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx -verify-machineinstrs | FileCheck %s --check-prefix X86 --check-prefix X86-AVX --check-prefix X86-AVX1
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512f -verify-machineinstrs | FileCheck %s --check-prefix X86 --check-prefix X86-AVX --check-prefix X86-AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix X64 --check-prefix X64-SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -verify-machineinstrs | FileCheck %s --check-prefix X64 --check-prefix X64-AVX --check-prefix X64-AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f -verify-machineinstrs | FileCheck %s --check-prefix X64 --check-prefix X64-AVX --check-prefix X64-AVX512
 
 ; ----- FADD -----
 
 define void @fadd_32r(float* %loc, float %val) nounwind {
 ; X86-NOSSE-LABEL: fadd_32r:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    subl $8, %esp
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax

 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE1-NEXT:    movl %ecx, (%eax)
 ; X86-SSE1-NEXT:    addl $8, %esp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_32r:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    addss (%eax), %xmm0
 ; X86-SSE2-NEXT:    movss %xmm0, (%eax)
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_32r:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-AVX-NEXT:    addss (%eax), %xmm0
-; X86-AVX-NEXT:    movss %xmm0, (%eax)
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vaddss (%eax), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%eax)
 ; X86-AVX-NEXT:    retl
 ;
-; X64-LABEL: fadd_32r:
-; X64:       # %bb.0:
-; X64-NEXT:    addss (%rdi), %xmm0
-; X64-NEXT:    movss %xmm0, (%rdi)
-; X64-NEXT:    retq
+; X64-SSE-LABEL: fadd_32r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    addss (%rdi), %xmm0
+; X64-SSE-NEXT:    movss %xmm0, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fadd_32r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vaddss (%rdi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
   %floc = bitcast float* %loc to i32*
   %1 = load atomic i32, i32* %floc seq_cst, align 4
   %2 = bitcast i32 %1 to float
   %add = fadd float %2, %val
   %3 = bitcast float %add to i32
   store atomic i32 %3, i32* %floc release, align 4
   ret void
 }

 ; X86-AVX-NEXT:    movl (%esp), %ebx
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl (%esi), %eax
 ; X86-AVX-NEXT:    movl 4(%esi), %edx
 ; X86-AVX-NEXT:    .p2align 4, 0x90
 ; X86-AVX-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-AVX-NEXT:    lock cmpxchg8b (%esi)
 ; X86-AVX-NEXT:    jne .LBB1_1
 ; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
 ; X86-AVX-NEXT:    leal -8(%ebp), %esp
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    popl %ebx
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
-; X64-LABEL: fadd_64r:
-; X64:       # %bb.0:
-; X64-NEXT:    addsd (%rdi), %xmm0
-; X64-NEXT:    movsd %xmm0, (%rdi)
-; X64-NEXT:    retq
+; X64-SSE-LABEL: fadd_64r:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    addsd (%rdi), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rdi)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fadd_64r:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vaddsd (%rdi), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
   %floc = bitcast double* %loc to i64*
   %1 = load atomic i64, i64* %floc seq_cst, align 8
   %2 = bitcast i64 %1 to double
   %add = fadd double %2, %val
   %3 = bitcast double %add to i64
   store atomic i64 %3, i64* %floc release, align 8
   ret void
 }

 ; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    addss glob32, %xmm0
 ; X86-SSE2-NEXT:    movss %xmm0, glob32
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_32g:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-AVX-NEXT:    addss glob32, %xmm0
-; X86-AVX-NEXT:    movss %xmm0, glob32
+; X86-AVX-NEXT:    vaddss glob32, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, glob32
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_32g:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    addss {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movss %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_32g:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-AVX-NEXT:    addss {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    movss %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    vaddss {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    retq
   %i = load atomic i32, i32* bitcast (float* @glob32 to i32*) monotonic, align 4
   %f = bitcast i32 %i to float
   %add = fadd float %f, 1.000000e+00
   %s = bitcast float %add to i32
   store atomic i32 %s, i32* bitcast (float* @glob32 to i32*) monotonic, align 4
   ret void
 }

 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    addsd {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movsd %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_64g:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-AVX-NEXT:    addsd {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    movsd %xmm0, {{.*}}(%rip)
+; X64-AVX-NEXT:    vaddsd {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    retq
   %i = load atomic i64, i64* bitcast (double* @glob64 to i64*) monotonic, align 8
   %f = bitcast i64 %i to double
   %add = fadd double %f, 1.000000e+00
   %s = bitcast double %add to i64
   store atomic i64 %s, i64* bitcast (double* @glob64 to i64*) monotonic, align 8
   ret void
 }

 ; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    addss -559038737, %xmm0
 ; X86-SSE2-NEXT:    movss %xmm0, -559038737
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_32imm:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-AVX-NEXT:    addss -559038737, %xmm0
-; X86-AVX-NEXT:    movss %xmm0, -559038737
+; X86-AVX-NEXT:    vaddss -559038737, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, -559038737
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_32imm:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    addss (%rax), %xmm0
 ; X64-SSE-NEXT:    movss %xmm0, (%rax)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_32imm:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
-; X64-AVX-NEXT:    addss (%rax), %xmm0
-; X64-AVX-NEXT:    movss %xmm0, (%rax)
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vaddss (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, (%rax)
 ; X64-AVX-NEXT:    retq
   %i = load atomic i32, i32* inttoptr (i32 3735928559 to i32*) monotonic, align 4
   %f = bitcast i32 %i to float
   %add = fadd float %f, 1.000000e+00
   %s = bitcast float %add to i32
   store atomic i32 %s, i32* inttoptr (i32 3735928559 to i32*) monotonic, align 4
   ret void
 }

 ; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
 ; X86-AVX-NEXT:    leal -4(%ebp), %esp
 ; X86-AVX-NEXT:    popl %ebx
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_64imm:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    addsd (%rax), %xmm0
 ; X64-SSE-NEXT:    movsd %xmm0, (%rax)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_64imm:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-AVX-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
-; X64-AVX-NEXT:    addsd (%rax), %xmm0
-; X64-AVX-NEXT:    movsd %xmm0, (%rax)
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-AVX-NEXT:    vaddsd (%rax), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rax)
 ; X64-AVX-NEXT:    retq
   %i = load atomic i64, i64* inttoptr (i64 3735928559 to i64*) monotonic, align 8
   %f = bitcast i64 %i to double
   %add = fadd double %f, 1.000000e+00
   %s = bitcast double %add to i64
   store atomic i64 %s, i64* inttoptr (i64 3735928559 to i64*) monotonic, align 8
   ret void
 }

 ; X86-SSE2-NEXT:    movss %xmm0, (%esp)
 ; X86-SSE2-NEXT:    popl %eax
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_32stack:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %eax
 ; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-AVX-NEXT:    addss (%esp), %xmm0
-; X86-AVX-NEXT:    movss %xmm0, (%esp)
+; X86-AVX-NEXT:    vaddss (%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
 ; X86-AVX-NEXT:    popl %eax
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_32stack:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    addss -{{[0-9]+}}(%rsp), %xmm0
 ; X64-SSE-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_32stack:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-AVX-NEXT:    addss -{{[0-9]+}}(%rsp), %xmm0
-; X64-AVX-NEXT:    movss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    vaddss -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-AVX-NEXT:    retq
   %ptr = alloca i32, align 4
   %bc3 = bitcast i32* %ptr to float*
   %load = load atomic i32, i32* %ptr acquire, align 4
   %bc0 = bitcast i32 %load to float
   %fadd = fadd float 1.000000e+00, %bc0
   %bc1 = bitcast float %fadd to i32
   store atomic i32 %bc1, i32* %ptr release, align 4

 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    addsd -{{[0-9]+}}(%rsp), %xmm0
 ; X64-SSE-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: fadd_64stack:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-AVX-NEXT:    addsd -{{[0-9]+}}(%rsp), %xmm0
-; X64-AVX-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    vaddsd -{{[0-9]+}}(%rsp), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-AVX-NEXT:    retq
   %ptr = alloca i64, align 8
   %bc3 = bitcast i64* %ptr to double*
   %load = load atomic i64, i64* %ptr acquire, align 8
   %bc0 = bitcast i64 %load to double
   %fadd = fadd double 1.000000e+00, %bc0
   %bc1 = bitcast double %fadd to i64
   store atomic i64 %bc1, i64* %ptr release, align 8

 ; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
 ; X86-AVX-NEXT:    leal -12(%ebp), %esp
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    popl %edi
 ; X86-AVX-NEXT:    popl %ebx
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
-; X64-LABEL: fadd_array:
-; X64:       # %bb.0: # %bb
-; X64-NEXT:    addsd (%rdi,%rsi,8), %xmm0
-; X64-NEXT:    movsd %xmm0, (%rdi,%rsi,8)
-; X64-NEXT:    retq
+; X64-SSE-LABEL: fadd_array:
+; X64-SSE:       # %bb.0: # %bb
+; X64-SSE-NEXT:    addsd (%rdi,%rsi,8), %xmm0
+; X64-SSE-NEXT:    movsd %xmm0, (%rdi,%rsi,8)
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: fadd_array:
+; X64-AVX:       # %bb.0: # %bb
+; X64-AVX-NEXT:    vaddsd (%rdi,%rsi,8), %xmm0, %xmm0
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi,%rsi,8)
+; X64-AVX-NEXT:    retq
 bb:
   %tmp4 = getelementptr inbounds i64, i64* %arg, i64 %arg2
   %tmp6 = load atomic i64, i64* %tmp4 monotonic, align 8
   %tmp7 = bitcast i64 %tmp6 to double
   %tmp8 = fadd double %tmp7, %arg1
   %tmp9 = bitcast double %tmp8 to i64
   store atomic i64 %tmp9, i64* %tmp4 monotonic, align 8
   ret void
 }

llvm/test/CodeGen/X86/atomic-non-integer.ll

 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %ecx, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-SSE-LABEL: store_float:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movd %xmm0, %eax
-; X64-SSE-NEXT:    movl %eax, (%rdi)
+; X64-SSE-NEXT:    movss %xmm0, (%rdi)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: store_float:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovd %xmm0, %eax
-; X64-AVX-NEXT:    movl %eax, (%rdi)
+; X64-AVX-NEXT:    vmovss %xmm0, (%rdi)
 ; X64-AVX-NEXT:    retq
   store atomic float %v, float* %fptr unordered, align 4
   ret void
 }
 
 define void @store_double(double* %fptr, double %v) {
 ; X86-LABEL: store_double:
 ; X86:       # %bb.0:

 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
 ;
 ; X64-SSE-LABEL: store_double:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movq %xmm0, %rax
-; X64-SSE-NEXT:    movq %rax, (%rdi)
+; X64-SSE-NEXT:    movsd %xmm0, (%rdi)
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: store_double:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovq %xmm0, %rax
-; X64-AVX-NEXT:    movq %rax, (%rdi)
+; X64-AVX-NEXT:    vmovsd %xmm0, (%rdi)
 ; X64-AVX-NEXT:    retq
   store atomic double %v, double* %fptr unordered, align 8
   ret void
 }
 
 define void @store_fp128(fp128* %fptr, fp128 %v) {
 ; X86-SSE-LABEL: store_fp128:
 ; X86-SSE:       # %bb.0:

 ; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: load_float:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %eax
 ; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT:    movd (%eax), %xmm0
-; X86-SSE2-NEXT:    movd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    movss %xmm0, (%esp)
 ; X86-SSE2-NEXT:    flds (%esp)
 ; X86-SSE2-NEXT:    popl %eax
 ; X86-SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: load_float:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %eax
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-AVX-NEXT:    vmovd (%eax), %xmm0
-; X86-AVX-NEXT:    vmovd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
 ; X86-AVX-NEXT:    flds (%esp)
 ; X86-AVX-NEXT:    popl %eax
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
 ; X86-AVX-NEXT:    retl
 ;
 ; X86-NOSSE-LABEL: load_float:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    pushl %eax
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    movl (%eax), %eax
 ; X86-NOSSE-NEXT:    movl %eax, (%esp)
 ; X86-NOSSE-NEXT:    flds (%esp)
 ; X86-NOSSE-NEXT:    popl %eax
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: load_float:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movd (%rdi), %xmm0
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: load_float:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovd (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    retq
   %v = load atomic float, float* %fptr unordered, align 4
   ret float %v
 }
 
 define double @load_double(double* %fptr) {
 ; X86-SSE1-LABEL: load_double:
 ; X86-SSE1:       # %bb.0:

 ; X86-NOSSE-NEXT:    popl %esi
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    popl %ebx
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: load_double:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movq (%rdi), %xmm0
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: load_double:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovq (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-AVX-NEXT:    retq
   %v = load atomic double, double* %fptr unordered, align 8
   ret double %v
 }
 
 define fp128 @load_fp128(fp128* %fptr) {
 ; X86-SSE-LABEL: load_fp128:
 ; X86-SSE:       # %bb.0:

 ; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: load_float_seq_cst:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %eax
 ; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT:    movl (%eax), %eax
-; X86-SSE2-NEXT:    movd %eax, %xmm0
-; X86-SSE2-NEXT:    movd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    movss %xmm0, (%esp)
 ; X86-SSE2-NEXT:    flds (%esp)
 ; X86-SSE2-NEXT:    popl %eax
 ; X86-SSE2-NEXT:    .cfi_def_cfa_offset 4
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: load_float_seq_cst:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %eax
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-AVX-NEXT:    movl (%eax), %eax
-; X86-AVX-NEXT:    vmovd %eax, %xmm0
-; X86-AVX-NEXT:    vmovd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vmovss %xmm0, (%esp)
 ; X86-AVX-NEXT:    flds (%esp)
 ; X86-AVX-NEXT:    popl %eax
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
 ; X86-AVX-NEXT:    retl
 ;
 ; X86-NOSSE-LABEL: load_float_seq_cst:
 ; X86-NOSSE:       # %bb.0:
 ; X86-NOSSE-NEXT:    pushl %eax
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    movl (%eax), %eax
 ; X86-NOSSE-NEXT:    movl %eax, (%esp)
 ; X86-NOSSE-NEXT:    flds (%esp)
 ; X86-NOSSE-NEXT:    popl %eax
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: load_float_seq_cst:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movl (%rdi), %eax
-; X64-SSE-NEXT:    movd %eax, %xmm0
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: load_float_seq_cst:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    movl (%rdi), %eax
-; X64-AVX-NEXT:    vmovd %eax, %xmm0
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    retq
   %v = load atomic float, float* %fptr seq_cst, align 4
   ret float %v
 }
 
 define double @load_double_seq_cst(double* %fptr) {
 ; X86-SSE1-LABEL: load_double_seq_cst:
 ; X86-SSE1:       # %bb.0:

 ; X86-NOSSE-NEXT:    popl %esi
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    popl %ebx
 ; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: load_double_seq_cst:
 ; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movq (%rdi), %rax
-; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX-LABEL: load_double_seq_cst:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    movq (%rdi), %rax
-; X64-AVX-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-AVX-NEXT:    retq
   %v = load atomic double, double* %fptr seq_cst, align 8
   ret double %v
 }