[X86] Use MOVQ for i64 atomic_stores when SSE2 is enabled

[craig.topper]

Summary

If we have SSE2 we can use a MOVQ to store 64-bits and avoid falling back to a cmpxchg8b loop. If its a seq_cst store we need to insert an mfence after the store.

[craig.topper]

llvm/test/CodeGen/X86/atomic-fp.ll

152

There is an extra stack temporary here due to type legalization of the bitcast from f64 to i64 being legalized as a stack store f64 and then two i32 loads. DAG combine was able to merge the loads probably using merge elts from consecutive loads to create a VZEXT_LOAD.

llvm/test/CodeGen/X86/atomic6432.ll

841

Not sure why we didn't merge consecutive loads here.

[craig.topper]

Update a another test I missed previously

[craig.topper]

Support seq_cst store by inserting an mfence after the store.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

26230

cast<AtomicSDNode>(Op) should work ?

llvm/test/CodeGen/X86/atomic6432.ll

841

https://bugs.llvm.org/show_bug.cgi?id=39473

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.h

594

If we used MOVSD (f64 store) could we avoid needing this NodeType?

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.h

594

if we don't use a dedicated X86ISD opcode, then we have to use ISD::ATOMIC_STORE. TargetSelectinoDAG has this type constraint which says that the store value type is integer. Perhaps we can relax that, but I don't know if there is some code assuming this.

def SDTAtomicStore : SDTypeProfile<0, 2, [
  SDTCisPtrTy<0>, SDTCisInt<1>
]>;
def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
                    [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;

[craig.topper]

Ping

[craig.topper]

Ping

[RKSimon]

LGTM

llvm/lib/Target/X86/X86ISelLowering.h

594

OK - please can you raise a bug about whether SDTCisInt<1> can be relaxed?

llvm/lib/Target/X86/X86ISelLowering.h

enum NodeType : unsigned {
 
       /// LOCK-prefixed arithmetic read-modify-write instructions.
       /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
       LADD, LSUB, LOR, LXOR, LAND,
 
       // Load, scalar_to_vector, and zero extend.
       VZEXT_LOAD,
 
+      // extract_vector_elt, store.
+      VEXTRACT_STORE,
+
       // Store FP control world into i16 memory.
       FNSTCW16m,
 
       /// This instruction implements FP_TO_SINT with the
       /// integer destination in memory and a FP reg source.  This corresponds
       /// to the X86::FIST*m instructions and the rounding mode change stuff. It
       /// has two inputs (token chain and address) and two outputs (int value
       /// and token chain). Memory VT specifies the type to store to.

llvm/lib/Target/X86/X86ISelLowering.cpp

bool X86TargetLowering::needsCmpXchgNb(Type *MemType) const {
   if (OpWidth == 64)
     return Subtarget.hasCmpxchg8b() && !Subtarget.is64Bit();
   if (OpWidth == 128)
     return Subtarget.hasCmpxchg16b();
 
   return false;
 }
 
+// TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
+// TODO: In 32-bit mode, use FISTP when X87 is available?
 bool X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
-  return needsCmpXchgNb(SI->getValueOperand()->getType());
+  Type *MemType = SI->getValueOperand()->getType();
+
+  bool NoImplicitFloatOps =
+      SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
+  if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
+      !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2() &&
+      SI->getOrdering() != AtomicOrdering::SequentiallyConsistent)
+    return false;
+
+  return needsCmpXchgNb(MemType);
 }
 
 // Note: this turns large loads into lock cmpxchg8b/16b.
 // TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
 // TODO: In 32-bit mode, use FILD/FISTP when X87 is available?
 TargetLowering::AtomicExpansionKind
 X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
   Type *MemType = LI->getType();
static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
   SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
   // RAUW the chain, but don't worry about the result, as it's unused.
   assert(!N->hasAnyUseOfValue(0));
   // NOTE: The getUNDEF is needed to give something for the unused result 0.
   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
                      DAG.getUNDEF(VT), LockOp.getValue(1));
 }
 
-static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
-  SDNode *Node = Op.getNode();
+static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
+                                 const X86Subtarget &Subtarget) {
+  auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDLoc dl(Node);
-  EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
+  EVT VT = Node->getMemoryVT();
+
+  bool IsSeqCst = Node->getOrdering() == AtomicOrdering::SequentiallyConsistent;
+
+  // If this store is not sequentially consistent and the type is legal
+  // we can just keep it.
+  if (!IsSeqCst && DAG.getTargetLoweringInfo().isTypeLegal(VT))
+    return Op;
+
+  if (!IsSeqCst && VT == MVT::i64) {
+    // For illegal i64 atomic_stores, we can try to use MOVQ if SSE2 is enabled.
+    // FIXME: Use movlps with SSE1.
+    // FIXME: Use fist with X87.
+    bool NoImplicitFloatOps =
+        DAG.getMachineFunction().getFunction().hasFnAttribute(
+            Attribute::NoImplicitFloat);
+    if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
+        Subtarget.hasSSE2()) {
+      SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
+                                     Node->getOperand(2));
+      SDVTList Tys = DAG.getVTList(MVT::Other);
+      SDValue Ops[] = { Node->getChain(), SclToVec, Node->getBasePtr() };
+      SDValue St = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
+                                           MVT::i64, Node->getMemOperand());
+      return St;
+    }
+  }
 
   // Convert seq_cst store -> xchg
   // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
-  // FIXME: On 32-bit, store -> fist or movq would be more efficient
-  //        (The only way to get a 16-byte store is cmpxchg16b)
   // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
-  if (cast<AtomicSDNode>(Node)->getOrdering() ==
-          AtomicOrdering::SequentiallyConsistent ||
-      !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
-    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
-                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
-                                 Node->getOperand(0),
-                                 Node->getOperand(1), Node->getOperand(2),
-                                 cast<AtomicSDNode>(Node)->getMemOperand());
-    return Swap.getValue(1);
-  }
-  // Other atomic stores have a simple pattern.
-  return Op;
-}
+                               Node->getMemoryVT(),
+                               Node->getMemOperand());
 
 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
   SDNode *N = Op.getNode();
   MVT VT = N->getSimpleValueType(0);
 
   // Let legalize expand this if it isn't a legal type yet.
   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     return LowerCMP_SWAP(Op, Subtarget, DAG);
   case ISD::CTPOP:              return LowerCTPOP(Op, Subtarget, DAG);
   case ISD::ATOMIC_LOAD_ADD:
   case ISD::ATOMIC_LOAD_SUB:
   case ISD::ATOMIC_LOAD_OR:
   case ISD::ATOMIC_LOAD_XOR:
   case ISD::ATOMIC_LOAD_AND:    return lowerAtomicArith(Op, DAG, Subtarget);
-  case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op, DAG);
+  case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op, DAG, Subtarget);
   case ISD::BITREVERSE:         return LowerBITREVERSE(Op, Subtarget, DAG);
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
   case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
   case ISD::VECTOR_SHUFFLE:     return lowerVectorShuffle(Op, Subtarget, DAG);
   case ISD::VSELECT:            return LowerVSELECT(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
case X86ISD::LCMPXCHG16_SAVE_RBX_DAG:
     return "X86ISD::LCMPXCHG16_SAVE_RBX_DAG";
   case X86ISD::LADD:               return "X86ISD::LADD";
   case X86ISD::LSUB:               return "X86ISD::LSUB";
   case X86ISD::LOR:                return "X86ISD::LOR";
   case X86ISD::LXOR:               return "X86ISD::LXOR";
   case X86ISD::LAND:               return "X86ISD::LAND";
   case X86ISD::VZEXT_MOVL:         return "X86ISD::VZEXT_MOVL";
   case X86ISD::VZEXT_LOAD:         return "X86ISD::VZEXT_LOAD";
+  case X86ISD::VEXTRACT_STORE:     return "X86ISD::VEXTRACT_STORE";
   case X86ISD::VTRUNC:             return "X86ISD::VTRUNC";
   case X86ISD::VTRUNCS:            return "X86ISD::VTRUNCS";
   case X86ISD::VTRUNCUS:           return "X86ISD::VTRUNCUS";
   case X86ISD::VMTRUNC:            return "X86ISD::VMTRUNC";
   case X86ISD::VMTRUNCS:           return "X86ISD::VMTRUNCS";
   case X86ISD::VMTRUNCUS:          return "X86ISD::VMTRUNCUS";
   case X86ISD::VTRUNCSTORES:       return "X86ISD::VTRUNCSTORES";
   case X86ISD::VTRUNCSTOREUS:      return "X86ISD::VTRUNCSTOREUS";

llvm/lib/Target/X86/X86InstrAVX512.td

def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst),
                              (ins VR128X:$src),
                              "vmovq\t{$src, $dst|$dst, $src}", []>,
                              EVEX, VEX_W, Sched<[SchedWriteVecLogic.XMM]>;
 } // ExeDomain = SSEPackedInt
 
 def : InstAlias<"vmovq.s\t{$src, $dst|$dst, $src}",
                 (VMOVPQI2QIZrr VR128X:$dst, VR128X:$src), 0>;
 
+let Predicates = [HasAVX512] in {
+  def : Pat<(X86vextractstore (v2i64 VR128X:$src), addr:$dst),
+            (VMOVPQI2QIZmr addr:$dst, VR128X:$src)>;
+}
+
 // Move Scalar Single to Double Int
 //
 let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {
 def VMOVSS2DIZrr  : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),
                       (ins FR32X:$src),
                       "vmovd\t{$src, $dst|$dst, $src}",
                       [(set GR32:$dst, (bitconvert FR32X:$src))]>,
                       EVEX, Sched<[WriteVecMoveToGpr]>;

llvm/lib/Target/X86/X86InstrFragmentsSIMD.td

 def X86insertps : SDNode<"X86ISD::INSERTPS",
                  SDTypeProfile<1, 3, [SDTCisVT<0, v4f32>, SDTCisSameAs<0,1>,
                                       SDTCisVT<2, v4f32>, SDTCisVT<3, i8>]>>;
 def X86vzmovl  : SDNode<"X86ISD::VZEXT_MOVL",
                  SDTypeProfile<1, 1, [SDTCisSameAs<0,1>]>>;
 
 def X86vzload  : SDNode<"X86ISD::VZEXT_LOAD", SDTLoad,
                         [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def X86vextractstore  : SDNode<"X86ISD::VEXTRACT_STORE", SDTStore,
+                        [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
 def SDTVtrunc    : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
                                         SDTCisInt<0>, SDTCisInt<1>,
                                         SDTCisOpSmallerThanOp<0, 1>]>;
 def SDTVmtrunc   : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>,
                                         SDTCisInt<0>, SDTCisInt<1>,
                                         SDTCisOpSmallerThanOp<0, 1>,
                                         SDTCisSameAs<0, 2>,

llvm/lib/Target/X86/X86InstrSSE.td

def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
             (VMOVQI2PQIrm addr:$src)>;
   def : Pat<(v2i64 (X86vzload addr:$src)),
             (VMOVQI2PQIrm addr:$src)>;
   def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
               (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
             (SUBREG_TO_REG (i64 0), (v2i64 (VMOVQI2PQIrm addr:$src)), sub_xmm)>;
   def : Pat<(v4i64 (X86vzload addr:$src)),
             (SUBREG_TO_REG (i64 0), (v2i64 (VMOVQI2PQIrm addr:$src)), sub_xmm)>;
+
+  def : Pat<(X86vextractstore (v2i64 VR128:$src), addr:$dst),
+            (VMOVPQI2QImr addr:$dst, VR128:$src)>;
 }
 
 let Predicates = [UseSSE2] in {
   def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
             (MOVQI2PQIrm addr:$src)>;
   def : Pat<(v2i64 (X86vzload addr:$src)), (MOVQI2PQIrm addr:$src)>;
+
+  def : Pat<(X86vextractstore (v2i64 VR128:$src), addr:$dst),
+            (MOVPQI2QImr addr:$dst, VR128:$src)>;
 }
 
 //===---------------------------------------------------------------------===//
 // Moving from XMM to XMM and clear upper 64 bits. Note, there is a bug in
 // IA32 document. movq xmm1, xmm2 does clear the high bits.
 //
 let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecLogic.XMM] in {
 def VMOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),

llvm/test/CodeGen/X86/atomic-fp.ll

 ; X86-SSE1-NEXT:    popl %ebx
 ; X86-SSE1-NEXT:    popl %ebp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_64r:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %ebp
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %ebx
-; X86-SSE2-NEXT:    pushl %esi
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $8, %esp
-; X86-SSE2-NEXT:    movl 8(%ebp), %esi
+; X86-SSE2-NEXT:    movl 8(%ebp), %eax
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    addsd 12(%ebp), %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
-; X86-SSE2-NEXT:    movl (%esp), %ebx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl (%esi), %eax
-; X86-SSE2-NEXT:    movl 4(%esi), %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
-; X86-SSE2-NEXT:  .LBB1_1: # %atomicrmw.start
-; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SSE2-NEXT:    lock cmpxchg8b (%esi)
-; X86-SSE2-NEXT:    jne .LBB1_1
-; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-SSE2-NEXT:    leal -8(%ebp), %esp
-; X86-SSE2-NEXT:    popl %esi
-; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%eax)
+; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_64r:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %ebp
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    pushl %ebx
-; X86-AVX-NEXT:    pushl %esi
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $8, %esp
-; X86-AVX-NEXT:    movl 8(%ebp), %esi
+; X86-AVX-NEXT:    movl 8(%ebp), %eax
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vaddsd 12(%ebp), %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
-; X86-AVX-NEXT:    movl (%esp), %ebx
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-AVX-NEXT:    movl (%esi), %eax
-; X86-AVX-NEXT:    movl 4(%esi), %edx
-; X86-AVX-NEXT:    .p2align 4, 0x90
-; X86-AVX-NEXT:  .LBB1_1: # %atomicrmw.start
-; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-AVX-NEXT:    lock cmpxchg8b (%esi)
-; X86-AVX-NEXT:    jne .LBB1_1
-; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-AVX-NEXT:    leal -8(%ebp), %esp
-; X86-AVX-NEXT:    popl %esi
-; X86-AVX-NEXT:    popl %ebx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%eax)
+; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-LABEL: fadd_64r:
 ; X64:       # %bb.0:
 ; X64-NEXT:    addsd (%rdi), %xmm0
 ; X64-NEXT:    movsd %xmm0, (%rdi)
 ; X64-NEXT:    retq

 ; X86-SSE1-NEXT:    popl %ebx
 ; X86-SSE1-NEXT:    popl %ebp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_64g:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %ebp
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %ebx
 ; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
-; X86-SSE2-NEXT:    movl (%esp), %ebx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl glob64+4, %edx
-; X86-SSE2-NEXT:    movl glob64, %eax
-; X86-SSE2-NEXT:    .p2align 4, 0x90
-; X86-SSE2-NEXT:  .LBB3_1: # %atomicrmw.start
-; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SSE2-NEXT:    lock cmpxchg8b glob64
-; X86-SSE2-NEXT:    jne .LBB3_1
-; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-SSE2-NEXT:    leal -4(%ebp), %esp
-; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, glob64
+; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_64g:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %ebp
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    pushl %ebx
 ; X86-AVX-NEXT:    andl $-8, %esp
-; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
-; X86-AVX-NEXT:    movl (%esp), %ebx
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-AVX-NEXT:    movl glob64+4, %edx
-; X86-AVX-NEXT:    movl glob64, %eax
-; X86-AVX-NEXT:    .p2align 4, 0x90
-; X86-AVX-NEXT:  .LBB3_1: # %atomicrmw.start
-; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-AVX-NEXT:    lock cmpxchg8b glob64
-; X86-AVX-NEXT:    jne .LBB3_1
-; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-AVX-NEXT:    leal -4(%ebp), %esp
-; X86-AVX-NEXT:    popl %ebx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, glob64
+; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_64g:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    addsd {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    movsd %xmm0, {{.*}}(%rip)

 ; X86-SSE1-NEXT:    popl %ebx
 ; X86-SSE1-NEXT:    popl %ebp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_64imm:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %ebp
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %ebx
 ; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
-; X86-SSE2-NEXT:    movl (%esp), %ebx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl -559038737, %eax
-; X86-SSE2-NEXT:    movl -559038733, %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
-; X86-SSE2-NEXT:  .LBB5_1: # %atomicrmw.start
-; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SSE2-NEXT:    lock cmpxchg8b -559038737
-; X86-SSE2-NEXT:    jne .LBB5_1
-; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-SSE2-NEXT:    leal -4(%ebp), %esp
-; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, -559038737
+; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_64imm:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %ebp
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    pushl %ebx
 ; X86-AVX-NEXT:    andl $-8, %esp
-; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
-; X86-AVX-NEXT:    movl (%esp), %ebx
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-AVX-NEXT:    movl -559038737, %eax
-; X86-AVX-NEXT:    movl -559038733, %edx
-; X86-AVX-NEXT:    .p2align 4, 0x90
-; X86-AVX-NEXT:  .LBB5_1: # %atomicrmw.start
-; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-AVX-NEXT:    lock cmpxchg8b -559038737
-; X86-AVX-NEXT:    jne .LBB5_1
-; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-AVX-NEXT:    leal -4(%ebp), %esp
-; X86-AVX-NEXT:    popl %ebx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, -559038737
+; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_64imm:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    movl $3735928559, %eax # imm = 0xDEADBEEF
 ; X64-SSE-NEXT:    addsd (%rax), %xmm0

 ; X86-SSE1-NEXT:    popl %ebx
 ; X86-SSE1-NEXT:    popl %ebp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_64stack:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %ebp
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %ebx
 ; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $24, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
-; X86-SSE2-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl (%esp), %eax
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
-; X86-SSE2-NEXT:  .LBB7_1: # %atomicrmw.start
-; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SSE2-NEXT:    lock cmpxchg8b (%esp)
-; X86-SSE2-NEXT:    jne .LBB7_1
-; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-SSE2-NEXT:    leal -4(%ebp), %esp
-; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_64stack:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %ebp
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    pushl %ebx
 ; X86-AVX-NEXT:    andl $-8, %esp
-; X86-AVX-NEXT:    subl $24, %esp
+; X86-AVX-NEXT:    subl $16, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp)
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-AVX-NEXT:    movl (%esp), %eax
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-AVX-NEXT:    .p2align 4, 0x90
-; X86-AVX-NEXT:  .LBB7_1: # %atomicrmw.start
-; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-AVX-NEXT:    lock cmpxchg8b (%esp)
-; X86-AVX-NEXT:    jne .LBB7_1
-; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-AVX-NEXT:    leal -4(%ebp), %esp
-; X86-AVX-NEXT:    popl %ebx
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fadd_64stack:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSE-NEXT:    addsd -{{[0-9]+}}(%rsp), %xmm0
 ; X64-SSE-NEXT:    movsd %xmm0, -{{[0-9]+}}(%rsp)

 ; X86-SSE1-NEXT:    popl %ebx
 ; X86-SSE1-NEXT:    popl %ebp
 ; X86-SSE1-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: fadd_array:
 ; X86-SSE2:       # %bb.0: # %bb
 ; X86-SSE2-NEXT:    pushl %ebp
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    pushl %ebx
-; X86-SSE2-NEXT:    pushl %edi
-; X86-SSE2-NEXT:    pushl %esi
 ; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $16, %esp
-; X86-SSE2-NEXT:    movl 20(%ebp), %esi
-; X86-SSE2-NEXT:    movl 8(%ebp), %edi
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movl 20(%ebp), %eax
+; X86-SSE2-NEXT:    movl 8(%ebp), %ecx
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    addsd 12(%ebp), %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
-; X86-SSE2-NEXT:    movl (%esp), %ebx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl (%edi,%esi,8), %eax
-; X86-SSE2-NEXT:    movl 4(%edi,%esi,8), %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
-; X86-SSE2-NEXT:  .LBB8_1: # %atomicrmw.start
-; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SSE2-NEXT:    lock cmpxchg8b (%edi,%esi,8)
-; X86-SSE2-NEXT:    jne .LBB8_1
-; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-SSE2-NEXT:    leal -12(%ebp), %esp
-; X86-SSE2-NEXT:    popl %esi
-; X86-SSE2-NEXT:    popl %edi
-; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlps %xmm0, (%ecx,%eax,8)
+; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: fadd_array:
 ; X86-AVX:       # %bb.0: # %bb
 ; X86-AVX-NEXT:    pushl %ebp
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    pushl %ebx
-; X86-AVX-NEXT:    pushl %edi
-; X86-AVX-NEXT:    pushl %esi
 ; X86-AVX-NEXT:    andl $-8, %esp
-; X86-AVX-NEXT:    subl $16, %esp
-; X86-AVX-NEXT:    movl 20(%ebp), %esi
-; X86-AVX-NEXT:    movl 8(%ebp), %edi
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    movl 20(%ebp), %eax
+; X86-AVX-NEXT:    movl 8(%ebp), %ecx
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vaddsd 12(%ebp), %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
-; X86-AVX-NEXT:    movl (%esp), %ebx
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-AVX-NEXT:    movl (%edi,%esi,8), %eax
-; X86-AVX-NEXT:    movl 4(%edi,%esi,8), %edx
-; X86-AVX-NEXT:    .p2align 4, 0x90
-; X86-AVX-NEXT:  .LBB8_1: # %atomicrmw.start
-; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-AVX-NEXT:    lock cmpxchg8b (%edi,%esi,8)
-; X86-AVX-NEXT:    jne .LBB8_1
-; X86-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-AVX-NEXT:    leal -12(%ebp), %esp
-; X86-AVX-NEXT:    popl %esi
-; X86-AVX-NEXT:    popl %edi
-; X86-AVX-NEXT:    popl %ebx
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovlps %xmm0, (%ecx,%eax,8)
+; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-LABEL: fadd_array:
 ; X64:       # %bb.0: # %bb
 ; X64-NEXT:    addsd (%rdi,%rsi,8), %xmm0
 ; X64-NEXT:    movsd %xmm0, (%rdi,%rsi,8)
 ; X64-NEXT:    retq

llvm/test/CodeGen/X86/atomic6432.ll

 ; X32-NEXT:    retl
   %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire acquire
   ret void
 }
 
 define void @atomic_fetch_store64(i64 %x) nounwind {
 ; X32-LABEL: atomic_fetch_store64:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    subl $20, %esp
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT:    movl sc64+4, %eax
-; X32-NEXT:    movl sc64, %edx
-; X32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    jmp .LBB11_1
-; X32-NEXT:  .LBB11_1: # %atomicrmw.start
-; X32-NEXT:    # =>This Inner Loop Header: Depth=1
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    movl (%esp), %edx # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT:    lock cmpxchg8b sc64
-; X32-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT:    jne .LBB11_1
-; X32-NEXT:    jmp .LBB11_2
-; X32-NEXT:  .LBB11_2: # %atomicrmw.end
-; X32-NEXT:    addl $20, %esp
-; X32-NEXT:    popl %ebx
+; X32-NEXT:    movd %ecx, %xmm0
+; X32-NEXT:    pinsrd $1, %eax, %xmm0
+; X32-NEXT:    movq %xmm0, sc64
 ; X32-NEXT:    retl
   store atomic i64 %x, i64* @sc64 release, align 8
   ret void
 }
 
 define void @atomic_fetch_swap64(i64 %x) nounwind {
 ; X32-LABEL: atomic_fetch_swap64:
 ; X32:       # %bb.0: