[x86] Provide execution domains for scalar floating point operations.

[chandlerc]

Summary

While we might think of these as not executing in the "packed" domain,
in fact they do. They use the same set of resources as the packed
floating point operations in every sense, and we want to ensure that we
use the correct shuffle, load, and store instructions when interacting
with them.

Fixing this today is largely aesthetic because we just ended up using
'packed single' as the fallback default. But it removes a false
domain break and will ensure that if the default is the integer domain
that doesn't interfere with producing good floating point code.

Note that this use a 'let ExeDomain = ... in' mechanism rather than
threading it through all of the multiclasses the way I did in r228135.
I'm interested in which approach seems more appropriate, and I'm happy
to make them both converge on that.

[qcolombet]

Hi Chandler,

LGTM.

Note that this use a 'let ExeDomain = ... in' mechanism rather than threading it through all of the multiclasses the way I did in r228135. I'm interested in which approach seems more appropriate, and I'm happy to make them both converge on that.

I am not sure there is a right answer here. 'let ExeDomain = ... in' is convenient when we have a bunch of instructions we want to put on one domain. However, in the instruction per instruction case it is more verbose. That being said, I tend to prefer the 'let' writing as it makes the information stands out.

Thanks,
Q.

[RKSimon]

An updated version of this was reviewed in D9095 and committed in rL235372

lib/Target/X86/X86InstrSSE.td

let Constraints = "$src1 = $dst" in {
     defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128,
                               v2f64, f128mem, memopv2f64, SSEPackedDouble,
                               itins.d>, PD;
   }
 }
 
 multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
                                   SizeItins itins> {
+  let ExeDomain = SSEPackedSingle in
   defm V#NAME#SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),
                          OpNode, FR32, f32mem, itins.s, 0>, XS, VEX_4V, VEX_LIG;
+  let ExeDomain = SSEPackedDouble in
   defm V#NAME#SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),
                          OpNode, FR64, f64mem, itins.d, 0>, XD, VEX_4V, VEX_LIG;
 
   let Constraints = "$src1 = $dst" in {
+    let ExeDomain = SSEPackedSingle in
     defm SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),
                               OpNode, FR32, f32mem, itins.s>, XS;
+    let ExeDomain = SSEPackedDouble in
     defm SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),
                               OpNode, FR64, f64mem, itins.d>, XD;
   }
 }
 
 multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr,
                                       SizeItins itins> {
+  let ExeDomain = SSEPackedSingle in
   defm V#NAME#SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32,
                    itins.s, 0>, XS, VEX_4V, VEX_LIG;
+  let ExeDomain = SSEPackedDouble in
   defm V#NAME#SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64,
                    itins.d, 0>, XD, VEX_4V, VEX_LIG;
 
   let Constraints = "$src1 = $dst" in {
+    let ExeDomain = SSEPackedSingle in
     defm SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32,
                    itins.s>, XS;
+    let ExeDomain = SSEPackedDouble in
     defm SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128,
                    !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64,
                    itins.d>, XD;
   }
 }
 
 // Binary Arithmetic instructions
 defm ADD : basic_sse12_fp_binop_p<0x58, "add", fadd, SSE_ALU_ITINS_P>,

 >;
 }
 
 /// sse1_fp_unop_s - SSE1 unops in scalar form
 /// For the non-AVX defs, we need $src1 to be tied to $dst because
 /// the HW instructions are 2 operand / destructive.
 multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
                            OpndItins itins> {
+  let ExeDomain = SSEPackedSingle in {
     let Predicates = [HasAVX], hasSideEffects = 0 in {
       def V#NAME#SSr : SSI<opc, MRMSrcReg, (outs FR32:$dst),
                            (ins FR32:$src1, FR32:$src2),
                            !strconcat("v", OpcodeStr,
                                "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                     []>, VEX_4V, VEX_LIG, Sched<[itins.Sched]>;
       let mayLoad = 1 in {
       def V#NAME#SSm : SSI<opc, MRMSrcMem, (outs FR32:$dst),
                           (ins FR32:$src1,f32mem:$src2),
                           !strconcat("v", OpcodeStr,
                                      "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                           []>, VEX_4V, VEX_LIG,
                        Sched<[itins.Sched.Folded, ReadAfterLd]>;
       let isCodeGenOnly = 1 in
       def V#NAME#SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst),
                           (ins VR128:$src1, ssmem:$src2),
                           !strconcat("v", OpcodeStr,
                                      "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                           []>, VEX_4V, VEX_LIG,
                           Sched<[itins.Sched.Folded, ReadAfterLd]>;
       }
     }
 
     def SSr : SSI<opc, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src),
                   !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
                   [(set FR32:$dst, (OpNode FR32:$src))]>, Sched<[itins.Sched]>;
     // For scalar unary operations, fold a load into the operation
     // only in OptForSize mode. It eliminates an instruction, but it also
     // eliminates a whole-register clobber (the load), so it introduces a
     // partial register update condition.
     def SSm : I<opc, MRMSrcMem, (outs FR32:$dst), (ins f32mem:$src),
                   !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
                   [(set FR32:$dst, (OpNode (load addr:$src)))], itins.rm>, XS,
               Requires<[UseSSE1, OptForSize]>, Sched<[itins.Sched.Folded]>;
     let isCodeGenOnly = 1, Constraints = "$src1 = $dst" in {
       def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst),
                         (ins VR128:$src1, VR128:$src2),
                         !strconcat(OpcodeStr, "ss\t{$src2, $dst|$dst, $src2}"),
                         [], itins.rr>, Sched<[itins.Sched]>;
       let mayLoad = 1, hasSideEffects = 0 in
       def SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst),
                         (ins VR128:$src1, ssmem:$src2),
                         !strconcat(OpcodeStr, "ss\t{$src2, $dst|$dst, $src2}"),
                         [], itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
     }
   }
+}
 
 /// sse1_fp_unop_p - SSE1 unops in packed form.
 multiclass sse1_fp_unop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
                           OpndItins itins> {
 let Predicates = [HasAVX] in {
   def V#NAME#PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        !strconcat("v", OpcodeStr,
                                   "ps\t{$src, $dst|$dst, $src}"),

 } // isCodeGenOnly = 1
 }
 
 /// sse2_fp_unop_s - SSE2 unops in scalar form.
 // FIXME: Combine the following sse2 classes with the sse1 classes above.
 // The only usage of these is for SQRT[S/P]D. See sse12_fp_binop* for example.
 multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr,
                           SDNode OpNode, OpndItins itins> {
+  let ExeDomain = SSEPackedDouble in {
     let Predicates = [HasAVX], hasSideEffects = 0 in {
       def V#NAME#SDr : SDI<opc, MRMSrcReg, (outs FR64:$dst),
                           (ins FR64:$src1, FR64:$src2),
                           !strconcat("v", OpcodeStr,
                                      "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                           []>, VEX_4V, VEX_LIG, Sched<[itins.Sched]>;
       let mayLoad = 1 in {
       def V#NAME#SDm : SDI<opc, MRMSrcMem, (outs FR64:$dst),
                           (ins FR64:$src1,f64mem:$src2),
                           !strconcat("v", OpcodeStr,
                                      "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                           []>, VEX_4V, VEX_LIG,
                        Sched<[itins.Sched.Folded, ReadAfterLd]>;
       let isCodeGenOnly = 1 in
       def V#NAME#SDm_Int : SDI<opc, MRMSrcMem, (outs VR128:$dst),
                           (ins VR128:$src1, sdmem:$src2),
                           !strconcat("v", OpcodeStr,
                                      "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                           []>, VEX_4V, VEX_LIG,
                           Sched<[itins.Sched.Folded, ReadAfterLd]>;
       }
     }
 
     def SDr : SDI<opc, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src),
                   !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"),
                   [(set FR64:$dst, (OpNode FR64:$src))], itins.rr>,
               Sched<[itins.Sched]>;
     // See the comments in sse1_fp_unop_s for why this is OptForSize.
     def SDm : I<opc, MRMSrcMem, (outs FR64:$dst), (ins f64mem:$src),
                   !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"),
                   [(set FR64:$dst, (OpNode (load addr:$src)))], itins.rm>, XD,
               Requires<[UseSSE2, OptForSize]>, Sched<[itins.Sched.Folded]>;
     let isCodeGenOnly = 1, Constraints = "$src1 = $dst" in {
     def SDr_Int :
       SDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
       !strconcat(OpcodeStr, "sd\t{$src2, $dst|$dst, $src2}"),
       [], itins.rr>, Sched<[itins.Sched]>;
-  
     let mayLoad = 1, hasSideEffects = 0 in
     def SDm_Int :
       SDI<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2),
       !strconcat(OpcodeStr, "sd\t{$src2, $dst|$dst, $src2}"),
       [], itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>;
     } // isCodeGenOnly, Constraints
   }
+}
 
 /// sse2_fp_unop_p - SSE2 unops in vector forms.
 multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr,
                           SDNode OpNode, OpndItins itins> {
 let Predicates = [HasAVX] in {
   def V#NAME#PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        !strconcat("v", OpcodeStr,
                                   "pd\t{$src, $dst|$dst, $src}"),

test/CodeGen/X86/sink-hoist.ll

 ; RUN: llc < %s -verify-machineinstrs -march=x86-64 -asm-verbose=false -mtriple=x86_64-unknown-linux-gnu -mcpu=nehalem -post-RA-scheduler=true -schedmodel=false | FileCheck %s
 
 ; Currently, floating-point selects are lowered to CFG triangles.
 ; This means that one side of the select is always unconditionally
 ; evaluated, however with MachineSink we can sink the other side so
 ; that it's conditionally evaluated.
 
 ; CHECK-LABEL: foo:
 ; CHECK-NEXT: testb $1, %dil
 ; CHECK-NEXT: jne
 ; CHECK-NEXT: divsd
-; CHECK-NEXT: movaps
+; CHECK-NEXT: movapd
 ; CHECK-NEXT: ret
 ; CHECK:      divsd
 
 define double @foo(double %x, double %y, i1 %c) nounwind {
   %a = fdiv double %x, 3.2
   %b = fdiv double %y, 3.3
   %z = select i1 %c, double %a, double %b
   ret double %z
 }
 
 ; Make sure the critical edge is broken so the divsd is sunken below
 ; the conditional branch.
 ; rdar://8454886
 
 ; CHECK-LABEL: split:
 ; CHECK-NEXT: testb $1, %dil
 ; CHECK-NEXT: je
 ; CHECK:      divsd
-; CHECK:      movaps
+; CHECK:      movapd
 ; CHECK:      ret
 define double @split(double %x, double %y, i1 %c) nounwind {
   %a = fdiv double %x, 3.2
   %z = select i1 %c, double %a, double %y
   ret double %z
 }
 
 

test/CodeGen/X86/sse-minmax.ll

define double @ule_inverse_y(double %x) nounwind {
   %d = select i1 %c, double -0.000000e+00, double %x
   ret double %d
 }
 ; Test a few more misc. cases.
 
 ; CHECK-LABEL: clampTo3k_a:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: minsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_a:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_a:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_a(double %x) nounwind readnone {
 entry:
   %0 = fcmp ogt double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_b:
 ; CHECK-NEXT: minsd {{[^,]*}}, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_b:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_b:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_b(double %x) nounwind readnone {
 entry:
   %0 = fcmp uge double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_c:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: maxsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_c:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_c:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_c(double %x) nounwind readnone {
 entry:
   %0 = fcmp olt double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_d:
 ; CHECK-NEXT: maxsd {{[^,]*}}, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_d:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_d:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_d(double %x) nounwind readnone {
 entry:
   %0 = fcmp ule double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_e:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: maxsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_e:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_e:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_e(double %x) nounwind readnone {
 entry:
   %0 = fcmp olt double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_f:
 ; CHECK-NEXT: maxsd {{[^,]*}}, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_f:
 ; UNSAFE-NEXT: maxsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_f:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: maxsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_f(double %x) nounwind readnone {
 entry:
   %0 = fcmp ule double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_g:
 ; CHECK-NEXT: movsd {{[^,]*}}, %xmm1
 ; CHECK-NEXT: minsd %xmm0, %xmm1
-; CHECK-NEXT: movaps %xmm1, %xmm0
+; CHECK-NEXT: movapd %xmm1, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_g:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_g:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_g(double %x) nounwind readnone {
 entry:
   %0 = fcmp ogt double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 
 ; CHECK-LABEL: clampTo3k_h:
 ; CHECK-NEXT: minsd {{[^,]*}}, %xmm0
 ; CHECK-NEXT: ret
 ; UNSAFE-LABEL: clampTo3k_h:
 ; UNSAFE-NEXT: minsd {{[^,]*}}, %xmm0
 ; UNSAFE-NEXT: ret
 ; FINITE-LABEL: clampTo3k_h:
 ; FINITE-NEXT: movsd {{[^,]*}}, %xmm1
 ; FINITE-NEXT: minsd %xmm0, %xmm1
-; FINITE-NEXT: movaps %xmm1, %xmm0
+; FINITE-NEXT: movapd %xmm1, %xmm0
 ; FINITE-NEXT: ret
 define double @clampTo3k_h(double %x) nounwind readnone {
 entry:
   %0 = fcmp uge double %x, 3.000000e+03           ; <i1> [#uses=1]
   %x_addr.0 = select i1 %0, double 3.000000e+03, double %x ; <double> [#uses=1]
   ret double %x_addr.0
 }
 

test/CodeGen/X86/sse-scalar-fp-arith.ll

; AVX-NEXT:    retq
   %3 = insertelement <4 x float> %b, float %div, i32 0
   ret <4 x float> %3
 }
 
 define <2 x double> @test2_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_add_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %b, i32 0
   %add = fadd double %1, %2
   %3 = insertelement <2 x double> %b, double %add, i32 0
   ret <2 x double> %3
 }
 
 define <2 x double> @test2_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_sub_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %b, i32 0
   %sub = fsub double %2, %1
   %3 = insertelement <2 x double> %b, double %sub, i32 0
   ret <2 x double> %3
 }
 
 define <2 x double> @test2_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_mul_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmulsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %b, i32 0
   %mul = fmul double %1, %2
   %3 = insertelement <2 x double> %b, double %mul, i32 0
   ret <2 x double> %3
 }
 
 define <2 x double> @test2_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: test2_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2_div_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %b, i32 0
; AVX-NEXT:    retq
   %2 = shufflevector <4 x float> %1, <4 x float> %b, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x float> %2
 }
 
 define <2 x double> @insert_test2_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_add_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fadd <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test2_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_sub_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fsub <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test2_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_mul_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmulsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fmul <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test2_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test2_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test2_div_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fdiv <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %b, <2 x i32> <i32 0, i32 3>
; AVX-NEXT:    retq
   %2 = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> %b, <4 x float> %1
   ret <4 x float> %2
 }
 
 define <2 x double> @insert_test4_add_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_add_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_add_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fadd <2 x double> %b, %a
   %2 = select <2 x i1> <i1 false, i1 true>, <2 x double> %b, <2 x double> %1
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test4_sub_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_sub_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    subsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_sub_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fsub <2 x double> %b, %a
   %2 = select <2 x i1> <i1 false, i1 true>, <2 x double> %b, <2 x double> %1
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test4_mul_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_mul_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    mulsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_mul_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmulsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fmul <2 x double> %b, %a
   %2 = select <2 x i1> <i1 false, i1 true>, <2 x double> %b, <2 x double> %1
   ret <2 x double> %2
 }
 
 define <2 x double> @insert_test4_div_sd(<2 x double> %a, <2 x double> %b) {
 ; SSE-LABEL: insert_test4_div_sd:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    divsd %xmm0, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movapd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_test4_div_sd:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vdivsd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %1 = fdiv <2 x double> %b, %a
   %2 = select <2 x i1> <i1 false, i1 true>, <2 x double> %b, <2 x double> %1
   ret <2 x double> %2
 }