Add scheduler classes to integer/float horizontal operations

[avt77]

Summary

This patch will close PR32801. It covers both ssse3 and avx versions.

[RKSimon]

This needs to be done in general, not just for Jaguar. Please can you add WriteFHAdd and WriteVecHAdd defs in X86Schedule.td, and then tag the relevant instructions in X86InstrSSE.td and X86InstrAVX512.td. Then in ScheduleBtVer2.td you need to add instances of the 2 defs and special case the ymm versions. Either add TODOs for the other x86 models or add them if you want to dig through Agner's tables.

[avt77]

I redesigned the implementation accordingly to Simon requirements. Now it's done in general way and every X86 should support horizontal operations modeling. I did not check the numbers for SB and SLM: I simply kept the current ones. And I separated Ymm version from Xmm version to be able to model the corresponding throughput difference for Jaguar.

[RKSimon]

Adding some Intel guys to look at the SLM/SB/HW model changes.

lib/Target/X86/X86InstrSSE.td

5194

Move on to previous line and re-add ReadAfterLd

lib/Target/X86/X86SchedSandyBridge.td

164

Isn't this comment out of date? You're modelling the horizontal operations below.

lib/Target/X86/X86Schedule.td

82

I'm not sure we should be introducing size specific versions - we don't do this for any other cases. Merge back into WriteHAdd?

lib/Target/X86/X86ScheduleBtVer2.td

68

You shouldn't need this - just use JFPU01

lib/Target/X86/X86ScheduleSLM.td

144

Isn't this comment out of date? You're modelling the horizontal operations below.

[avt77]

lib/Target/X86/X86Schedule.td

82

I did it specially because Jaguar has different numbers for Ymm and Xmm and we can't model such difference without the special SchedWrite. Is it OK?

[avt77]

lib/Target/X86/X86ScheduleSLM.td

144

In fact we did not model SLM here. I simply kept the current numbers: they could be wrong.

[javed.absar]

lib/Target/X86/X86SchedSandyBridge.td

169

This isn't needed I think as 'ResourceCycles = [1]' is default anyways.

[avt77]

I fixed all Simon's notes.

[avt77]

I removed default values defined for SB model (required by javed.absar).

[RKSimon]

lib/Target/X86/X86Schedule.td

82

Please can you use the instregex approach like you did in D33099?

lib/Target/X86/X86ScheduleSLM.td

144

That's fine, thanks.

[gadi.haber]

lib/Target/X86/X86SchedHaswell.td

1542

Please see the accurate modeling I added for these instrs in https://reviews.llvm.org/D33897

1544

Please see the accurate modeling I added for these instrs in https://reviews.llvm.org/D33897

1913

Please see the accurate modeling I added for these instrs in https://reviews.llvm.org/D33897

1920

Please see the accurate modeling I added for these instrs in https://reviews.llvm.org/D33897

[avt77]

I re-implemented all things mentioned by Simon and Gadi.

[javed.absar]

lib/Target/X86/X86ScheduleBtVer2.td

333

The 'let ResourceCycles = [1,1] ' is default and so redundant. You could shorten your patch by removing it if you like.

[RKSimon]

Almost there I think - couple of minors

lib/Target/X86/X86Schedule.td

80

// Horizontal Add/Sub (float and integer)

81

Rename this WriteFHAdd?

[avt77]

Some renames and other minor updates.

[RKSimon]

LGTM

lib/Target/X86/X86InstrSSE.td

 multiclass S3D_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,
                    X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag,
                    bit Is2Addr = 1> {
   def rr : S3DI<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
       [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>,
-      Sched<[WriteFAdd]>;
+      Sched<[WriteFHAdd]>;
 
   def rm : S3DI<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
       [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))],
-        IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>;
+        IIC_SSE_HADDSUB_RM>, Sched<[WriteFHAddLd, ReadAfterLd]>;
 }
 multiclass S3_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,
                   X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag,
                   bit Is2Addr = 1> {
   def rr : S3I<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
       [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>,
-      Sched<[WriteFAdd]>;
+      Sched<[WriteFHAdd]>;
 
   def rm : S3I<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
        !if(Is2Addr,
          !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
          !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
       [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))],
-        IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>;
+        IIC_SSE_HADDSUB_RM>, Sched<[WriteFHAddLd, ReadAfterLd]>;
 }
 
 let Predicates = [HasAVX] in {
   let ExeDomain = SSEPackedSingle in {
     defm VHADDPS  : S3D_Int<0x7C, "vhaddps", v4f32, VR128, f128mem,
                             X86fhadd, loadv4f32, 0>, VEX_4V, VEX_WIG;
     defm VHSUBPS  : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem,
                             X86fhsub, loadv4f32, 0>, VEX_4V, VEX_WIG;

 defm PABSB : SS3I_unop_rm<0x1C, "pabsb", v16i8, abs, memopv2i64>;
 defm PABSW : SS3I_unop_rm<0x1D, "pabsw", v8i16, abs, memopv2i64>;
 defm PABSD : SS3I_unop_rm<0x1E, "pabsd", v4i32, abs, memopv2i64>;
 
 //===---------------------------------------------------------------------===//
 // SSSE3 - Packed Binary Operator Instructions
 //===---------------------------------------------------------------------===//
 
-let Sched = WriteVecALU in {
+let Sched = WritePHAdd in {
 def SSE_PHADDSUBD : OpndItins<
   IIC_SSE_PHADDSUBD_RR, IIC_SSE_PHADDSUBD_RM
 >;
 def SSE_PHADDSUBSW : OpndItins<
   IIC_SSE_PHADDSUBSW_RR, IIC_SSE_PHADDSUBSW_RM
 >;
 def SSE_PHADDSUBW : OpndItins<
   IIC_SSE_PHADDSUBW_RR, IIC_SSE_PHADDSUBW_RM

lib/Target/X86/X86SchedHaswell.td

 // y.
 def WriteVPGATHERQQ256 : SchedWriteRes<[]> {
   let NumMicroOps = 22;
 }
 def : InstRW<[WriteVPGATHERQQ256, ReadAfterLd], (instregex "VPGATHERQQYrm")>;
 
 //-- Arithmetic instructions --//
 
+////////////////////////////////////////////////////////////////////////////////
+// Horizontal add/sub  instructions.
+////////////////////////////////////////////////////////////////////////////////
+
+// HADD, HSUB PS/PD
+// x,x / v,v,v.
+def : WriteRes<WriteFHAdd, [HWPort1, HWPort5]> {
+  let Latency = 5;
+  let NumMicroOps = 3;
+  let ResourceCycles = [1, 2];
+}
+
+// x,m / v,v,m.
+def : WriteRes<WriteFHAddLd, [HWPort1, HWPort5, HWPort23]> {
+  let Latency = 9;
+  let NumMicroOps = 4;
+  let ResourceCycles = [1, 2, 1];
+}
+
+// PHADD|PHSUB (S) W/D.
+// v <- v,v.
+def : WriteRes<WritePHAdd, [HWPort1, HWPort5]> {
+  let Latency = 3;
+  let NumMicroOps = 3;
+  let ResourceCycles = [1, 2];
+}
+// v <- v,m.
+def : WriteRes<WritePHAddLd, [HWPort1, HWPort5, HWPort23]> {
+  let Latency = 6;
+  let NumMicroOps = 3;
+  let ResourceCycles = [1, 2, 1];
+}
+
 // PHADD|PHSUB (S) W/D.
 // v <- v,v.
 def WritePHADDSUBr : SchedWriteRes<[HWPort1, HWPort5]> {
   let Latency = 3;
   let NumMicroOps = 3;
   let ResourceCycles = [1, 2];
 }
 def : InstRW<[WritePHADDSUBr], (instregex "MMX_PHADD(W?)rr64",
                                "MMX_PHADDSWrr64",
                                "MMX_PHSUB(W|D)rr64",
                                "MMX_PHSUBSWrr64",
                                "(V?)PH(ADD|SUB)(W|D)(Y?)rr",
                                "(V?)PH(ADD|SUB)SWrr(256)?")>;
 
 // v <- v,m.
 def WritePHADDSUBm : SchedWriteRes<[HWPort1, HWPort5, HWPort23]> {
   let Latency = 6;
   let NumMicroOps = 3;
   let ResourceCycles = [1, 2, 1];
 }
 def : InstRW<[WritePHADDSUBm, ReadAfterLd],
                               (instregex "MMX_PHADD(W?)rm64",
                                "MMX_PHADDSWrm64",
                                "MMX_PHSUB(W|D)rm64",
                                "MMX_PHSUBSWrm64",
                                "(V?)PH(ADD|SUB)(W|D)(Y?)rm",
                                "(V?)PH(ADD|SUB)SWrm(128|256)?")>;
 
 // PCMPGTQ.

 // HADD, HSUB PS/PD
 // x,x / v,v,v.
 def WriteHADDSUBPr : SchedWriteRes<[HWPort1, HWPort5]> {
   let Latency = 5;
   let NumMicroOps = 3;
   let ResourceCycles = [1, 2];
 }
 def : InstRW<[WriteHADDSUBPr], (instregex "(V?)H(ADD|SUB)P(S|D)(Y?)rr")>;
 
 // x,m / v,v,m.
 def WriteHADDSUBPm : SchedWriteRes<[HWPort1, HWPort5, HWPort23]> {
   let Latency = 9;
   let NumMicroOps = 4;
   let ResourceCycles = [1, 2, 1];
 }
 def : InstRW<[WriteHADDSUBPm], (instregex "(V?)H(ADD|SUB)P(S|D)(Y?)rm")>;
 
 // MULL SS/SD PS/PD.
 // x,x / v,v,v.
 def WriteMULr : SchedWriteRes<[HWPort01]> {
   let Latency = 5;
 }
 def : InstRW<[WriteMULr], (instregex "(V?)MUL(P|S)(S|D)rr")>;
 

lib/Target/X86/X86SchedSandyBridge.td

def : WriteRes<WriteMPSAD, [SBPort0, SBPort1, SBPort5]> {
   let Latency = 6;
   let ResourceCycles = [1, 1, 1];
 }
 def : WriteRes<WriteMPSADLd, [SBPort0, SBPort1, SBPort5, SBPort23]> {
   let Latency = 6;
   let ResourceCycles = [1, 1, 1, 1];
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// Horizontal add/sub  instructions.
+////////////////////////////////////////////////////////////////////////////////
+// HADD, HSUB PS/PD
+// x,x / v,v,v.
+def : WriteRes<WriteFHAdd, [SBPort1]> {
+  let Latency = 3;
+}
+
+// x,m / v,v,m.
+def : WriteRes<WriteFHAddLd, [SBPort1, SBPort23]> {
+  let Latency = 7;
+  let ResourceCycles = [1, 1];
+}
+
+// PHADD|PHSUB (S) W/D.
+// v <- v,v.
+def : WriteRes<WritePHAdd, [SBPort15]>;
+
+// v <- v,m.
+def : WriteRes<WritePHAddLd, [SBPort15, SBPort23]> {
+  let Latency = 5;
+  let ResourceCycles = [1, 1];
+}
+
 // String instructions.
 // Packed Compare Implicit Length Strings, Return Mask
 def : WriteRes<WritePCmpIStrM, [SBPort015]> {
   let Latency = 11;
   let ResourceCycles = [3];
 }
 def : WriteRes<WritePCmpIStrMLd, [SBPort015, SBPort23]> {
   let Latency = 11;

lib/Target/X86/X86Schedule.td

 defm WriteFMA    : X86SchedWritePair; // Fused Multiply Add.
 defm WriteFShuffle  : X86SchedWritePair; // Floating point vector shuffles.
 defm WriteFBlend  : X86SchedWritePair; // Floating point vector blends.
 defm WriteFVarBlend  : X86SchedWritePair; // Fp vector variable blends.
 
 // FMA Scheduling helper class.
 class FMASC { X86FoldableSchedWrite Sched = WriteFAdd; }
 
+// Horizontal Add/Sub (float and integer)
+defm WriteFHAdd  : X86SchedWritePair;
+defm WritePHAdd : X86SchedWritePair;
+
 // Vector integer operations.
 defm WriteVecALU   : X86SchedWritePair; // Vector integer ALU op, no logicals.
 defm WriteVecShift : X86SchedWritePair; // Vector integer shifts.
 defm WriteVecIMul  : X86SchedWritePair; // Vector integer multiply.
 defm WriteShuffle  : X86SchedWritePair; // Vector shuffles.
 defm WriteBlend  : X86SchedWritePair; // Vector blends.
 defm WriteVarBlend  : X86SchedWritePair; // Vector variable blends.
 defm WriteMPSAD : X86SchedWritePair; // Vector MPSAD.

lib/Target/X86/X86ScheduleBtVer2.td

 def JMul    : ProcResource<1>; // integer multiplication
 def JVALU0  : ProcResource<1>; // vector integer
 def JVALU1  : ProcResource<1>; // vector integer
 def JVIMUL  : ProcResource<1>; // vector integer multiplication
 def JSTC    : ProcResource<1>; // vector store/convert
 def JFPM    : ProcResource<1>; // FP multiplication
 def JFPA    : ProcResource<1>; // FP addition
 
 // Integer loads are 3 cycles, so ReadAfterLd registers needn't be available until 3
 // cycles after the memory operand.
 def : ReadAdvance<ReadAfterLd, 3>;
 
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when dispatched by the schedulers.
 // This multiclass defines the resource usage for variants with and without
 // folded loads.
def : WriteRes<WriteAESKeyGen, [JVIMUL]> {
   let ResourceCycles = [1];
 }
 def : WriteRes<WriteAESKeyGenLd, [JLAGU, JVIMUL]> {
   let Latency = 7;
   let ResourceCycles = [1, 1];
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+// Horizontal add/sub  instructions.
+////////////////////////////////////////////////////////////////////////////////
+
+def : WriteRes<WriteFHAdd, [JFPU0]> {
+  let Latency = 3;
+}
+
+def : WriteRes<WriteFHAddLd, [JLAGU, JFPU0]> {
+  let Latency = 8;
+}
+
+def : WriteRes<WritePHAdd, [JFPU01]> {
+  let ResourceCycles = [1];
+}
+def : WriteRes<WritePHAddLd, [JLAGU, JFPU01 ]> {
+  let Latency = 6;
+  let ResourceCycles = [1, 1];
+}
+
+def WriteFHAddY: SchedWriteRes<[JFPU0]> {
+  let Latency = 3;
+  let ResourceCycles = [2];
+}
+def : InstRW<[WriteFHAddY], (instregex "VH(ADD|SUB)P(S|D)Yrr")>;
+
+def WriteFHAddYLd: SchedWriteRes<[JLAGU, JFPU0]> {
+  let Latency = 8;
+  let ResourceCycles = [1, 2];
+}
+def : InstRW<[WriteFHAddYLd], (instregex "VH(ADD|SUB)P(S|D)Yrm")>;
+
+////////////////////////////////////////////////////////////////////////////////
 // Carry-less multiplication instructions.
 ////////////////////////////////////////////////////////////////////////////////
 
 def : WriteRes<WriteCLMul, [JVIMUL]> {
   let Latency = 2;
   let ResourceCycles = [1];
 }
 def : WriteRes<WriteCLMulLd, [JLAGU, JVIMUL]> {

lib/Target/X86/X86ScheduleSLM.td

 defm : SMWriteResPair<WriteVecShift, FPC_RSV0,  1>;
 defm : SMWriteResPair<WriteVecLogic, FPC_RSV01, 1>;
 defm : SMWriteResPair<WriteVecALU,   FPC_RSV01,  1>;
 defm : SMWriteResPair<WriteVecIMul,  FPC_RSV0,   4>;
 defm : SMWriteResPair<WriteShuffle,  FPC_RSV0,  1>;
 defm : SMWriteResPair<WriteBlend,  FPC_RSV0,  1>;
 defm : SMWriteResPair<WriteMPSAD,  FPC_RSV0,  7>;
 
+////////////////////////////////////////////////////////////////////////////////
+// Horizontal add/sub  instructions.
+////////////////////////////////////////////////////////////////////////////////
+
+// HADD, HSUB PS/PD
+
+def : WriteRes<WriteFHAdd,  [FPC_RSV01]> {
+  let Latency = 3;
+  let ResourceCycles = [2];
+}
+
+def : WriteRes<WriteFHAddLd,  [FPC_RSV01, MEC_RSV]> {
+  let Latency = 6;
+  let ResourceCycles = [2, 1];
+}
+
+// PHADD|PHSUB (S) W/D.
+def : WriteRes<WritePHAdd,  [FPC_RSV01]> {
+  let Latency = 1;
+  let ResourceCycles = [1];
+}
+
+def : WriteRes<WritePHAddLd,  [FPC_RSV01, MEC_RSV]> {
+  let Latency = 4;
+  let ResourceCycles = [1, 1];
+}
+
 // String instructions.
 // Packed Compare Implicit Length Strings, Return Mask
 def : WriteRes<WritePCmpIStrM, [FPC_RSV0]> {
   let Latency = 13;
   let ResourceCycles = [13];
 }
 def : WriteRes<WritePCmpIStrMLd, [FPC_RSV0, MEC_RSV]> {
   let Latency = 13;

test/CodeGen/X86/avx-schedule.ll

 ; HASWELL-LABEL: test_haddpd:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
 ; HASWELL-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_haddpd:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; BTVER2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; BTVER2-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_haddpd:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; ZNVER1-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; ZNVER1-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %a0, <4 x double> %a1)
   %2 = load <4 x double>, <4 x double> *%a2, align 32
   %3 = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %1, <4 x double> %2)
   ret <4 x double> %3
 }
 declare <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <8 x float> @test_haddps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
 ; SANDY-LABEL: test_haddps:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
 ; SANDY-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_haddps:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
 ; HASWELL-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_haddps:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; BTVER2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; BTVER2-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_haddps:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; ZNVER1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; ZNVER1-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %a0, <8 x float> %a1)
   %2 = load <8 x float>, <8 x float> *%a2, align 32
   %3 = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %1, <8 x float> %2)
   ret <8 x float> %3
 }
 declare <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float>, <8 x float>) nounwind readnone
 
 define <4 x double> @test_hsubpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
 ; SANDY-LABEL: test_hsubpd:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
 ; SANDY-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_hsubpd:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
 ; HASWELL-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_hsubpd:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; BTVER2-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; BTVER2-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_hsubpd:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; ZNVER1-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; ZNVER1-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %a0, <4 x double> %a1)
   %2 = load <4 x double>, <4 x double> *%a2, align 32
   %3 = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %1, <4 x double> %2)
   ret <4 x double> %3
 }
 declare <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
 
 define <8 x float> @test_hsubps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
 ; SANDY-LABEL: test_hsubps:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
 ; SANDY-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_hsubps:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
 ; HASWELL-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_hsubps:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; BTVER2-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; BTVER2-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_hsubps:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
+; ZNVER1-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
+; ZNVER1-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %a0, <8 x float> %a1)
   %2 = load <8 x float>, <8 x float> *%a2, align 32
   %3 = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %1, <8 x float> %2)
   ret <8 x float> %3
 }
 declare <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float>, <8 x float>) nounwind readnone