[X86] Correct the placement of ReadAfterLd in BEXTR and BZHI

[craig.topper]

Summary

These instructions have the memory operand before the register operand. So we need to put ReadDefault for all the load ops first. Then the ReadAfterLd

[andreadb]

Hi Craig.

The change looks good. The ReadAdvance is now applied to the right operand.

However, Haswell model should be fixed too.
At the moment, an InstrRW definition is overriding the SchedRW list for BZHI(32|64)rm. So, we don't get the ReadAdvance value at all.
You can see this if you look at scheduling class id "BZHI32rm_BZHI64rm" (which should be index #1246) in X86GenSubtargetInfo.inc. That scheduling class declares 0 read-advance entries.

Same for BEXTR (see the InstrRW at line 1609 in X86SchedHaswell.td). I didn't check other Intel models. However I expect that the same issue affects all intel processors for which the model extends haswell model.

If you test the following code in llvm-mca (-mcpu=haswell -iterations=1 -timeline):

add    %edi, %esi
bzhil   %esi, (%rdi), %eax

You get the following output:

Timeline view:

Index   0123456789

[0,0]   DeER .   .      addl    %edi, %esi
[0,1]   D=eeeeeeER      bzhil   %esi, (%rdi), %eax
         ^

The bzhil should be able to start immediately at cycle 1. However, it waits until the addl reaches the write-back stage.
The right timeline should have been:

Timeline view:

Index   0123456789

[0,0]   DeER .   .      addl    %edi, %esi
[0,1]   DeeeeeeER.      bzhil   %esi, (%rdi), %eax

You can see the same problem if you replace bzhil with bextrl.

[andreadb]

lib/Target/X86/X86ScheduleZnver1.td

573 ↗

(On Diff #139622)

You can just enumerate opcodes using (instrs BEXTR32rm, BEXTR64rm).

580 ↗

(On Diff #139622)

Same. You can use (instrs BZHI32rm, BZHI64rm).

[craig.topper]

So basically the Intel per instruction overrides have broken every instruction with a folded load because we lost the ReadAfterLd on all of them?

For example
[0,0] DeER . . addl %edi, %esi
[0,1] D=eeeeeeER addl (%rdi), %esi

[craig.topper]

Add dedicated SchedRW for BZHI and BEXTR and remove all of the regexes.

[andreadb]

In D44838#1052205, @craig.topper wrote:

So basically the Intel per instruction overrides have broken every instruction with a folded load because we lost the ReadAfterLd on all of them?

For example
[0,0] DeER . . addl %edi, %esi
[0,1] D=eeeeeeER addl (%rdi), %esi

Looks like it... Sounds scary.

I just tried that same code snippet.
For btver2 I get the expected result:

Index   0123456

[0,0]   DeER .. addl    %edi, %esi
[0,1]   DeeeeER addl    (%rdi), %esi
`

Unfortunately I don't get the correct ReadAdvance on Haswell (I get the same output that you posted).

On Haswell, the SchedClassID for the add with the folded memory operand is #912.
According to X86GenSubtargetInfo.inc

{DBGFIELD("ADD16rm_ADD32rm_ADD64rm_ADD8rm_AND16rm_AND32rm_AND64rm_AND8rm_CMP16mi_CMP16mi8_CMP32mi_CMP32mi8_CMP64mi32_CMP64mi8_CMP8mi_CMP8mi8_CMP16mr_CMP32mr_CMP64mr_CMP8mr_CMP16rm_CMP32rm_CMP64rm_CMP8rm_OR16rm_OR32rm_OR64rm_OR8rm_SUB16rm_SUB32rm_SUB64rm_SUB8rm_TEST16mr_TEST32mr_TEST64mr_TEST8mr_TEST16mi_TEST32mi_TEST64mi32_TEST8mi_XOR16rm_XOR32rm_XOR64rm_XOR8rm") 2, false, false, 12, 4,  3, 1,  0, 0}, // #912

The last two numbers are respectively the "ReadAdvanceIdx", and the number of read-advance entries. Both are zero for Haswell, which means we lost that information.
Basically, all the instructions that use HWWriteResGroup18 had lost the ReadAdvance info.

[andreadb]

About the bzhi/bextr change:
Your new patch looks good to me.

I didn't check the new timeline with llvm-mca for the bzhi/bextr. However I trust you that the read-advance is now there.

Thanks,
-Andrea

llvm/trunk/lib/Target/X86/X86InstrInfo.td

let Predicates = [HasBMI] in {
   def : Pat<(and GR32:$src, (ineg GR32:$src)),
             (BLSI32rr GR32:$src)>;
   def : Pat<(and GR64:$src, (ineg GR64:$src)),
             (BLSI64rr GR64:$src)>;
 }
 
 multiclass bmi_bextr_bzhi<bits<8> opc, string mnemonic, RegisterClass RC,
                           X86MemOperand x86memop, Intrinsic Int,
-                          PatFrag ld_frag> {
+                          PatFrag ld_frag, X86FoldableSchedWrite Sched> {
   def rr : I<opc, MRMSrcReg4VOp3, (outs RC:$dst), (ins RC:$src1, RC:$src2),
              !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set RC:$dst, (Int RC:$src1, RC:$src2)), (implicit EFLAGS)], IIC_BIN_NONMEM>,
-             T8PS, VEX, Sched<[WriteALU]>;
+             T8PS, VEX, Sched<[Sched]>;
   def rm : I<opc, MRMSrcMem4VOp3, (outs RC:$dst), (ins x86memop:$src1, RC:$src2),
              !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set RC:$dst, (Int (ld_frag addr:$src1), RC:$src2)),
               (implicit EFLAGS)], IIC_BIN_MEM>, T8PS, VEX,
-             Sched<[WriteALULd, ReadAfterLd]>;
+             Sched<[Sched.Folded,
+                    // x86memop:$src1
+                    ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                    ReadDefault,
+                    // RC:$src2
+                    ReadAfterLd]>;
 }
 
 let Predicates = [HasBMI], Defs = [EFLAGS] in {
   defm BEXTR32 : bmi_bextr_bzhi<0xF7, "bextr{l}", GR32, i32mem,
-                                int_x86_bmi_bextr_32, loadi32>;
+                                int_x86_bmi_bextr_32, loadi32, WriteBEXTR>;
   defm BEXTR64 : bmi_bextr_bzhi<0xF7, "bextr{q}", GR64, i64mem,
-                                int_x86_bmi_bextr_64, loadi64>, VEX_W;
+                                int_x86_bmi_bextr_64, loadi64, WriteBEXTR>, VEX_W;
 }
 
 let Predicates = [HasBMI2], Defs = [EFLAGS] in {
   defm BZHI32 : bmi_bextr_bzhi<0xF5, "bzhi{l}", GR32, i32mem,
-                               int_x86_bmi_bzhi_32, loadi32>;
+                               int_x86_bmi_bzhi_32, loadi32, WriteBZHI>;
   defm BZHI64 : bmi_bextr_bzhi<0xF5, "bzhi{q}", GR64, i64mem,
-                               int_x86_bmi_bzhi_64, loadi64>, VEX_W;
+                               int_x86_bmi_bzhi_64, loadi64, WriteBZHI>, VEX_W;
 }
 
 def CountTrailingOnes : SDNodeXForm<imm, [{
   // Count the trailing ones in the immediate.
   return getI8Imm(countTrailingOnes(N->getZExtValue()), SDLoc(N));
 }]>;
 
 def BEXTRMaskXForm : SDNodeXForm<imm, [{

llvm/trunk/lib/Target/X86/X86SchedBroadwell.td

 defm : BWWriteResPair<WriteBitScan, [BWPort1], 3>;
 defm : BWWriteResPair<WriteLZCNT,   [BWPort1], 3>;
 defm : BWWriteResPair<WriteTZCNT,   [BWPort1], 3>;
 defm : BWWriteResPair<WritePOPCNT,  [BWPort1], 3>;
 
 // Integer shifts and rotates.
 defm : BWWriteResPair<WriteShift, [BWPort06],  1>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : BWWriteResPair<WriteBEXTR, [BWPort06,BWPort15], 2, [1,1], 2>;
+defm : BWWriteResPair<WriteBZHI, [BWPort15], 1>;
+
 // Loads, stores, and moves, not folded with other operations.
 def : WriteRes<WriteLoad,  [BWPort23]> { let Latency = 5; }
 def : WriteRes<WriteStore, [BWPort237, BWPort4]>;
 def : WriteRes<WriteMove,  [BWPort0156]>;
 
 // Idioms that clear a register, like xorps %xmm0, %xmm0.
 // These can often bypass execution ports completely.
 def : WriteRes<WriteZero,  []>;
def BWWriteResGroup7 : SchedWriteRes<[BWPort15]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[BWWriteResGroup7], (instregex "ANDN(32|64)rr",
                                            "BLSI(32|64)rr",
                                            "BLSMSK(32|64)rr",
                                            "BLSR(32|64)rr",
-                                           "BZHI(32|64)rr",
                                            "LEA(16|32|64)(_32)?r",
                                            "MMX_PABSBrr",
                                            "MMX_PABSDrr",
                                            "MMX_PABSWrr",
                                            "MMX_PADDBirr",
                                            "MMX_PADDDirr",
                                            "MMX_PADDQirr",
                                            "MMX_PADDSBirr",

 }
 def: InstRW<[BWWriteResGroup18], (instregex "SFENCE")>;
 
 def BWWriteResGroup19 : SchedWriteRes<[BWPort06,BWPort15]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[BWWriteResGroup19], (instregex "BEXTR(32|64)rr",
-                                            "BSWAP(16|32|64)r")>;
+def: InstRW<[BWWriteResGroup19], (instregex "BSWAP(16|32|64)r")>;
 
 def BWWriteResGroup20 : SchedWriteRes<[BWPort06,BWPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[BWWriteResGroup20], (instrs CWD)>;
 def: InstRW<[BWWriteResGroup20], (instrs JCXZ, JECXZ, JRCXZ)>;
def BWWriteResGroup64 : SchedWriteRes<[BWPort23,BWPort15]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[BWWriteResGroup64], (instregex "ANDN(32|64)rm",
                                             "BLSI(32|64)rm",
                                             "BLSMSK(32|64)rm",
                                             "BLSR(32|64)rm",
-                                            "BZHI(32|64)rm",
                                             "MMX_PABSBrm",
                                             "MMX_PABSDrm",
                                             "MMX_PABSWrm",
                                             "MMX_PADDBirm",
                                             "MMX_PADDDirm",
                                             "MMX_PADDQirm",
                                             "MMX_PADDSBirm",
                                             "MMX_PADDSWirm",

 def BWWriteResGroup84 : SchedWriteRes<[BWPort6,BWPort23,BWPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[BWWriteResGroup84], (instregex "LRETQ",
                                             "RETQ")>;
 
-def BWWriteResGroup85 : SchedWriteRes<[BWPort23,BWPort06,BWPort15]> {
-  let Latency = 7;
-  let NumMicroOps = 3;
-  let ResourceCycles = [1,1,1];
-}
-def: InstRW<[BWWriteResGroup85], (instregex "BEXTR(32|64)rm")>;
-
 def BWWriteResGroup86 : SchedWriteRes<[BWPort23,BWPort06,BWPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[BWWriteResGroup86], (instregex "CMOV(A|BE)(16|32|64)rm")>;
 
 def BWWriteResGroup87 : SchedWriteRes<[BWPort4,BWPort23,BWPort237,BWPort06]> {

llvm/trunk/lib/Target/X86/X86SchedHaswell.td

 def : WriteRes<WriteLEA, [HWPort15]>;
 
 // Bit counts.
 defm : HWWriteResPair<WriteBitScan, [HWPort1], 3>;
 defm : HWWriteResPair<WriteLZCNT,   [HWPort1], 3>;
 defm : HWWriteResPair<WriteTZCNT,   [HWPort1], 3>;
 defm : HWWriteResPair<WritePOPCNT,  [HWPort1], 3>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : HWWriteResPair<WriteBEXTR, [HWPort06,HWPort15], 2, [1,1], 2>;
+defm : HWWriteResPair<WriteBZHI, [HWPort15], 1>;
+
 // This is quite rough, latency depends on the dividend.
 defm : HWWriteResPair<WriteIDiv,  [HWPort0, HWDivider], 25, [1,10], 1, 4>;
 // Scalar and vector floating point.
 def  : WriteRes<WriteFStore,       [HWPort237, HWPort4]>;
 def  : WriteRes<WriteFLoad,        [HWPort23]> { let Latency = 5; }
 def  : WriteRes<WriteFMove,        [HWPort5]>;
 
 defm : HWWriteResPair<WriteFAdd,   [HWPort1], 3>;
def HWWriteResGroup8 : SchedWriteRes<[HWPort15]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[HWWriteResGroup8], (instregex "ANDN(32|64)rr",
                                            "BLSI(32|64)rr",
                                            "BLSMSK(32|64)rr",
                                            "BLSR(32|64)rr",
-                                           "BZHI(32|64)rr",
                                            "LEA(16|32|64)(_32)?r",
                                            "MMX_PABSBrr",
                                            "MMX_PABSDrr",
                                            "MMX_PABSWrr",
                                            "MMX_PADDBirr",
                                            "MMX_PADDDirr",
                                            "MMX_PADDQirr",
                                            "MMX_PADDSBirr",
def HWWriteResGroup16 : SchedWriteRes<[HWPort23,HWPort15]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[HWWriteResGroup16], (instregex "ANDN(32|64)rm",
                                             "BLSI(32|64)rm",
                                             "BLSMSK(32|64)rm",
                                             "BLSR(32|64)rm",
-                                            "BZHI(32|64)rm",
                                             "MMX_PABSBrm",
                                             "MMX_PABSDrm",
                                             "MMX_PABSWrm",
                                             "MMX_PADDBirm",
                                             "MMX_PADDDirm",
                                             "MMX_PADDQirm",
                                             "MMX_PADDSBirm",
                                             "MMX_PADDSWirm",

 }
 def: InstRW<[HWWriteResGroup33], (instregex "MMX_MOVDQ2Qrr")>;
 
 def HWWriteResGroup34 : SchedWriteRes<[HWPort06,HWPort15]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[HWWriteResGroup34], (instregex "BEXTR(32|64)rr",
-                                            "BSWAP(16|32|64)r")>;
+def: InstRW<[HWWriteResGroup34], (instregex "BSWAP(16|32|64)r")>;
 
 def HWWriteResGroup35 : SchedWriteRes<[HWPort06,HWPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[HWWriteResGroup35], (instrs CWD, JCXZ, JECXZ, JRCXZ)>;
 def: InstRW<[HWWriteResGroup35], (instregex "ADC(8|16|32|64)ri",
def HWWriteResGroup41 : SchedWriteRes<[HWPort6,HWPort23,HWPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[HWWriteResGroup41], (instregex "LRETQ",
                                             "RETL",
                                             "RETQ")>;
 
-def HWWriteResGroup42 : SchedWriteRes<[HWPort23,HWPort06,HWPort15]> {
-  let Latency = 7;
-  let NumMicroOps = 3;
-  let ResourceCycles = [1,1,1];
-}
-def: InstRW<[HWWriteResGroup42], (instregex "BEXTR(32|64)rm")>;
-
 def HWWriteResGroup43 : SchedWriteRes<[HWPort23,HWPort06,HWPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[HWWriteResGroup43], (instregex "ADC(8|16|32|64)rm")>;
 def: InstRW<[HWWriteResGroup43], (instregex "CMOV(AE|B|E|G|GE|L|LE|NE|NO|NP|NS|O|P|S)(16|32|64)rm")>;
 def: InstRW<[HWWriteResGroup43], (instregex "SBB(8|16|32|64)rm")>;

llvm/trunk/lib/Target/X86/X86SchedSandyBridge.td

 def : WriteRes<WriteLEA, [SBPort15]>;
 
 // Bit counts.
 defm : SBWriteResPair<WriteBitScan, [SBPort1], 3, [1], 1, 5>;
 defm : SBWriteResPair<WriteLZCNT,   [SBPort1], 3, [1], 1, 5>;
 defm : SBWriteResPair<WriteTZCNT,   [SBPort1], 3, [1], 1, 5>;
 defm : SBWriteResPair<WritePOPCNT,  [SBPort1], 3, [1], 1, 5>;
 
+// BMI1 BEXTR, BMI2 BZHI
+// NOTE: These don't exist on Sandy Bridge. Ports are guesses.
+defm : SBWriteResPair<WriteBEXTR, [SBPort05,SBPort1], 2, [1,1], 2>;
+defm : SBWriteResPair<WriteBZHI, [SBPort1], 1>;
+
 // Scalar and vector floating point.
 def  : WriteRes<WriteFStore,       [SBPort23, SBPort4]>;
 def  : WriteRes<WriteFLoad,        [SBPort23]> { let Latency = 6; }
 def  : WriteRes<WriteFMove,        [SBPort5]>;
 
 defm : SBWriteResPair<WriteFAdd,   [SBPort1], 3>;
 defm : SBWriteResPair<WriteFMul,   [SBPort0], 5>;
 defm : SBWriteResPair<WriteFDiv,   [SBPort0], 24>;

llvm/trunk/lib/Target/X86/X86SchedSkylakeClient.td

 defm : SKLWriteResPair<WriteBitScan, [SKLPort1], 3>;
 defm : SKLWriteResPair<WriteLZCNT,   [SKLPort1], 3>;
 defm : SKLWriteResPair<WriteTZCNT,   [SKLPort1], 3>;
 defm : SKLWriteResPair<WritePOPCNT,  [SKLPort1], 3>;
 
 // Integer shifts and rotates.
 defm : SKLWriteResPair<WriteShift, [SKLPort06],  1>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : SKLWriteResPair<WriteBEXTR, [SKLPort06,SKLPort15], 2, [1,1], 2>;
+defm : SKLWriteResPair<WriteBZHI, [SKLPort15], 1>;
+
 // Loads, stores, and moves, not folded with other operations.
 def : WriteRes<WriteLoad,  [SKLPort23]> { let Latency = 5; }
 def : WriteRes<WriteStore, [SKLPort237, SKLPort4]>;
 def : WriteRes<WriteMove,  [SKLPort0156]>;
 
 // Idioms that clear a register, like xorps %xmm0, %xmm0.
 // These can often bypass execution ports completely.
 def : WriteRes<WriteZero,  []>;
def SKLWriteResGroup8 : SchedWriteRes<[SKLPort15]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[SKLWriteResGroup8], (instregex "ANDN(32|64)rr",
                                             "BLSI(32|64)rr",
                                             "BLSMSK(32|64)rr",
                                             "BLSR(32|64)rr",
-                                            "BZHI(32|64)rr",
                                             "LEA(16|32|64)(_32)?r")>;
 
 def SKLWriteResGroup9 : SchedWriteRes<[SKLPort015]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[SKLWriteResGroup9], (instregex "(V?)ANDNPD(Y?)rr",

 }
 def: InstRW<[SKLWriteResGroup21], (instregex "SFENCE")>;
 
 def SKLWriteResGroup22 : SchedWriteRes<[SKLPort06,SKLPort15]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup22], (instregex "BEXTR(32|64)rr",
-                                             "BSWAP(16|32|64)r")>;
+def: InstRW<[SKLWriteResGroup22], (instregex "BSWAP(16|32|64)r")>;
 
 def SKLWriteResGroup23 : SchedWriteRes<[SKLPort06,SKLPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKLWriteResGroup23], (instrs CWD)>;
 def: InstRW<[SKLWriteResGroup23], (instrs JCXZ, JECXZ, JRCXZ)>;
def SKLWriteResGroup75 : SchedWriteRes<[SKLPort23,SKLPort15]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKLWriteResGroup75], (instregex "ANDN(32|64)rm",
                                              "BLSI(32|64)rm",
                                              "BLSMSK(32|64)rm",
                                              "BLSR(32|64)rm",
-                                             "BZHI(32|64)rm",
                                              "MOVBE(16|32|64)rm")>;
 
 def SKLWriteResGroup76 : SchedWriteRes<[SKLPort23,SKLPort0156]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKLWriteResGroup76], (instrs POP16r, POP32r, POP64r)>;

 def SKLWriteResGroup98 : SchedWriteRes<[SKLPort6,SKLPort23,SKLPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[SKLWriteResGroup98], (instregex "LRETQ",
                                              "RETQ")>;
 
-def SKLWriteResGroup99 : SchedWriteRes<[SKLPort23,SKLPort06,SKLPort15]> {
-  let Latency = 7;
-  let NumMicroOps = 3;
-  let ResourceCycles = [1,1,1];
-}
-def: InstRW<[SKLWriteResGroup99], (instregex "BEXTR(32|64)rm")>;
-
 def SKLWriteResGroup100 : SchedWriteRes<[SKLPort4,SKLPort23,SKLPort237,SKLPort06]> {
   let Latency = 7;
   let NumMicroOps = 5;
   let ResourceCycles = [1,1,1,2];
 }
 def: InstRW<[SKLWriteResGroup100], (instregex "ROL(8|16|32|64)m1",
                                               "ROL(8|16|32|64)mi",
                                               "ROR(8|16|32|64)m1",

llvm/trunk/lib/Target/X86/X86SchedSkylakeServer.td

 defm : SKXWriteResPair<WriteShift, [SKXPort06],  1>;
 
 // Bit counts.
 defm : SKXWriteResPair<WriteBitScan, [SKXPort1], 3>;
 defm : SKXWriteResPair<WriteLZCNT,   [SKXPort1], 3>;
 defm : SKXWriteResPair<WriteTZCNT,   [SKXPort1], 3>;
 defm : SKXWriteResPair<WritePOPCNT,  [SKXPort1], 3>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : SKXWriteResPair<WriteBEXTR, [SKXPort06,SKXPort15], 2, [1,1], 2>;
+defm : SKXWriteResPair<WriteBZHI, [SKXPort15], 1>;
+
 // Loads, stores, and moves, not folded with other operations.
 def : WriteRes<WriteLoad,  [SKXPort23]> { let Latency = 5; }
 def : WriteRes<WriteStore, [SKXPort237, SKXPort4]>;
 def : WriteRes<WriteMove,  [SKXPort0156]>;
 
 // Idioms that clear a register, like xorps %xmm0, %xmm0.
 // These can often bypass execution ports completely.
 def : WriteRes<WriteZero,  []>;
def SKXWriteResGroup8 : SchedWriteRes<[SKXPort15]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[SKXWriteResGroup8], (instregex "ANDN(32|64)rr",
                                             "BLSI(32|64)rr",
                                             "BLSMSK(32|64)rr",
                                             "BLSR(32|64)rr",
-                                            "BZHI(32|64)rr",
                                             "LEA(16|32|64)(_32)?r")>;
 
 def SKXWriteResGroup9 : SchedWriteRes<[SKXPort015]> {
   let Latency = 1;
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
 def: InstRW<[SKXWriteResGroup9], (instregex "ANDNPDrr",

 }
 def: InstRW<[SKXWriteResGroup21], (instregex "SFENCE")>;
 
 def SKXWriteResGroup22 : SchedWriteRes<[SKXPort06,SKXPort15]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKXWriteResGroup22], (instregex "BEXTR(32|64)rr",
-                                             "BSWAP(16|32|64)r")>;
+def: InstRW<[SKXWriteResGroup22], (instregex "BSWAP(16|32|64)r")>;
 
 def SKXWriteResGroup23 : SchedWriteRes<[SKXPort06,SKXPort0156]> {
   let Latency = 2;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKXWriteResGroup23], (instrs CWD)>;
 def: InstRW<[SKXWriteResGroup23], (instrs JCXZ, JECXZ, JRCXZ)>;
def SKXWriteResGroup79 : SchedWriteRes<[SKXPort23,SKXPort15]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKXWriteResGroup79], (instregex "ANDN(32|64)rm",
                                              "BLSI(32|64)rm",
                                              "BLSMSK(32|64)rm",
                                              "BLSR(32|64)rm",
-                                             "BZHI(32|64)rm",
                                              "MOVBE(16|32|64)rm")>;
 
 def SKXWriteResGroup80 : SchedWriteRes<[SKXPort23,SKXPort015]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
 def: InstRW<[SKXWriteResGroup80], (instregex "VMOV(64to|QI2)PQIZrm(b?)",

 def SKXWriteResGroup104 : SchedWriteRes<[SKXPort6,SKXPort23,SKXPort0156]> {
   let Latency = 7;
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
 def: InstRW<[SKXWriteResGroup104], (instregex "LRETQ",
                                               "RETQ")>;
 
-def SKXWriteResGroup105 : SchedWriteRes<[SKXPort23,SKXPort06,SKXPort15]> {
-  let Latency = 7;
-  let NumMicroOps = 3;
-  let ResourceCycles = [1,1,1];
-}
-def: InstRW<[SKXWriteResGroup105], (instregex "BEXTR(32|64)rm")>;
-
 def SKXWriteResGroup106 : SchedWriteRes<[SKXPort4,SKXPort5,SKXPort237]> {
   let Latency = 7;
   let NumMicroOps = 4;
   let ResourceCycles = [1,2,1];
 }
 def: InstRW<[SKXWriteResGroup106], (instregex "VCOMPRESSPDZ128mr(b?)",
                                               "VCOMPRESSPDZ256mr(b?)",
                                               "VCOMPRESSPDZmr(b?)",

llvm/trunk/lib/Target/X86/X86Schedule.td

 defm WriteBitScan : X86SchedWritePair; // Bit scan forward/reverse.
 defm WritePOPCNT : X86SchedWritePair; // Bit population count.
 defm WriteLZCNT : X86SchedWritePair; // Leading zero count.
 defm WriteTZCNT : X86SchedWritePair; // Trailing zero count.
 
 // Integer shifts and rotates.
 defm WriteShift : X86SchedWritePair;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm WriteBEXTR : X86SchedWritePair;
+defm WriteBZHI  : X86SchedWritePair;
+
 // Loads, stores, and moves, not folded with other operations.
 def WriteLoad  : SchedWrite;
 def WriteStore : SchedWrite;
 def WriteMove  : SchedWrite;
 
 // Idioms that clear a register, like xorps %xmm0, %xmm0.
 // These can often bypass execution ports completely.
 def WriteZero : SchedWrite;

llvm/trunk/lib/Target/X86/X86ScheduleBtVer2.td

 def : WriteRes<WriteLEA, [JALU01]>;
 
 // Bit counts.
 defm : JWriteResIntPair<WriteBitScan, [JALU01], 5, [4], 8>;
 defm : JWriteResIntPair<WritePOPCNT,  [JALU01], 1>;
 defm : JWriteResIntPair<WriteLZCNT,   [JALU01], 1>;
 defm : JWriteResIntPair<WriteTZCNT,   [JALU01], 2, [2]>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : JWriteResIntPair<WriteBEXTR, [JALU01], 1>;
+defm : JWriteResIntPair<WriteBZHI, [JALU01], 1>; // NOTE: Doesn't exist on Jaguar.
+
 def JWriteIMul64 : SchedWriteRes<[JALU1, JMul]> {
   let Latency = 6;
   let ResourceCycles = [1, 4];
   let NumMicroOps = 2;
 }
 def JWriteIMul64Ld : SchedWriteRes<[JLAGU, JALU1, JMul]> {
   let Latency = 9;
   let ResourceCycles = [1, 1, 4];

llvm/trunk/lib/Target/X86/X86ScheduleSLM.td

 def : WriteRes<WriteLEA, [SLM_IEC_RSV1]>;
 
 // Bit counts.
 defm : SLMWriteResPair<WriteBitScan, [SLM_IEC_RSV01], 10, [20], 10>;
 defm : SLMWriteResPair<WriteLZCNT,   [SLM_IEC_RSV0], 3>;
 defm : SLMWriteResPair<WriteTZCNT,   [SLM_IEC_RSV0], 3>;
 defm : SLMWriteResPair<WritePOPCNT,  [SLM_IEC_RSV0], 3>;
 
+// BMI1 BEXTR, BMI2 BZHI
+// NOTE: These don't exist on Silvermont. Ports are guesses.
+defm : SBWriteResPair<WriteBEXTR, [SLM_IEC_RSV0], 1>;
+defm : SBWriteResPair<WriteBZHI, [SLM_IEC_RSV0], 1>;
+
 // This is quite rough, latency depends on the dividend.
 defm : SLMWriteResPair<WriteIDiv, [SLM_IEC_RSV01, SLMDivider], 25, [1,25], 1, 4>;
 
 // Scalar and vector floating point.
 def  : WriteRes<WriteFStore,       [SLM_FPC_RSV01, SLM_MEC_RSV]>;
 def  : WriteRes<WriteFLoad,        [SLM_MEC_RSV]> { let Latency = 3; }
 def  : WriteRes<WriteFMove,        [SLM_FPC_RSV01]>;
 

llvm/trunk/lib/Target/X86/X86ScheduleZnver1.td

 defm : ZnWriteResPair<WriteBitScan, [ZnALU], 3>;
 defm : ZnWriteResPair<WriteLZCNT,   [ZnALU], 2>;
 defm : ZnWriteResPair<WriteTZCNT,   [ZnALU], 2>;
 defm : ZnWriteResPair<WritePOPCNT,  [ZnALU], 1>;
 
 // Treat misc copies as a move.
 def : InstRW<[WriteMove], (instrs COPY)>;
 
+// BMI1 BEXTR, BMI2 BZHI
+defm : ZnWriteResPair<WriteBEXTR, [ZnALU], 1>;
+defm : ZnWriteResPair<WriteBZHI, [ZnALU], 1>;
+
 // IDIV
 def : WriteRes<WriteIDiv, [ZnALU2, ZnDivider]> {
   let Latency = 41;
   let ResourceCycles = [1, 41];
 }
 
 def : WriteRes<WriteIDivLd, [ZnALU2, ZnAGU, ZnDivider]> {
   let Latency = 45;

 def : InstRW<[ZnWriteBTRSCm], (instregex "BT(R|S|C)(16|32|64)m(r|i8)")>;
 
 // BLSI BLSMSK BLSR.
 // r,r.
 def : InstRW<[ZnWriteALULat2], (instregex "BLS(I|MSK|R)(32|64)rr")>;
 // r,m.
 def : InstRW<[ZnWriteALULat2Ld, ReadAfterLd], (instregex "BLS(I|MSK|R)(32|64)rm")>;
 
-// BEXTR.
-// r,r,r.
-def : InstRW<[WriteALU], (instregex "BEXTR(32|64)rr")>;
-// r,m,r.
-def : InstRW<[WriteALULd, ReadAfterLd], (instregex "BEXTR(32|64)rm")>;
-
-// BZHI.
-// r,r,r.
-def : InstRW<[WriteALU], (instregex "BZHI(32|64)rr")>;
-// r,m,r.
-def : InstRW<[WriteALULd, ReadAfterLd], (instregex "BZHI(32|64)rm")>;
-
 // CLD STD.
 def : InstRW<[WriteALU], (instregex "STD", "CLD")>;
 
 // PDEP PEXT.
 // r,r,r.
 def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rr", "PEXT(32|64)rr")>;
-// r,m,r.
+// r,r,m.
 def : InstRW<[WriteMicrocoded], (instregex "PDEP(32|64)rm", "PEXT(32|64)rm")>;
 
 // ROR ROL.
 def : InstRW<[WriteShift], (instregex "RO(R|L)(8|16|32|64)r1")>;
 
 // RCR RCL.
 // r,1.
 def : InstRW<[WriteShift], (instregex "RC(R|L)(8|16|32|64)r1")>;

llvm/trunk/test/CodeGen/X86/bmi-schedule.ll

; ZNVER1-NEXT:    retq # sched: [1:0.50]
   %4 = and i64 %2, %1
   %5 = add i64 %3, %4
   ret i64 %5
 }
 
 define i32 @test_bextr_i32(i32 %a0, i32 %a1, i32 *%a2) {
 ; GENERIC-LABEL: test_bextr_i32:
 ; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [5:0.50]
-; GENERIC-NEXT:    bextrl %edi, %esi, %eax # sched: [1:0.33]
+; GENERIC-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [6:1.00]
+; GENERIC-NEXT:    bextrl %edi, %esi, %eax # sched: [2:1.00]
 ; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
 ; GENERIC-NEXT:    retq # sched: [1:1.00]
 ;
 ; HASWELL-LABEL: test_bextr_i32:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [7:0.50]
 ; HASWELL-NEXT:    bextrl %edi, %esi, %eax # sched: [2:0.50]
 ; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
; ZNVER1-NEXT:    retq # sched: [1:0.50]
   %4 = add i32 %2, %3
   ret i32 %4
 }
 declare i32 @llvm.x86.bmi.bextr.32(i32, i32)
 
 define i64 @test_bextr_i64(i64 %a0, i64 %a1, i64 *%a2) {
 ; GENERIC-LABEL: test_bextr_i64:
 ; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [5:0.50]
-; GENERIC-NEXT:    bextrq %rdi, %rsi, %rax # sched: [1:0.33]
+; GENERIC-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [6:1.00]
+; GENERIC-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:1.00]
 ; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
 ; GENERIC-NEXT:    retq # sched: [1:1.00]
 ;
 ; HASWELL-LABEL: test_bextr_i64:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [7:0.50]
 ; HASWELL-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:0.50]
 ; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]

llvm/trunk/test/CodeGen/X86/bmi2-schedule.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+bmi2 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl     | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1  | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
 
 define i32 @test_bzhi_i32(i32 %a0, i32 %a1, i32 *%a2) {
 ; GENERIC-LABEL: test_bzhi_i32:
 ; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [5:0.50]
-; GENERIC-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.33]
+; GENERIC-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [5:1.00]
+; GENERIC-NEXT:    bzhil %edi, %esi, %eax # sched: [1:1.00]
 ; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
 ; GENERIC-NEXT:    retq # sched: [1:1.00]
 ;
 ; HASWELL-LABEL: test_bzhi_i32:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:0.50]
 ; HASWELL-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.50]
 ; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
; ZNVER1-NEXT:    retq # sched: [1:0.50]
   %4 = add i32 %2, %3
   ret i32 %4
 }
 declare i32 @llvm.x86.bmi.bzhi.32(i32, i32)
 
 define i64 @test_bzhi_i64(i64 %a0, i64 %a1, i64 *%a2) {
 ; GENERIC-LABEL: test_bzhi_i64:
 ; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [5:0.50]
-; GENERIC-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.33]
+; GENERIC-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [5:1.00]
+; GENERIC-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:1.00]
 ; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
 ; GENERIC-NEXT:    retq # sched: [1:1.00]
 ;
 ; HASWELL-LABEL: test_bzhi_i64:
 ; HASWELL:       # %bb.0:
 ; HASWELL-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:0.50]
 ; HASWELL-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.50]
 ; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]