[X86] Add ReadAfterLds to some 3 src instructions

[craig.topper]

Summary

I think we're supposed to have a ReadAfterLd for each operand that's read after load?

Sometimes the operand comes after the memory operand so we need 5 ReadDefaults first.

I suspect we also need to do something for the mask operand for masked avx512 instructions? I'm not sure if the mask should be ReadAfterLd or not since it can mask faults. If it shouldn't be ReadAfterLd then we're probably wrong for zero masking instructions already.

[craig.topper]

Ping

[craig.topper]

@andreadb does this one look ok? Ignoring that the InstRWs in the scheduler models drop the ReadAdvances anyway.

[andreadb]

It looks good to me.

Unfortunately I cannot test it now (I am not at work..). Would it be possible to add a couple of simple llvm-mca tests (maybe one for a 3 operands FMA and one for a FMA4 instruction)?
Something that runs for a single iteration and prints the timeline view would be nice (you can commit those tests later if you prefer).

Thanks,
Andrea

[andreadb]

In D44726#1052328, @craig.topper wrote:

@andreadb does this one look ok? Ignoring that the InstRWs in the scheduler models drop the ReadAdvances anyway.

Ah, forget about the tests then...
I mean, you can still add them to show the problem.

lib/Target/X86/X86InstrFMA.td

multiclass fma3p_rm_213<bits<8> opc, string OpcodeStr, RegisterClass RC,
 
   let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
                    (ins RC:$src1, RC:$src2, x86memop:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op RC:$src2, RC:$src1,
                                           (MemFrag addr:$src3))))]>,
-                   Sched<[WriteFMALd, ReadAfterLd]>;
+                   Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 multiclass fma3p_rm_231<bits<8> opc, string OpcodeStr, RegisterClass RC,
                         ValueType VT, X86MemOperand x86memop, PatFrag MemFrag,
                         SDNode Op> {
   let hasSideEffects = 0 in
   def r     : FMA3<opc, MRMSrcReg, (outs RC:$dst),
                    (ins RC:$src1, RC:$src2, RC:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
                    (ins RC:$src1, RC:$src2, x86memop:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op RC:$src2, (MemFrag addr:$src3),
-                                          RC:$src1)))]>, Sched<[WriteFMALd, ReadAfterLd]>;
+                                          RC:$src1)))]>,
+                   Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 multiclass fma3p_rm_132<bits<8> opc, string OpcodeStr, RegisterClass RC,
                         ValueType VT, X86MemOperand x86memop, PatFrag MemFrag,
                         SDNode Op> {
   let hasSideEffects = 0 in
   def r     : FMA3<opc, MRMSrcReg, (outs RC:$dst),
                    (ins RC:$src1, RC:$src2, RC:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    []>, Sched<[WriteFMA]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
   let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
                    (ins RC:$src1, RC:$src2, x86memop:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set RC:$dst, (VT (Op (MemFrag addr:$src3), RC:$src1,
-                                          RC:$src2)))]>, Sched<[WriteFMALd, ReadAfterLd]>;
+                                          RC:$src2)))]>,
+                   Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 let Constraints = "$src1 = $dst", hasSideEffects = 0, isCommutable = 1 in
 multiclass fma3p_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
                        string OpcodeStr, string PackTy, string Suff,
                        PatFrag MemFrag128, PatFrag MemFrag256,
                        SDNode Op, ValueType OpTy128, ValueType OpTy256> {
   defm NAME#213#Suff : fma3p_rm_213<opc213, !strconcat(OpcodeStr, "213", PackTy),
multiclass fma3s_rm_213<bits<8> opc, string OpcodeStr,
 
   let mayLoad = 1 in
   def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
                 (ins RC:$src1, RC:$src2, x86memop:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
                   (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>,
-                Sched<[WriteFMALd, ReadAfterLd]>;
+                Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 multiclass fma3s_rm_231<bits<8> opc, string OpcodeStr,
                         X86MemOperand x86memop, RegisterClass RC,
                         SDPatternOperator OpNode> {
   let hasSideEffects = 0 in
   def r : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
                 (ins RC:$src1, RC:$src2, RC:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
                 (ins RC:$src1, RC:$src2, x86memop:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
                   (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>,
-                Sched<[WriteFMALd, ReadAfterLd]>;
+                Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 multiclass fma3s_rm_132<bits<8> opc, string OpcodeStr,
                         X86MemOperand x86memop, RegisterClass RC,
                         SDPatternOperator OpNode> {
   let hasSideEffects = 0 in
   def r : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
                 (ins RC:$src1, RC:$src2, RC:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 []>, Sched<[WriteFMA]>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
   let mayLoad = 1 in
   def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
                 (ins RC:$src1, RC:$src2, x86memop:$src3),
                 !strconcat(OpcodeStr,
                            "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                 [(set RC:$dst,
                   (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>,
-                Sched<[WriteFMALd, ReadAfterLd]>;
+                Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 let Constraints = "$src1 = $dst", isCommutable = 1, hasSideEffects = 0 in
 multiclass fma3s_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
                        string OpStr, string PackTy, string Suff,
                        SDNode OpNode, RegisterClass RC,
                        X86MemOperand x86memop> {
   defm NAME#213#Suff : fma3s_rm_213<opc213, !strconcat(OpStr, "213", PackTy),
def r_Int : FMA3S_Int<opc, MRMSrcReg, (outs RC:$dst),
                                    "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                         []>, Sched<[WriteFMA]>;
 
   let mayLoad = 1 in
   def m_Int : FMA3S_Int<opc, MRMSrcMem, (outs RC:$dst),
                         (ins RC:$src1, RC:$src2, memopr:$src3),
                         !strconcat(OpcodeStr,
                                    "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                        []>, Sched<[WriteFMALd, ReadAfterLd]>;
+                        []>, Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
 }
 
 // The FMA 213 form is created for lowering of scalar FMA intrinscis
 // to machine instructions.
 // The FMA 132 form can trivially be get by commuting the 2nd and 3rd operands
 // of FMA 213 form.
 // The FMA 231 form can be get only by commuting the 1st operand of 213 or 132
 // forms and is possible only after special analysis of all uses of the initial
def rr : FMA4S<opc, MRMSrcRegOp4, (outs RC:$dst),
              (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, VEX_W, VEX_LIG,
            Sched<[WriteFMA]>;
   def rm : FMA4S<opc, MRMSrcMemOp4, (outs RC:$dst),
            (ins RC:$src1, RC:$src2, x86memop:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst, (OpNode RC:$src1, RC:$src2,
                            (mem_frag addr:$src3)))]>, VEX_W, VEX_LIG,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
   def mr : FMA4S<opc, MRMSrcMem, (outs RC:$dst),
            (ins RC:$src1, x86memop:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst,
              (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>, VEX_LIG,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd,
+                  // x86memop:$src2
+                  ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                  ReadDefault,
+                  // RC:$src3
+                  ReadAfterLd]>;
 // For disassembler
 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def rr_REV : FMA4S<opc, MRMSrcReg, (outs RC:$dst),
                (ins RC:$src1, RC:$src2, RC:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
                VEX_LIG, FoldGenData<NAME#rr>, Sched<[WriteFMA]>;
 }
def rr_Int : FMA4S_Int<opc, MRMSrcRegOp4, (outs VR128:$dst),
                  (VT (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>, VEX_W,
                VEX_LIG, Sched<[WriteFMA]>;
   def rm_Int : FMA4S_Int<opc, MRMSrcMemOp4, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, memop:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst, (VT (OpNode VR128:$src1, VR128:$src2,
                                   mem_cpat:$src3)))]>, VEX_W, VEX_LIG,
-               Sched<[WriteFMALd, ReadAfterLd]>;
+               Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
   def mr_Int : FMA4S_Int<opc, MRMSrcMem, (outs VR128:$dst),
                (ins VR128:$src1, memop:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst,
                  (VT (OpNode VR128:$src1, mem_cpat:$src2, VR128:$src3)))]>,
-               VEX_LIG, Sched<[WriteFMALd, ReadAfterLd]>;
+               VEX_LIG, Sched<[WriteFMALd, ReadAfterLd,
+                               // memop:$src2
+                               ReadDefault, ReadDefault, ReadDefault,
+                               ReadDefault, ReadDefault,
+                               // VR128::$src3
+                               ReadAfterLd]>;
 let hasSideEffects = 0 in
   def rr_Int_REV : FMA4S_Int<opc, MRMSrcReg, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                []>, VEX_LIG, FoldGenData<NAME#rr_Int>, Sched<[WriteFMA]>;
 } // isCodeGenOnly = 1
 }
def rr : FMA4<opc, MRMSrcRegOp4, (outs VR128:$dst),
              (OpVT128 (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>,
            VEX_W, Sched<[WriteFMA]>;
   def rm : FMA4<opc, MRMSrcMemOp4, (outs VR128:$dst),
            (ins VR128:$src1, VR128:$src2, f128mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst, (OpNode VR128:$src1, VR128:$src2,
                               (ld_frag128 addr:$src3)))]>, VEX_W,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
   def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
            (ins VR128:$src1, f128mem:$src2, VR128:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst,
              (OpNode VR128:$src1, (ld_frag128 addr:$src2), VR128:$src3))]>,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd,
+                  // f128mem:$src2
+                  ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                  ReadDefault,
+                  // VR128::$src3
+                  ReadAfterLd]>;
   let isCommutable = 1 in
   def Yrr : FMA4<opc, MRMSrcRegOp4, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst,
              (OpVT256 (OpNode VR256:$src1, VR256:$src2, VR256:$src3)))]>,
            VEX_W, VEX_L, Sched<[WriteFMA]>;
   def Yrm : FMA4<opc, MRMSrcMemOp4, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, f256mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst, (OpNode VR256:$src1, VR256:$src2,
                               (ld_frag256 addr:$src3)))]>, VEX_W, VEX_L,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>;
   def Ymr : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, f256mem:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst, (OpNode VR256:$src1,
                               (ld_frag256 addr:$src2), VR256:$src3))]>, VEX_L,
-           Sched<[WriteFMALd, ReadAfterLd]>;
+           Sched<[WriteFMALd, ReadAfterLd,
+                  // f256mem:$src2
+                  ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                  ReadDefault,
+                  // VR256::$src3
+                  ReadAfterLd]>;
 // For disassembler
 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in {
   def rr_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
                Sched<[WriteFMA]>, FoldGenData<NAME#rr>;
   def Yrr_REV : FMA4<opc, MRMSrcReg, (outs VR256:$dst),

lib/Target/X86/X86InstrSSE.td

multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr,
   def rm : Ii8Reg<opc, MRMSrcMem, (outs RC:$dst),
                   (ins RC:$src1, x86memop:$src2, RC:$src3),
                   !strconcat(OpcodeStr,
                     "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                   [(set RC:$dst,
                         (IntId RC:$src1, (bitconvert (mem_frag addr:$src2)),
                                RC:$src3))],
                   NoItinerary, SSEPackedInt>, TAPD, VEX_4V,
-                Sched<[Sched.Folded, ReadAfterLd]>;
+                Sched<[Sched.Folded, ReadAfterLd,
+                       // x86memop:$src2
+                       ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                       ReadDefault,
+                       // RC::$src3
+                       ReadAfterLd]>;
 }
 
 let Predicates = [HasAVX] in {
 let ExeDomain = SSEPackedDouble in {
 defm VBLENDVPD  : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem,
                                            loadv2f64, int_x86_sse41_blendvpd,
                                            WriteFVarBlend>;
 defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem,

lib/Target/X86/X86InstrShiftRotate.td

let hasSideEffects = 0 in {
   let mayLoad = 1 in
   def rm : I<0xF7, MRMSrcMem4VOp3,
              (outs RC:$dst), (ins x86memop:$src1, RC:$src2),
              !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>,
              VEX, Sched<[WriteShiftLd,
                          // x86memop:$src1
                          ReadDefault, ReadDefault, ReadDefault, ReadDefault,
                          ReadDefault,
-                         // RC:$src1
+                         // RC:$src2
                          ReadAfterLd]>;
 }
 }
 
 let Predicates = [HasBMI2] in {
   defm RORX32 : bmi_rotate<"rorx{l}", GR32, i32mem>;
   defm RORX64 : bmi_rotate<"rorx{q}", GR64, i64mem>, VEX_W;
   defm SARX32 : bmi_shift<"sarx{l}", GR32, i32mem>, T8XS;

lib/Target/X86/X86InstrXOP.td

def rrr : IXOPi8Reg<opc, MRMSrcReg, (outs VR128:$dst),
             XOP_4V, Sched<[WriteShuffle]>;
   def rrm : IXOPi8Reg<opc, MRMSrcMemOp4, (outs VR128:$dst),
             (ins VR128:$src1, VR128:$src2, i128mem:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
             [(set VR128:$dst,
               (vt128 (OpNode (vt128 VR128:$src1), (vt128 VR128:$src2),
                              (vt128 (bitconvert (loadv2i64 addr:$src3))))))]>,
-            XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd]>;
+            XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd, ReadAfterLd]>;
   def rmr : IXOPi8Reg<opc, MRMSrcMem, (outs VR128:$dst),
             (ins VR128:$src1, i128mem:$src2, VR128:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
             [(set VR128:$dst,
               (v16i8 (OpNode (vt128 VR128:$src1), (vt128 (bitconvert (loadv2i64 addr:$src2))),
                              (vt128 VR128:$src3))))]>,
-            XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd]>;
+            XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd,
+                           // 128mem:$src2
+                           ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                           ReadDefault,
+                           // VR128:$src3
+                           ReadAfterLd]>;
   // For disassembler
   let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def rrr_REV : IXOPi8Reg<opc, MRMSrcRegOp4, (outs VR128:$dst),
                 (ins VR128:$src1, VR128:$src2, VR128:$src3),
                 !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                 []>, XOP_4V, VEX_W, Sched<[WriteShuffle]>, FoldGenData<NAME#rrr>;
 }
def rrr : IXOPi8Reg<opc, MRMSrcReg, (outs RC:$dst),
                                    (X86andnp RC:$src3, RC:$src2))))]>, XOP_4V,
             Sched<[WriteShuffle]>;
   def rrm : IXOPi8Reg<opc, MRMSrcMemOp4, (outs RC:$dst),
             (ins RC:$src1, RC:$src2, x86memop:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
             [(set RC:$dst, (VT (or (and (load addr:$src3), RC:$src1),
                                    (X86andnp (load addr:$src3), RC:$src2))))]>,
-            XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd]>;
+            XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd, ReadAfterLd]>;
   def rmr : IXOPi8Reg<opc, MRMSrcMem, (outs RC:$dst),
             (ins RC:$src1, x86memop:$src2, RC:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
             [(set RC:$dst, (VT (or (and RC:$src3, RC:$src1),
                                    (X86andnp RC:$src3, (load addr:$src2)))))]>,
-            XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd]>;
+            XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd,
+                           // x86memop:$src2
+                           ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+                           ReadDefault,
+                           // RC::$src3
+                           ReadAfterLd]>;
   // For disassembler
   let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def rrr_REV : IXOPi8Reg<opc, MRMSrcRegOp4, (outs RC:$dst),
             (ins RC:$src1, RC:$src2, RC:$src3),
             !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
             []>, XOP_4V, VEX_W, Sched<[WriteShuffle]>, FoldGenData<NAME#rrr>;
 }
multiclass xop_vpermil2<bits<8> Opc, string OpcodeStr, RegisterClass RC,
   def rm : IXOP5<Opc, MRMSrcMemOp4, (outs RC:$dst),
         (ins RC:$src1, RC:$src2, intmemop:$src3, u8imm:$src4),
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set RC:$dst,
           (VT (X86vpermil2 RC:$src1, RC:$src2,
                            (bitconvert (IntLdFrag addr:$src3)),
                            (i8 imm:$src4))))]>, VEX_W,
-        Sched<[WriteFShuffleLd, ReadAfterLd]>;
+        Sched<[WriteFShuffleLd, ReadAfterLd, ReadAfterLd]>;
   def mr : IXOP5<Opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, fpmemop:$src2, RC:$src3, u8imm:$src4),
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set RC:$dst,
           (VT (X86vpermil2 RC:$src1, (FPLdFrag addr:$src2),
                            RC:$src3, (i8 imm:$src4))))]>,
-        Sched<[WriteFShuffleLd, ReadAfterLd]>;
+        Sched<[WriteFShuffleLd, ReadAfterLd,
+               // fpmemop:$src2
+               ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault,
+               // RC:$src3
+               ReadAfterLd]>;
   // For disassembler
   let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
   def rr_REV : IXOP5<Opc, MRMSrcRegOp4, (outs RC:$dst),
         (ins RC:$src1, RC:$src2, RC:$src3, u8imm:$src4),
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         []>, VEX_W, Sched<[WriteFShuffle]>, FoldGenData<NAME#rr>;
 }