[X86] Correct the scheduling information for AVX-VNNI and AVX512-VNNI instructons.

[craig.topper]

Summary

The AVXVNNI load instructions weren't using the Folded load write
class and they had no ReadAdvance.
The YMM versions were using the XMM schedule class.

The AVX512VNNI instructions had the right classes, but not enough
ReadAdvances to account for the 2 sources.

Noticed while investigating #62026.

[pengfei]

had the write classes

Do you mean right?

llvm/lib/Target/X86/X86InstrAVX512.td

12580–12581

It's a mystery to me that AVX512 has a lot of 3 ops instructions but not even have one ReadAfterFold, in X86InstrAVX512.td

grep -rn 'ReadAfterFold,' llvm/lib/Target/X86/X86InstrAVX512.td

[craig.topper]

In D147872#4253742, @pengfei wrote:

had the write classes

Do you mean right?

Yes. Fixed. Thanks!

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

12580–12581

Yeah just from a quick check, vpermt2 vpermi2, and vpternlog are also wrong.

Also interesting, EVEX FMA instructions are wrong but are correct for the VEX versions in X86InstrFMA.td

[pengfei]

LGTM.

[LuoYuanke]

llvm/lib/Target/X86/X86InstrAVX512.td

12589–12590

@craig.topper, I don't understand why it needs another sched.ReadAfterFold? Is it because there are 3 operands?

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

12589–12590

Yes each read is associated with an operand. So we need one for each vector source.

[LuoYuanke]

llvm/lib/Target/X86/X86InstrAVX512.td

12589–12590

Why in line 12580, for rr version, there is only one sched (WriteVecIMulX) in the sched list?

[LuoYuanke]

llvm/lib/Target/X86/X86InstrAVX512.td

12589–12590

I mean

defm r  :   AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),
                                 (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,
                                 "$src3, $src2", "$src2, $src3",
                                 (VTI.VT (OpNode VTI.RC:$src1,
                                          VTI.RC:$src2, VTI.RC:$src3)),
                                 IsCommutable, IsCommutable>,
                                 EVEX_4V, T8PD, Sched<[sched]>;

[pengfei]

llvm/lib/Target/X86/X86InstrAVX512.td

12589–12590

My understand is it's intended for memory variant https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/X86/X86Schedule.td#L41-L51

llvm/lib/Target/X86/X86InstrAVX512.td

defm r  :   AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),
                                    IsCommutable, IsCommutable>,
                                    EVEX_4V, T8PD, Sched<[sched]>;
   defm m  :   AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
                                    (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,
                                    "$src3, $src2", "$src2, $src3",
                                    (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
                                             (VTI.VT (VTI.LdFrag addr:$src3))))>,
                                    EVEX_4V, EVEX_CD8<32, CD8VF>, T8PD,
-                                   Sched<[sched.Folded, sched.ReadAfterFold]>;
+                                   Sched<[sched.Folded, sched.ReadAfterFold,
+                                          sched.ReadAfterFold]>;
   defm mb :   AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
                                    (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3),
                                    OpStr, "${src3}"#VTI.BroadcastStr#", $src2",
                                    "$src2, ${src3}"#VTI.BroadcastStr,
                                    (OpNode VTI.RC:$src1, VTI.RC:$src2,
                                     (VTI.VT (VTI.BroadcastLdFrag addr:$src3)))>,
                                    EVEX_4V, EVEX_CD8<32, CD8VF>, EVEX_B,
-                                   T8PD, Sched<[sched.Folded, sched.ReadAfterFold]>;
+                                   T8PD, Sched<[sched.Folded, sched.ReadAfterFold,
+                                                sched.ReadAfterFold]>;
   }
 }
 
 multiclass VNNI_common<bits<8> Op, string OpStr, SDNode OpNode,
                        X86SchedWriteWidths sched, bit IsCommutable> {
   let Predicates = [HasVNNI] in
   defm Z      :   VNNI_rmb<Op, OpStr, OpNode, sched.ZMM, v16i32_info,
                            IsCommutable>, EVEX_V512;

llvm/lib/Target/X86/X86InstrSSE.td

def rr  : AVX8I<opc, MRMSrcReg, (outs VR128:$dst),
                                        VR128:$src2, VR128:$src3)))]>,
              VEX_4V, Sched<[SchedWriteVecIMul.XMM]>;
 
   def rm  : AVX8I<opc, MRMSrcMem, (outs VR128:$dst),
              (ins VR128:$src1, VR128:$src2, i128mem:$src3),
              !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
              [(set VR128:$dst, (v4i32 (OpNode VR128:$src1, VR128:$src2,
                                       (loadv4i32 addr:$src3))))]>,
-             VEX_4V, Sched<[SchedWriteVecIMul.XMM]>;
+             VEX_4V, Sched<[SchedWriteVecIMul.XMM.Folded,
+                            SchedWriteVecIMul.XMM.ReadAfterFold,
+                            SchedWriteVecIMul.XMM.ReadAfterFold]>;
 
   let isCommutable = IsCommutable in
   def Yrr  : AVX8I<opc, MRMSrcReg, (outs VR256:$dst),
              (ins VR256:$src1, VR256:$src2, VR256:$src3),
              !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
              [(set VR256:$dst, (v8i32 (OpNode VR256:$src1,
                                        VR256:$src2, VR256:$src3)))]>,
-             VEX_4V, VEX_L, Sched<[SchedWriteVecIMul.XMM]>;
+             VEX_4V, VEX_L, Sched<[SchedWriteVecIMul.YMM]>;
 
   def Yrm  : AVX8I<opc, MRMSrcMem, (outs VR256:$dst),
              (ins VR256:$src1, VR256:$src2, i256mem:$src3),
              !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
              [(set VR256:$dst, (v8i32 (OpNode VR256:$src1, VR256:$src2,
                                       (loadv8i32 addr:$src3))))]>,
-             VEX_4V, VEX_L, Sched<[SchedWriteVecIMul.XMM]>;
+             VEX_4V, VEX_L, Sched<[SchedWriteVecIMul.YMM.Folded,
+                                   SchedWriteVecIMul.YMM.ReadAfterFold,
+                                   SchedWriteVecIMul.YMM.ReadAfterFold]>;
 }
 
 defm VPDPBUSD   : avx_vnni_rm<0x50, "vpdpbusd", X86Vpdpbusd, 0>;
 defm VPDPBUSDS  : avx_vnni_rm<0x51, "vpdpbusds", X86Vpdpbusds, 0>;
 defm VPDPWSSD   : avx_vnni_rm<0x52, "vpdpwssd",  X86Vpdpwssd, 1>;
 defm VPDPWSSDS  : avx_vnni_rm<0x53, "vpdpwssds", X86Vpdpwssds, 1>;
 
 def X86vpmaddwd_su : PatFrag<(ops node:$lhs, node:$rhs),