[RISCV][Codegen] Account for LMUL in Vector Mask instructions

[michaelmaitland]

Summary

It is likley that subtargets act differently for vector fixed-point arithmetic instructions based on the LMUL. This patch creates seperate SchedRead, SchedWrite, WriteRes, ReadAdvance for each relevant LMUL.

[kito-cheng]

llvm/lib/Target/RISCV/RISCVScheduleV.td

832–837

I think that could be fixed in another small document fix only NFC patch?

[craig.topper]

llvm/lib/Target/RISCV/RISCVInstrInfoV.td

722

Line this up after the bracket on the previous line. Similar to parens in C.

[michaelmaitland]

Fix formatting

[michaelmaitland]

llvm/lib/Target/RISCV/RISCVScheduleV.td

832–837

Fixed in https://reviews.llvm.org/D138311

[michaelmaitland]

Rebase.

[craig.topper]

LGTM

llvm/lib/Target/RISCV/RISCVInstrInfoV.td

 }
 
 multiclass VWREDO_FV_V<string opcodestr, bits<6> funct6> {
   def _VS : VALUVV<funct6, OPFVV, opcodestr # ".vs">,
             Sched<[WriteVFWRedOV, ReadVFWRedOV, ReadVFWRedOV0, ReadVMask]>;
 }
 
 multiclass VMALU_MV_Mask<string opcodestr, bits<6> funct6, string vm = "v"> {
-  def M : VALUVVNoVm<funct6, OPMVV, opcodestr # "." # vm # "m">,
-          Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
+  def M : VALUVVNoVm<funct6, OPMVV, opcodestr #"." #vm #"m">,
+          Sched<[WriteVMALUV_UpperBound, ReadVMALUV_UpperBound,
+                 ReadVMALUV_UpperBound]>;
 }
 
 multiclass VMSFS_MV_V<string opcodestr, bits<6> funct6, bits<5> vs1> {
   def "" : VALUVs2<funct6, vs1, OPMVV, opcodestr>,
-           Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>;
+           Sched<[WriteVMSFSV_UpperBound, ReadVMSFSV_UpperBound, ReadVMask]>;
 }
 
 multiclass VMIOT_MV_V<string opcodestr, bits<6> funct6, bits<5> vs1> {
   def "" : VALUVs2<funct6, vs1, OPMVV, opcodestr>,
-           Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
+           Sched<[WriteVMIotV_UpperBound, ReadVMIotV_UpperBound, ReadVMask]>;
 }
 
 multiclass VSHT_IV_V_X_I<string opcodestr, bits<6> funct6, Operand optype = simm5, string vw = "v"> {
   def V  : VALUVV<funct6, OPIVV, opcodestr # "." # vw # "v">,
            Sched<[WriteVShiftV_UpperBound, ReadVShiftV_UpperBound,
              ReadVShiftV_UpperBound, ReadVMask]>;
   def X  : VALUVX<funct6, OPIVX, opcodestr # "." # vw # "x">,
            Sched<[WriteVShiftX_UpperBound, ReadVShiftV_UpperBound,

 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0,
     RVVConstraint = NoConstraint  in {
 
 // Vector mask population count vcpop
 def VCPOP_M : RVInstV<0b010000, 0b10000, OPMVV, (outs GPR:$vd),
                       (ins VR:$vs2, VMaskOp:$vm),
                       "vcpop.m", "$vd, $vs2$vm">,
-              Sched<[WriteVMPopV, ReadVMPopV, ReadVMask]>;
+              Sched<[WriteVMPopV_UpperBound, ReadVMPopV_UpperBound,
+                     ReadVMask]>;
 
 // vfirst find-first-set mask bit
 def VFIRST_M : RVInstV<0b010000, 0b10001, OPMVV, (outs GPR:$vd),
                        (ins VR:$vs2, VMaskOp:$vm),
                        "vfirst.m", "$vd, $vs2$vm">,
-              Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMask]>;
+              Sched<[WriteVMFFSV_UpperBound, ReadVMFFSV_UpperBound,
+                     ReadVMask]>;
 
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
 def : InstAlias<"vpopc.m $vd, $vs2$vm",
                 (VCPOP_M GPR:$vd, VR:$vs2, VMaskOp:$vm), 0>;
 
 let Constraints = "@earlyclobber $vd", RVVConstraint = Iota in {
 

 } // Constraints = "@earlyclobber $vd", RVVConstraint = Iota
 
 // Vector Element Index Instruction
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
 
 let vs2 = 0 in
 def VID_V : RVInstV<0b010100, 0b10001, OPMVV, (outs VR:$vd),
                     (ins VMaskOp:$vm), "vid.v", "$vd$vm">,
-            Sched<[WriteVMIdxV, ReadVMask]>;
+            Sched<[WriteVMIdxV_UpperBound, ReadVMask]>;
 
 // Integer Scalar Move Instructions
 let vm = 1, RVVConstraint = NoConstraint in {
 def VMV_X_S : RVInstV<0b010000, 0b00000, OPMVV, (outs GPR:$vd),
                       (ins VR:$vs2), "vmv.x.s", "$vd, $vs2">,
               Sched<[WriteVIMovVX, ReadVIMovVX]>;
 let Constraints = "$vd = $vd_wb" in
 def VMV_S_X : RVInstV2<0b010000, 0b00000, OPMVX, (outs VR:$vd_wb),

llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

foreach sew = EEWList in {
       }
     }
   }
 }
 
 multiclass VPseudoVPOP_M {
   foreach mti = AllMasks in
   {
+    defvar mx = mti.LMul.MX;
+    defvar WriteVMPopV_MX = !cast<SchedWrite>("WriteVMPopV_" # mx);
+    defvar ReadVMPopV_MX = !cast<SchedRead>("ReadVMPopV_" # mx);
     let VLMul = mti.LMul.value in {
       def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>,
-                           Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>;
+                           Sched<[WriteVMPopV_MX, ReadVMPopV_MX, ReadVMPopV_MX]>;
       def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask,
-                                     Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>;
+                                     Sched<[WriteVMPopV_MX, ReadVMPopV_MX, ReadVMPopV_MX]>;
     }
   }
 }
 
 multiclass VPseudoV1ST_M {
   foreach mti = AllMasks in
   {
+    defvar mx = mti.LMul.MX;
+    defvar WriteVMFFSV_MX = !cast<SchedWrite>("WriteVMFFSV_" # mx);
+    defvar ReadVMFFSV_MX = !cast<SchedRead>("ReadVMFFSV_" # mx);
     let VLMul = mti.LMul.value in {
       def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>,
-                           Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>;
+                           Sched<[WriteVMFFSV_MX, ReadVMFFSV_MX, ReadVMFFSV_MX]>;
       def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask,
-                                     Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>;
+                                     Sched<[WriteVMFFSV_MX, ReadVMFFSV_MX, ReadVMFFSV_MX]>;
     }
   }
 }
 
 multiclass VPseudoVSFS_M {
   defvar constraint = "@earlyclobber $rd";
   foreach mti = AllMasks in
   {
+    defvar mx = mti.LMul.MX;
+    defvar WriteVMSFSV_MX = !cast<SchedWrite>("WriteVMSFSV_" # mx);
+    defvar ReadVMSFSV_MX = !cast<SchedRead>("ReadVMSFSV_" # mx);
     let VLMul = mti.LMul.value in {
       def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>,
-                           Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>;
+                           Sched<[WriteVMSFSV_MX, ReadVMSFSV_MX, ReadVMask]>;
       def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>,
-                                     Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>;
+                                     Sched<[WriteVMSFSV_MX, ReadVMSFSV_MX, ReadVMask]>;
     }
   }
 }
 
 multiclass VPseudoVID_V {
   foreach m = MxList in {
+    defvar mx = m.MX;
+    defvar WriteVMIdxV_MX = !cast<SchedWrite>("WriteVMIdxV_" # mx);
+    defvar ReadVMIdxV_MX = !cast<SchedRead>("ReadVMIdxV_" # mx);
+
     let VLMul = m.value in {
       def "_V_" # m.MX : VPseudoNullaryNoMask<m.vrclass>,
-                         Sched<[WriteVMIdxV, ReadVMask]>;
+                         Sched<[WriteVMIdxV_MX, ReadVMask]>;
       def "_V_" # m.MX # "_TU": VPseudoNullaryNoMaskTU<m.vrclass>,
-                                Sched<[WriteVMIdxV, ReadVMask]>;
+                                Sched<[WriteVMIdxV_MX, ReadVMask]>;
       def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask<m.vrclass>,
                                    RISCVMaskedPseudo</*MaskOpIdx*/ 1>,
-                                   Sched<[WriteVMIdxV, ReadVMask]>;
+                                   Sched<[WriteVMIdxV_MX, ReadVMask]>;
     }
   }
 }
 
 multiclass VPseudoNullaryPseudoM <string BaseInst> {
   foreach mti = AllMasks in {
+    defvar mx = mti.LMul.MX;
+    defvar WriteVMALUV_MX = !cast<SchedWrite>("WriteVMALUV_" # mx);
+    defvar ReadVMALUV_MX = !cast<SchedRead>("ReadVMALUV_" # mx);
+
     let VLMul = mti.LMul.value in {
-      def "_M_" # mti.BX : VPseudoNullaryPseudoM<BaseInst # "_MM">;
+      def "_M_" # mti.BX : VPseudoNullaryPseudoM<BaseInst # "_MM">,
+        Sched<[WriteVMALUV_MX, ReadVMALUV_MX, ReadVMALUV_MX]>;
     }
   }
 }
 
 multiclass VPseudoVIOT_M {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxList in {
+    defvar mx = m.MX;
+    defvar WriteVMIotV_MX = !cast<SchedWrite>("WriteVMIotV_" # mx);
+    defvar ReadVMIotV_MX = !cast<SchedRead>("ReadVMIotV_" # mx);
     let VLMul = m.value in {
       def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, VR, constraint>,
-                       Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
+                       Sched<[WriteVMIotV_MX, ReadVMIotV_MX, ReadVMask]>;
       def "_" # m.MX # "_TU" : VPseudoUnaryNoMaskTU<m.vrclass, VR, constraint>,
-                               Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
+                               Sched<[WriteVMIotV_MX, ReadVMIotV_MX, ReadVMask]>;
       def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, VR, constraint>,
                                  RISCVMaskedPseudo</*MaskOpIdx*/ 2>,
-                                 Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
+                                 Sched<[WriteVMIotV_MX, ReadVMIotV_MX, ReadVMask]>;
     }
   }
 }
 
 multiclass VPseudoVCPR_V {
   foreach m = MxList in {
     let VLMul = m.value in
       def _VM # "_" # m.MX : VPseudoUnaryAnyMask<m.vrclass, m.vrclass>,
foreach m = MxList in
     defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
 multiclass VPseudoBinaryV_VI_LMUL<Operand ImmType = simm5, LMULInfo m, string Constraint = ""> {
   defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
 multiclass VPseudoVALU_MM {
-  foreach m = MxList in
+  foreach m = MxList in {
+    defvar mx = m.MX;
+    defvar WriteVMALUV_MX = !cast<SchedWrite>("WriteVMALUV_" # mx);
+    defvar ReadVMALUV_MX = !cast<SchedRead>("ReadVMALUV_" # mx);
+
     let VLMul = m.value in {
-      def "_MM_" # m.MX : VPseudoBinaryNoMask<VR, VR, VR, "", /*DummyMask*/0>,
-                          Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
-    }
+      def "_MM_" # mx : VPseudoBinaryNoMask<VR, VR, VR, "", /*DummyMask*/0>,
+                          Sched<[WriteVMALUV_MX, ReadVMALUV_MX, ReadVMALUV_MX]>;
+    }
 }
 
 // We use earlyclobber here due to
 // * The destination EEW is smaller than the source EEW and the overlap is
 //   in the lowest-numbered part of the source register group is legal.
 //   Otherwise, it is illegal.
 // * The destination EEW is greater than the source EEW, the source EMUL is
 //   at least 1, and the overlap is in the highest-numbered part of the

 defm PseudoVMANDN: VPseudoVALU_MM;
 defm PseudoVMXOR: VPseudoVALU_MM;
 defm PseudoVMOR: VPseudoVALU_MM;
 defm PseudoVMNOR: VPseudoVALU_MM;
 defm PseudoVMORN: VPseudoVALU_MM;
 defm PseudoVMXNOR: VPseudoVALU_MM;
 
 // Pseudo instructions
-defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">,
-                   Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
-defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">,
-                   Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
+defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">;
+defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">;
 
 //===----------------------------------------------------------------------===//
 // 16.2. Vector mask population count vcpop
 //===----------------------------------------------------------------------===//
 
 defm PseudoVCPOP: VPseudoVPOP_M;
 
 //===----------------------------------------------------------------------===//

llvm/lib/Target/RISCV/RISCVScheduleV.td

 def WriteVFRedV       : SchedWrite;
 def WriteVFRedOV      : SchedWrite;
 // 14.4. Vector Widening Floating-Point Reduction Instructions
 def WriteVFWRedV      : SchedWrite;
 def WriteVFWRedOV     : SchedWrite;
 
 // 15. Vector Mask Instructions
 // 15.1. Vector Mask-Register Logical Instructions
-def WriteVMALUV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMALUV">;
 // 15.2. Vector Mask Population Count
-def WriteVMPopV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMPopV">;
 // 15.3. Vector Find-First-Set Mask Bit
-def WriteVMFFSV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMFFSV">;
 // 15.4. Vector Set-Before-First Mask Bit
 // 15.5. Vector Set-Including-First Mask Bit
 // 15.6. Vector Set-only-First Mask Bit
-def WriteVMSFSV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMSFSV">;
 // 15.8. Vector Iota Instruction
-def WriteVMIotV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMIotV">;
 // 15.9. Vector Element Index Instruction
-def WriteVMIdxV       : SchedWrite;
+defm "" : LMULSchedWrites<"WriteVMIdxV">;
 
 // 16. Vector Permutation Instructions
 // 16.1. Integer Scalar Move Instructions
 def WriteVIMovVX      : SchedWrite;
 def WriteVIMovXV      : SchedWrite;
 // 16.2. Floating-Point Scalar Move Instructions
 def WriteVFMovVF      : SchedWrite;
 def WriteVFMovFV      : SchedWrite;

 // 14.4. Vector Widening Floating-Point Reduction Instructions
 def ReadVFWRedV       : SchedRead;
 def ReadVFWRedV0      : SchedRead;
 def ReadVFWRedOV      : SchedRead;
 def ReadVFWRedOV0     : SchedRead;
 
 // 15. Vector Mask Instructions
 // 15.1. Vector Mask-Register Logical Instructions
-def ReadVMALUV        : SchedRead;
+defm "" : LMULSchedReads<"ReadVMALUV">;
 // 15.2. Vector Mask Population Count
-def ReadVMPopV        : SchedRead;
+defm "" : LMULSchedReads<"ReadVMPopV">;
 // 15.3. Vector Find-First-Set Mask Bit
-def ReadVMFFSV        : SchedRead;
+defm "" : LMULSchedReads<"ReadVMFFSV">;
 // 15.4. Vector Set-Before-First Mask Bit
 // 15.5. Vector Set-Including-First Mask Bit
 // 15.6. Vector Set-only-First Mask Bit
-def ReadVMSFSV        : SchedRead;
+defm "" : LMULSchedReads<"ReadVMSFSV">;
 // 15.8. Vector Iota Instruction
-def ReadVMIotV        : SchedRead;
+defm "" : LMULSchedReads<"ReadVMIotV">;
 
 // 16. Vector Permutation Instructions
 // 16.1. Integer Scalar Move Instructions
 def ReadVIMovVX       : SchedRead;
 def ReadVIMovXV       : SchedRead;
 def ReadVIMovXX       : SchedRead;
 // 16.2. Floating-Point Scalar Move Instructions
 def ReadVFMovVF       : SchedRead;

 defm "" : LMULWriteRes<"WriteVFCvtFToFV", []>;
 defm "" : LMULWriteRes<"WriteVFWCvtIToFV", [], SchedMxListW>;
 defm "" : LMULWriteRes<"WriteVFWCvtFToIV", [], SchedMxListFW>;
 defm "" : LMULWriteRes<"WriteVFWCvtFToFV", [], SchedMxListFW>;
 defm "" : LMULWriteRes<"WriteVFNCvtIToFV", [], SchedMxListFW>;
 defm "" : LMULWriteRes<"WriteVFNCvtFToIV", [], SchedMxListW>;
 defm "" : LMULWriteRes<"WriteVFNCvtFToFV", [], SchedMxListFW>;
 
-// 15. Vector Reduction Operations
+// 14. Vector Reduction Operations
 def : WriteRes<WriteVIRedV, []>;
 def : WriteRes<WriteVIWRedV, []>;
 def : WriteRes<WriteVFRedV, []>;
 def : WriteRes<WriteVFRedOV, []>;
 def : WriteRes<WriteVFWRedV, []>;
 def : WriteRes<WriteVFWRedOV, []>;
 
-// 16. Vector Mask Instructions
-def : WriteRes<WriteVMALUV, []>;
-def : WriteRes<WriteVMPopV, []>;
-def : WriteRes<WriteVMFFSV, []>;
-def : WriteRes<WriteVMSFSV, []>;
-def : WriteRes<WriteVMIotV, []>;
-def : WriteRes<WriteVMIdxV, []>;
+// 15. Vector Mask Instructions
+defm "" : LMULWriteRes<"WriteVMALUV", []>;
+defm "" : LMULWriteRes<"WriteVMPopV", []>;
+defm "" : LMULWriteRes<"WriteVMFFSV", []>;
+defm "" : LMULWriteRes<"WriteVMSFSV", []>;
+defm "" : LMULWriteRes<"WriteVMIotV", []>;
+defm "" : LMULWriteRes<"WriteVMIdxV", []>;
 
 // 17. Vector Permutation Instructions
 def : WriteRes<WriteVIMovVX, []>;
 def : WriteRes<WriteVIMovXV, []>;
 def : WriteRes<WriteVFMovVF, []>;
 def : WriteRes<WriteVFMovFV, []>;
 def : WriteRes<WriteVISlideX, []>;
 def : WriteRes<WriteVISlideI, []>;

 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULReadAdvance<"ReadVFWCvtIToFV", 0, SchedMxListW>;
 defm "" : LMULReadAdvance<"ReadVFWCvtFToIV", 0, SchedMxListFW>;
 defm "" : LMULReadAdvance<"ReadVFWCvtFToFV", 0, SchedMxListFW>;
 defm "" : LMULReadAdvance<"ReadVFNCvtIToFV", 0, SchedMxListFW>;
 defm "" : LMULReadAdvance<"ReadVFNCvtFToIV", 0, SchedMxListW>;
 defm "" : LMULReadAdvance<"ReadVFNCvtFToFV", 0, SchedMxListFW>;
 
-// 15. Vector Reduction Operations
+// 14. Vector Reduction Operations
 def : ReadAdvance<ReadVIRedV, 0>;
 def : ReadAdvance<ReadVIRedV0, 0>;
 def : ReadAdvance<ReadVIWRedV, 0>;
 def : ReadAdvance<ReadVIWRedV0, 0>;
 def : ReadAdvance<ReadVFRedV, 0>;
 def : ReadAdvance<ReadVFRedV0, 0>;
 def : ReadAdvance<ReadVFRedOV, 0>;
 def : ReadAdvance<ReadVFRedOV0, 0>;
 def : ReadAdvance<ReadVFWRedV, 0>;
 def : ReadAdvance<ReadVFWRedV0, 0>;
 def : ReadAdvance<ReadVFWRedOV, 0>;
 def : ReadAdvance<ReadVFWRedOV0, 0>;
 
-// 16. Vector Mask Instructions
-def : ReadAdvance<ReadVMALUV, 0>;
-def : ReadAdvance<ReadVMPopV, 0>;
-def : ReadAdvance<ReadVMFFSV, 0>;
-def : ReadAdvance<ReadVMSFSV, 0>;
-def : ReadAdvance<ReadVMIotV, 0>;
+// 15. Vector Mask Instructions
+defm "" : LMULReadAdvance<"ReadVMALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVMPopV", 0>;
+defm "" : LMULReadAdvance<"ReadVMFFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVMSFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVMIotV", 0>;
 
 // 17. Vector Permutation Instructions
 def : ReadAdvance<ReadVIMovVX, 0>;
 def : ReadAdvance<ReadVIMovXV, 0>;
 def : ReadAdvance<ReadVIMovXX, 0>;
 def : ReadAdvance<ReadVFMovVF, 0>;
 def : ReadAdvance<ReadVFMovFV, 0>;
 def : ReadAdvance<ReadVFMovFX, 0>;