Improved throughput calculation

[avt77]

Summary

Some instructions (zerroall, zeroupper, etc.) don't have any modeled throughput values. The patch suggests usage of latency as throughput values in such cases.

[RKSimon]

lib/CodeGen/TargetSchedule.cpp

355

Aren't this change (and its equivalent in the next function) separate patches that need to go in first?

lib/Target/X86/X86ScheduleBtVer2.td

345

This change to the btver2 model is a separate patch as well

[RKSimon]

rebase?

[avt77]

I'm not sure we need rebase this patch because 2 pre-patches do almost everything I'd like to see here. There is only one last thing: a possibilty to return latency as value of throughput if we don't have any sched model or don't have ResourceCycles in the existing model. But now I think that programmer can do this functionality by himself (if s/he needs it). Am I right? I'm going to abandon this review. OK?

[avt77]

There were 2 commits covering this patch

include/llvm/CodeGen/TargetSchedule.h

class TargetSchedModel {
   const TargetSubtargetInfo *STI = nullptr;
   const TargetInstrInfo *TII = nullptr;
 
   SmallVector<unsigned, 16> ResourceFactors;
   unsigned MicroOpFactor; // Multiply to normalize microops to resource units.
   unsigned ResourceLCM;   // Resource units per cycle. Latency normalization factor.
 
   unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const;
+  Optional<double>
+  getRThroughputFromInstrSchedModel(const MCSchedClassDesc *SCDesc) const;
 
 public:
   TargetSchedModel() : SchedModel(MCSchedModel::GetDefaultSchedModel()) {}
 
   /// \brief Initialize the machine model for instruction scheduling.
   ///
   /// The machine model API keeps a copy of the top-level MCSchedModel table
   /// indices and may query TargetSubtargetInfo and TargetInstrInfo to resolve

lib/CodeGen/TargetSchedule.cpp

if (SCDesc->isValid()) {
       }
     }
   }
   return 0;
 }
 
 static Optional<double>
 getRThroughputFromItineraries(unsigned schedClass,
-                              const InstrItineraryData *IID){
+                              const InstrItineraryData *IID) {
   double Unknown = std::numeric_limits<double>::infinity();
   double Throughput = Unknown;
 
   for (const InstrStage *IS = IID->beginStage(schedClass),
                         *E = IID->endStage(schedClass);
        IS != E; ++IS) {
     unsigned Cycles = IS->getCycles();
     if (!Cycles)
       continue;
     Throughput =
         std::min(Throughput, countPopulation(IS->getUnits()) * 1.0 / Cycles);
   }
+  if (Throughput == Unknown)
+    return Optional<double>();
   // We need reciprocal throughput that's why we return such value.
   return 1 / Throughput;
 }
 
-static Optional<double>
-getRThroughputFromInstrSchedModel(const MCSchedClassDesc *SCDesc,
-                                  const TargetSubtargetInfo *STI,
-                                  const MCSchedModel &SchedModel) {
+Optional<double> TargetSchedModel::getRThroughputFromInstrSchedModel(
+    const MCSchedClassDesc *SCDesc) const {
   double Unknown = std::numeric_limits<double>::infinity();
   double Throughput = Unknown;
-
-  for (const MCWriteProcResEntry *WPR = STI->getWriteProcResBegin(SCDesc),
-                                 *WEnd = STI->getWriteProcResEnd(SCDesc);
-       WPR != WEnd; ++WPR) {
+  const MCWriteProcResEntry *WPR = STI->getWriteProcResBegin(SCDesc),
+  if ((WPR == WEnd) && SCDesc->isValid())
+    return computeInstrLatency(*SCDesc);
+  for (; WPR != WEnd; ++WPR) {
     unsigned Cycles = WPR->Cycles;
     if (!Cycles)
-      return Optional<double>();
+      continue;
 
     unsigned NumUnits =
         SchedModel.getProcResource(WPR->ProcResourceIdx)->NumUnits;
     Throughput = std::min(Throughput, NumUnits * 1.0 / Cycles);
   }
+  if (Throughput == Unknown)
+    return Optional<double>();
   // We need reciprocal throughput that's why we return such value.
   return 1 / Throughput;
 }
 
 Optional<double>
 TargetSchedModel::computeInstrRThroughput(const MachineInstr *MI) const {
   if (hasInstrItineraries())
     return getRThroughputFromItineraries(MI->getDesc().getSchedClass(),
                                          getInstrItineraries());
   if (hasInstrSchedModel())
-    return getRThroughputFromInstrSchedModel(resolveSchedClass(MI), STI,
-                                             SchedModel);
+    return getRThroughputFromInstrSchedModel(resolveSchedClass(MI));
   return Optional<double>();
 }
 
 Optional<double>
 TargetSchedModel::computeInstrRThroughput(unsigned Opcode) const {
   unsigned SchedClass = TII->get(Opcode).getSchedClass();
   if (hasInstrItineraries())
     return getRThroughputFromItineraries(SchedClass, getInstrItineraries());
   if (hasInstrSchedModel()) {
     const MCSchedClassDesc *SCDesc = SchedModel.getSchedClassDesc(SchedClass);
     if (SCDesc->isValid() && !SCDesc->isVariant())
-      return getRThroughputFromInstrSchedModel(SCDesc, STI, SchedModel);
+      return getRThroughputFromInstrSchedModel(SCDesc);
   }
   return Optional<double>();
 }

lib/Target/X86/X86ScheduleBtVer2.td

def : WriteRes<WriteCLMul, [JVIMUL]> {
   let Latency = 2;
   let ResourceCycles = [1];
 }
 def : WriteRes<WriteCLMulLd, [JLAGU, JVIMUL]> {
   let Latency = 7;
   let ResourceCycles = [1, 1];
 }
 
+def WriteJVZEROALL: SchedWriteRes<[]> {
+  let Latency = 90;
+  let NumMicroOps = 73;
+}
+def : InstRW<[WriteJVZEROALL], (instregex "VZEROALL")>;
+
+def WriteJVZEROUPPER: SchedWriteRes<[]> {
+  let Latency = 46;
+  let NumMicroOps = 37;
+}
+def : InstRW<[WriteJVZEROUPPER], (instregex "VZEROUPPER")>;
+
 // FIXME: pipe for system/microcode?
 def : WriteRes<WriteSystem,     [JAny]> { let Latency = 100; }
 def : WriteRes<WriteMicrocoded, [JAny]> { let Latency = 100; }
 def : WriteRes<WriteFence,  [JSAGU]>;
 def : WriteRes<WriteNop, []>;
 } // SchedModel
 

test/CodeGen/X86/avx-schedule.ll

 ; SANDY-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
 ; SANDY-NEXT:    vzeroupper # sched: [?:0.000000e+00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_extractf128:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [3:1.00]
 ; HASWELL-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [4:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_extractf128:
 ; BTVER2:       # BB#0:
 ; BTVER2-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:0.50]
 ; BTVER2-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_extractf128:
 ; ZNVER1:       # BB#0:
 ; ZNVER1-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:0.50]
 ; ZNVER1-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = shufflevector <8 x float> %a0, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %2 = shufflevector <8 x float> %a1, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   store <4 x float> %2, <4 x float> *%a2
   ret <4 x float> %1
 }
 
 define <4 x double> @test_haddpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {

 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vmovmskpd %ymm0, %eax # sched: [1:0.33]
 ; SANDY-NEXT:    vzeroupper # sched: [?:0.000000e+00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_movmskpd:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vmovmskpd %ymm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_movmskpd:
 ; BTVER2:       # BB#0:
 ; BTVER2-NEXT:    vmovmskpd %ymm0, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_movmskpd:
 ; ZNVER1:       # BB#0:
 ; ZNVER1-NEXT:    vmovmskpd %ymm0, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
   ret i32 %1
 }
 declare i32 @llvm.x86.avx.movmsk.pd.256(<4 x double>) nounwind readnone
 
 define i32 @test_movmskps(<8 x float> %a0) {
 ; SANDY-LABEL: test_movmskps:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vmovmskps %ymm0, %eax # sched: [1:0.33]
 ; SANDY-NEXT:    vzeroupper # sched: [?:0.000000e+00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_movmskps:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    vmovmskps %ymm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_movmskps:
 ; BTVER2:       # BB#0:
 ; BTVER2-NEXT:    vmovmskps %ymm0, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_movmskps:
 ; ZNVER1:       # BB#0:
 ; ZNVER1-NEXT:    vmovmskps %ymm0, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
   ret i32 %1
 }
 declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) nounwind readnone
 
 define <4 x double> @test_movntpd(<4 x double> %a0, <4 x double> *%a1) {
 ; SANDY-LABEL: test_movntpd:

 ;
 ; HASWELL-LABEL: test_testpd_ymm:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
 ; HASWELL-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:0.33]
 ; HASWELL-NEXT:    setb %al # sched: [1:0.50]
 ; HASWELL-NEXT:    vtestpd (%rdi), %ymm0 # sched: [5:0.50]
 ; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_testpd_ymm:
 ; BTVER2:       # BB#0:
 ; BTVER2-NEXT:    xorl %eax, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:0.50]
 ; BTVER2-NEXT:    setb %al # sched: [1:0.50]
 ; BTVER2-NEXT:    vtestpd (%rdi), %ymm0 # sched: [6:1.00]
 ; BTVER2-NEXT:    adcl $0, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_testpd_ymm:
 ; ZNVER1:       # BB#0:
 ; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.50]
 ; ZNVER1-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:0.50]
 ; ZNVER1-NEXT:    setb %al # sched: [1:0.50]
 ; ZNVER1-NEXT:    vtestpd (%rdi), %ymm0 # sched: [6:1.00]
 ; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %a1)
   %2 = load <4 x double>, <4 x double> *%a2, align 32
   %3 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %2)
   %4 = add i32 %1, %3
   ret i32 %4
 }
 declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind readnone

 ;
 ; HASWELL-LABEL: test_testps_ymm:
 ; HASWELL:       # BB#0:
 ; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
 ; HASWELL-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:0.33]
 ; HASWELL-NEXT:    setb %al # sched: [1:0.50]
 ; HASWELL-NEXT:    vtestps (%rdi), %ymm0 # sched: [5:0.50]
 ; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_testps_ymm:
 ; BTVER2:       # BB#0:
 ; BTVER2-NEXT:    xorl %eax, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:0.50]
 ; BTVER2-NEXT:    setb %al # sched: [1:0.50]
 ; BTVER2-NEXT:    vtestps (%rdi), %ymm0 # sched: [6:1.00]
 ; BTVER2-NEXT:    adcl $0, %eax # sched: [1:0.50]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_testps_ymm:
 ; ZNVER1:       # BB#0:
 ; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.50]
 ; ZNVER1-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:0.50]
 ; ZNVER1-NEXT:    setb %al # sched: [1:0.50]
 ; ZNVER1-NEXT:    vtestps (%rdi), %ymm0 # sched: [6:1.00]
 ; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   %1 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %a1)
   %2 = load <8 x float>, <8 x float> *%a2, align 32
   %3 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %2)
   %4 = add i32 %1, %3
   ret i32 %4
 }
 declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readnone

 define void @test_zeroall() {
 ; SANDY-LABEL: test_zeroall:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vzeroall # sched: [?:0.000000e+00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_zeroall:
 ; HASWELL:       # BB#0:
-; HASWELL-NEXT:    vzeroall # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroall # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_zeroall:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vzeroall # sched: [?:0.000000e+00]
+; BTVER2-NEXT:    vzeroall # sched: [90:90.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_zeroall:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vzeroall # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroall # sched: [90:90.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   call void @llvm.x86.avx.vzeroall()
   ret void
 }
 declare void @llvm.x86.avx.vzeroall() nounwind
 
 define void @test_zeroupper() {
 ; SANDY-LABEL: test_zeroupper:
 ; SANDY:       # BB#0:
 ; SANDY-NEXT:    vzeroupper # sched: [?:0.000000e+00]
 ; SANDY-NEXT:    retq # sched: [5:1.00]
 ;
 ; HASWELL-LABEL: test_zeroupper:
 ; HASWELL:       # BB#0:
-; HASWELL-NEXT:    vzeroupper # sched: [1:0.00]
+; HASWELL-NEXT:    vzeroupper # sched: [1:1.00]
 ; HASWELL-NEXT:    retq # sched: [1:1.00]
 ;
 ; BTVER2-LABEL: test_zeroupper:
 ; BTVER2:       # BB#0:
-; BTVER2-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; BTVER2-NEXT:    vzeroupper # sched: [46:46.00]
 ; BTVER2-NEXT:    retq # sched: [4:1.00]
 ;
 ; ZNVER1-LABEL: test_zeroupper:
 ; ZNVER1:       # BB#0:
-; ZNVER1-NEXT:    vzeroupper # sched: [?:0.000000e+00]
+; ZNVER1-NEXT:    vzeroupper # sched: [46:46.00]
 ; ZNVER1-NEXT:    retq # sched: [4:1.00]
   call void @llvm.x86.avx.vzeroupper()
   ret void
 }
 declare void @llvm.x86.avx.vzeroupper() nounwind
 
 !0 = !{i32 1}