[llvm-profdata] Add option to ingest filepaths from a file

[vsk]

Summary

This is useful if the shell can't handle the number of files being passed in.

[davidxl]

lgtm

llvm/trunk/docs/CommandGuide/llvm-profdata.rst

 
 By default profile data is merged without modification. This means that the
 relative importance of each input file is proportional to the number of samples
 or counts it contains. In general, the input from a longer training run will be
 interpreted as relatively more important than a shorter run. Depending on the
 nature of the training runs it may be useful to adjust the weight given to each
 input file by using the ``-weighted-input`` option.
 
+Profiles passed in via ``-weighted-input``, ``-input-files``, or via positional
+arguments are processed once for each time they are seen.
+
 
 OPTIONS
 ^^^^^^^
 
 .. option:: -help
 
  Print a summary of command line options.
 
 .. option:: -output=output, -o=output
 
  Specify the output file name.  *Output* cannot be ``-`` as the resulting
  indexed profile data can't be written to standard output.
 
 .. option:: -weighted-input=weight,filename
 
  Specify an input file name along with a weight. The profile counts of the
  supplied ``filename`` will be scaled (multiplied) by the supplied
  ``weight``, where where ``weight`` is a decimal integer >= 1.
  Input files specified without using this option are assigned a default
  weight of 1. Examples are shown below.
 
+.. option:: -input-files=path, -f=path
+
+  Specify a file which contains a list of files to merge. The entries in this
+  file are newline-separated. Lines starting with '#' are skipped. Entries may
+  be of the form <filename> or <weight>,<filename>.
+
 .. option:: -instr (default)
 
  Specify that the input profile is an instrumentation-based profile.
 
 .. option:: -sample
 
  Specify that the input profile is a sample-based profile.
  

llvm/trunk/test/tools/llvm-profdata/input-filenames.test

+# Create an input file.
+RUN: printf '# comment 1\n' > %t.input
+RUN: printf ' # comment 2\n' >> %t.input
+RUN: printf 'foo\n' >> %t.input
+RUN: printf ' bar\n' >> %t.input
+RUN: printf "2,%t.weighted\n" >> %t.input
+
+# Create the weighted file, since these actually need to exist.
+RUN: printf ' ' > %t.weighted
+
+# RUN: llvm-profdata merge -f %t.input -dump-input-file-list -o /dev/null | FileCheck %s
+# RUN: llvm-profdata merge -input-files %t.input -dump-input-file-list -o /dev/null | FileCheck %s
+
+# CHECK: 1,foo
+# CHECK-NEXT: 1,bar
+# CHECK-NEXT: 2,{{.*}}.weighted

llvm/trunk/tools/llvm-profdata/llvm-profdata.cpp

static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
 
   if (!sys::fs::exists(FileName))
     exitWithErrorCode(make_error_code(errc::no_such_file_or_directory),
                       FileName);
 
   return WeightedFile(FileName, Weight);
 }
 
+static void parseInputFilenamesFile(const StringRef &InputFilenamesFile,
+                                    WeightedFileVector &WFV) {
+  if (InputFilenamesFile == "")
+    return;
+
+  auto Buf = MemoryBuffer::getFileOrSTDIN(InputFilenamesFile);
+  if (!Buf)
+    exitWithErrorCode(Buf.getError(), InputFilenamesFile);
+
+  StringRef Data = Buf.get()->getBuffer();
+  SmallVector<StringRef, 8> Entries;
+  Data.split(Entries, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+  for (const StringRef &FileWeightEntry : Entries) {
+    StringRef SanitizedEntry = FileWeightEntry.trim(" \t\v\f\r");
+    // Skip comments.
+    if (SanitizedEntry.startswith("#"))
+      continue;
+    // If there's no comma, it's an unweighted profile.
+    else if (SanitizedEntry.rfind(',') == StringRef::npos)
+      WFV.emplace_back(SanitizedEntry, 1);
+    else
+      WFV.emplace_back(parseWeightedFile(SanitizedEntry));
+  }
+}
+
 static int merge_main(int argc, const char *argv[]) {
   cl::list<std::string> InputFilenames(cl::Positional,
                                        cl::desc("<filename...>"));
   cl::list<std::string> WeightedInputFilenames("weighted-input",
                                                cl::desc("<weight>,<filename>"));
+  cl::opt<std::string> InputFilenamesFile(
+      "input-files", cl::init(""),
+      cl::desc("Path to file containing newline-separated "
+               "<filename>[,<weight>] entries"));
+  cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"),
+                                cl::aliasopt(InputFilenamesFile));
+  cl::opt<bool> DumpInputFileList(
+      "dump-input-file-list", cl::init(false), cl::Hidden,
+      cl::desc("Dump the list of input files and their weights, then exit"));
   cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
                                       cl::init("-"), cl::Required,
                                       cl::desc("Output file"));
   cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
                             cl::aliasopt(OutputFilename));
   cl::opt<ProfileKinds> ProfileKind(
       cl::desc("Profile kind:"), cl::init(instr),
       cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
                  clEnumVal(sample, "Sample profile"), clEnumValEnd));
   cl::opt<ProfileFormat> OutputFormat(
       cl::desc("Format of output profile"), cl::init(PF_Binary),
       cl::values(clEnumValN(PF_Binary, "binary", "Binary encoding (default)"),
                  clEnumValN(PF_Text, "text", "Text encoding"),
                  clEnumValN(PF_GCC, "gcc",
                             "GCC encoding (only meaningful for -sample)"),
                  clEnumValEnd));
   cl::opt<bool> OutputSparse("sparse", cl::init(false),
       cl::desc("Generate a sparse profile (only meaningful for -instr)"));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
 
-  if (InputFilenames.empty() && WeightedInputFilenames.empty())
-    exitWithError("No input files specified. See " +
-                  sys::path::filename(argv[0]) + " -help");
-
   WeightedFileVector WeightedInputs;
   for (StringRef Filename : InputFilenames)
     WeightedInputs.push_back(WeightedFile(Filename, 1));
   for (StringRef WeightedFilename : WeightedInputFilenames)
     WeightedInputs.push_back(parseWeightedFile(WeightedFilename));
+  parseInputFilenamesFile(InputFilenamesFile, WeightedInputs);
+
+  if (WeightedInputs.empty())
+    exitWithError("No input files specified. See " +
+                  sys::path::filename(argv[0]) + " -help");
+
+  if (DumpInputFileList) {
+    for (auto &WF : WeightedInputs)
+      outs() << WF.Weight << "," << WF.Filename << "\n";
+    return 0;
+  }
 
   if (ProfileKind == instr)
     mergeInstrProfile(WeightedInputs, OutputFilename, OutputFormat,
                       OutputSparse);
   else
     mergeSampleProfile(WeightedInputs, OutputFilename, OutputFormat);
 
   return 0;