[WIP] Add custom nested list option parser (do not commit).

[gysit]

the topic is discussed here https://llvm.discourse.group/t/list-of-lists-pass-option/5950/3

[rriddle]

mlir/include/mlir/Dialect/Linalg/Utils/Utils.h

40

Why can't the input format be [[1,2,3],[3,4],[0,1]]? Why the need for _?

[gysit]

mlir/include/mlir/Dialect/Linalg/Utils/Utils.h

40

Yeah I would prefer your notation. I did not do it in this attempt since ListOption splits the input string at every , . That means the nested lists would be split as well and InnerListOptionParser would get "[[1", "2", "3]", "[3", "4]" ... instead of "1_2_3", "3_4" ...

I think having a proper nested angular bracket notation would require a custom ListOfListOption instead of just extending ListOption, which may be the better solution anyways. The parser would then need to match opening and closing angular brackets to identify the nested lists.

[rriddle]

mlir/include/mlir/Dialect/Linalg/Utils/Utils.h

40

Hmmm, that is quite annoying. Do you know where in the code this is coming from? We could just look into refactoring this to support what we want, even if we have to diverge from llvm::cl.

[gysit]

mlir/include/mlir/Dialect/Linalg/Utils/Utils.h

40

The CommaSeparateAndAddOccurrence function seems to split the string:
https://github.com/llvm/llvm-project/blob/b896334834305c8b68b9b6e6cb4386adb4ed1a07/llvm/lib/Support/CommandLine.cpp#L620

mlir/include/mlir/Dialect/Linalg/Utils/Utils.h

 #define MLIR_DIALECT_LINALG_UTILS_UTILS_H
 
 #include "mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Support/CommandLine.h"
 
 namespace mlir {
 class AffineExpr;
 class AffineForOp;
 class AffineMap;
 class PatternRewriter;
 
 namespace linalg {
 class LinalgDependenceGraph;
 
 //===----------------------------------------------------------------------===//
+// Custom command line option
+//===----------------------------------------------------------------------===//
+
+/// Custom option parser to parse list of list command line options.
+///
+/// Example:
+/// ```
+/// ListOption<std::vector<int64_t>,
+///            InnerListOptionParser<int64_t>> myOpt{*this, "my-opt", ... }
+/// ```
+/// Translates --my-opt=1_2_3,3_4,0_1 to [[1,2,3],[3,4],[0,1]]
+template <typename DataType>
+class InnerListOptionParser {
+public:
+  InnerListOptionParser(llvm::cl::Option &O) : elemParser(O) {}
+
+  using parser_data_type = std::vector<DataType>;
+
+  void initialize() {}
+  void getExtraOptionNames(SmallVectorImpl<StringRef> &OptionNames) {}
+  enum llvm::cl::ValueExpected getValueExpectedFlagDefault() const {
+    return llvm::cl::ValueRequired;
+  }
+
+  /// Split `Arg` into list elements and parse them using `elemParser`.
+  bool parse(llvm::cl::Option &O, StringRef ArgName, StringRef Arg,
+             parser_data_type &Val) {
+    SmallVector<StringRef> elements;
+    Arg.split(elements, '_');
+    for (StringRef elem : elements) {
+      int64_t curr;
+      if (elemParser.parse(O, ArgName, elem, curr))
+        return O.error("Cannot parse list element '" + elem + "'!");
+      Val.push_back(curr);
+    }
+    return false;
+  }
+
+  /// Add the `vector<>` qualifier to the option info.
+  void printOptionInfo(const llvm::cl::Option &O, size_t GlobalWidth) const {
+    llvm::outs() << "  --" << O.ArgStr;
+    llvm::outs() << "=<vector<" << elemParser.getValueName() << ">>";
+    llvm::cl::Option::printHelpStr(O.HelpStr, GlobalWidth, getOptionWidth(O));
+  }
+
+  /// Add the `vector<>` qualifier to the option width.
+  size_t getOptionWidth(const llvm::cl::Option &O) const {
+    StringRef vectorExt("vector<>");
+    return elemParser.getOptionWidth(O) + vectorExt.size();
+  }
+
+private:
+  llvm::cl::parser<DataType> elemParser;
+};
+
+//===----------------------------------------------------------------------===//
 // General utilities
 //===----------------------------------------------------------------------===//
 
 /// Check if `permutation` is a permutation of the range
 /// `[0, permutation.size())`.
 bool isPermutation(ArrayRef<int64_t> permutation);
 
 /// Apply the permutation defined by `permutation` to `inVec`.

mlir/include/mlir/Pass/PassOptions.h

if (Optional<StringRef> argStr = parser.findArgStrForValue(value))
       os << argStr;
     else
       llvm_unreachable("unknown data value for option");
   }
   template <typename DataT, typename ParserT>
   static void printValue(raw_ostream &os, ParserT &parser, const DataT &value) {
     os << value;
   }
+  template <typename DataT, typename ParserT>
+  static void printValue(raw_ostream &os, ParserT &parser,
+                         const std::vector<DataT> &value) {
+    llvm::interleave(value, os, "_");
+  }
   template <typename ParserT>
   static void printValue(raw_ostream &os, ParserT &parser, const bool &value) {
     os << (value ? StringRef("true") : StringRef("false"));
   }
 
 public:
   /// The specific parser to use depending on llvm::cl parser used. This is only
   /// necessary because we need to provide additional methods for certain data
public:
 
   /// This class represents a specific pass option, with a provided data type.
   template <typename DataType, typename OptionParser = OptionParser<DataType>>
   class Option
       : public llvm::cl::opt<DataType, /*ExternalStorage=*/false, OptionParser>,
         public OptionBase {
   public:
     template <typename... Args>
-    Option(PassOptions &parent, StringRef arg, Args &&... args)
+    Option(PassOptions &parent, StringRef arg, Args &&...args)
         : llvm::cl::opt<DataType, /*ExternalStorage=*/false, OptionParser>(
               arg, llvm::cl::sub(parent), std::forward<Args>(args)...) {
       assert(!this->isPositional() && !this->isSink() &&
              "sink and positional options are not supported");
       parent.options.push_back(this);
 
       // Set a callback to track if this option has a value.
       this->setCallback([this](const auto &) { this->optHasValue = true; });
public:
   /// This class represents a specific pass option that contains a list of
   /// values of the provided data type.
   template <typename DataType, typename OptionParser = OptionParser<DataType>>
   class ListOption
       : public llvm::cl::list<DataType, /*StorageClass=*/bool, OptionParser>,
         public OptionBase {
   public:
     template <typename... Args>
-    ListOption(PassOptions &parent, StringRef arg, Args &&... args)
+    ListOption(PassOptions &parent, StringRef arg, Args &&...args)
         : llvm::cl::list<DataType, /*StorageClass=*/bool, OptionParser>(
               arg, llvm::cl::sub(parent), std::forward<Args>(args)...) {
       assert(!this->isPositional() && !this->isSink() &&
              "sink and positional options are not supported");
       parent.options.push_back(this);
 
       // Set a callback to track if this option has a value.
       this->setCallback([this](const auto &) { this->optHasValue = true; });

 /// A default empty option struct to be used for passes that do not need to take
 /// any options.
 struct EmptyPipelineOptions : public PassPipelineOptions<EmptyPipelineOptions> {
 };
 
 } // namespace mlir
 
 #endif // MLIR_PASS_PASSOPTIONS_H_
-

mlir/test/Dialect/Linalg/hoist-padding.mlir

 // RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-op=linalg.matvec pad hoist-paddings=1,1,0 run-enable-pass=false" -cse -canonicalize -split-input-file | FileCheck %s --check-prefix=MATVEC
-// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-op=linalg.matvec pad hoist-paddings=1,1,0 transpose-paddings=1:0,0,0 run-enable-pass=false" -cse -canonicalize -split-input-file | FileCheck %s --check-prefix=TRANSP
+// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-op=linalg.matvec pad hoist-paddings=1,1,0 transpose-paddings=1_0,0,0 run-enable-pass=false" -cse -canonicalize -split-input-file | FileCheck %s --check-prefix=TRANSP
 // RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-op=linalg.matmul pad hoist-paddings=1,2,1 run-enable-pass=false" -cse -canonicalize -split-input-file | FileCheck %s --check-prefix=MATMUL
 
 //  MATVEC-DAG: #[[DIV4:[0-9a-z]+]] = affine_map<(d0) -> (d0 ceildiv 4)>
 
 //      MATVEC:  static_size_divisible
 // MATVEC-SAME:    %[[ARG1:[0-9a-zA-Z]*]]: tensor<12xf32>
 func @static_size_divisible(%arg0: tensor<24x12xf32>,
                             %arg1: tensor<12xf32>,

mlir/test/lib/Dialect/Linalg/TestLinalgCodegenStrategy.cpp

struct TestLinalgCodegenStrategy
   ListOption<int64_t> packPaddings{
       *this, "pack-paddings",
       llvm::cl::desc("Operand packing flags when test-pad-pattern."),
       llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
   ListOption<int64_t> hoistPaddings{
       *this, "hoist-paddings",
       llvm::cl::desc("Operand hoisting depths when test-pad-pattern."),
       llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
-  ListOption<std::string> transposePaddings{
+  ListOption<std::vector<int64_t>, InnerListOptionParser<int64_t>>
+      transposePaddings{
           *this, "transpose-paddings",
           llvm::cl::desc(
-          "Transpose paddings when test-pad-pattern. Specify a "
-          "operand dimension interchange using the following format:\n"
-          "-transpose-paddings=1:0:2,0:1,0:1\n"
-          "It defines the interchange [1, 0, 2] for operand one and "
-          "the interchange [0, 1] (no transpose) for the remaining operands."
-          "All interchange vectors have to be permuations matching the "
+              "Transpose paddings when test-pad-pattern. Specify a\n"
+              "\t  operand dimension interchange using the following format:\n"
+              "\t\t--transpose-paddings=1_0_2,0_1,0_1\n"
+              "\t  It defines the interchange [1, 0, 2] for operand one and\n"
+              "\t  the interchange [0, 1] (no transpose) for the remaining "
+              "operands.\n"
+              "\t  All interchange vectors have to be permuations matching the "
               "operand rank."),
           llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
   Option<bool> generalize{*this, "generalize",
                           llvm::cl::desc("Generalize named operations."),
                           llvm::cl::init(false)};
   ListOption<int64_t> iteratorInterchange{
       *this, "iterator-interchange", llvm::cl::MiscFlags::CommaSeparated,
       llvm::cl::desc("Specifies the iterator interchange.")};
   Option<bool> decompose{
       *this, "decompose",
auto hoistingFunc = [&](OpOperand &opOperand) {
     return opOperand.getOperandNumber() < hoistPaddings.size()
                ? hoistPaddings[opOperand.getOperandNumber()]
                : 0;
   };
   auto transposeFunc = [&](OpOperand &opOperand) {
     SmallVector<int64_t> transposeVector = {};
     if (opOperand.getOperandNumber() >= transposePaddings.size())
       return transposeVector;
-    SmallVector<StringRef> elems;
-    StringRef(transposePaddings[opOperand.getOperandNumber()])
-        .split(elems, ':');
-    for (StringRef elem : elems)
-      transposeVector.push_back(std::stoi(elem.str()));
+    transposeVector.append(
+        transposePaddings[opOperand.getOperandNumber()].begin(),
+        transposePaddings[opOperand.getOperandNumber()].end());
     return transposeVector;
   };
   paddingOptions.setPaddingValueComputationFunction(getNeutralOfLinalgOp);
   paddingOptions.setPaddingNoFoldComputationFunction(packFunc);
   paddingOptions.setPaddingHoistComputationFunction(hoistingFunc);
   paddingOptions.setPaddingTransposeComputationFunction(transposeFunc);
 
   // Compute input padding values only an return failure for output operands.