[GlobalISel][RISCV][TableGen] Teach GlobalISelEmitter about HwMode.

[craig.topper]

Summary

Similar to SelectionDAG, this patch treats HwMode as an additional
predicate that needs to be satisfied for GIM_CheckFeatures.

The existing predicate passes around Record * that point to predicate
objects. While HwMode expansion creates a string that needs to be
checked.

Each HwMode predicate string is uniqued by a new map that assigns
it an index. Each Rule stores the index, or -1 if HwMode doesn't
apply.

The HwMode indices each create a new Predicate feature bit and the
check string from the HwMode is used to set the feature bit.
GIM_CheckFeatures is emitted when the rule has Predicates or the HwModeIdx
is not -1.

[arsenm]

llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h

227

StringMap?

[craig.topper]

llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h

227

Would that cause non-determinism on the output of the loops in emitSubtargetFeatureBitEnumeration and emitComputeAvailableFeatures?

[arsenm]

llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h

227

Yes I think so

llvm/utils/TableGen/GlobalISelEmitter.cpp

importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
                              const std::vector<Record *> &ImplicitDefs) const;
 
   /// Analyze pattern \p P, returning a matcher for it if possible.
   /// Otherwise, return an Error explaining why we don't support it.
   Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
 
   void declareSubtargetFeature(Record *Predicate);
 
+  unsigned declareHwModeCheck(StringRef HwModeFeatures);
+
   MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
                              bool WithCoverage);
 
   /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
   /// CodeGenRegisterClass will support the CodeGenRegisterClass of
   /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
   /// If no register class is found, return std::nullopt.
   std::optional<const CodeGenRegisterClass *>
M.addAction<DebugCommentAction>(llvm::to_string(*P.getSrcPattern()) +
                                   "  =>  " +
                                   llvm::to_string(*P.getDstPattern()));
 
   SmallVector<Record *, 4> Predicates;
   P.getPredicateRecords(Predicates);
   if (auto Error = importRulePredicates(M, Predicates))
     return std::move(Error);
 
+  if (!P.getHwModeFeatures().empty())
+    M.addHwModeIdx(declareHwModeCheck(P.getHwModeFeatures()));
+
   // Next, analyze the pattern operators.
   TreePatternNode *Src = P.getSrcPattern();
   TreePatternNode *Dst = P.getDstPattern();
 
   // If the root of either pattern isn't a simple operator, ignore it.
   if (auto Err = isTrivialOperatorNode(Dst))
     return failedImport("Dst pattern root isn't a trivial operator (" +
                         toString(std::move(Err)) + ")");
void GlobalISelEmitter::run(raw_ostream &OS) {
   emitPredicatesDecl(OS, "GET_GLOBALISEL_PREDICATES_DECL");
   emitPredicatesInit(OS, "GET_GLOBALISEL_PREDICATES_INIT");
 }
 
 void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
   SubtargetFeatures.try_emplace(Predicate, Predicate, SubtargetFeatures.size());
 }
 
+unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
+  return HwModes.emplace(HwModeFeatures.str(), HwModes.size()).first->second;
+}
+
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
 
 static TableGen::Emitter::OptClass<GlobalISelEmitter>
     X("gen-global-isel", "Generate GlobalISel selector");

llvm/utils/TableGen/GlobalISelMatchTable.h

 enum {
   GISF_IgnoreCopies = 0x1,
 };
 
 using GISelFlags = std::uint16_t;
 
 //===- Helper functions ---------------------------------------------------===//
 
-std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset);
+std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset,
+                                    int HwModeIdx);
 
 /// Takes a sequence of \p Rules and group them based on the predicates
 /// they share. \p MatcherStorage is used as a memory container
 /// for the group that are created as part of this process.
 ///
 /// What this optimization does looks like if GroupT = GroupMatcher:
 /// Output without optimization:
 /// \verbatim
protected:
   unsigned NextInsnVarID;
 
   /// ID for the next output instruction allocated with allocateOutputInsnID()
   unsigned NextOutputInsnID;
 
   /// ID for the next temporary register ID allocated with allocateTempRegID()
   unsigned NextTempRegID;
 
+  // HwMode predicate index for this rule. -1 if no HwMode.
+  int HwModeIdx = -1;
+
   /// Current GISelFlags
   GISelFlags Flags = 0;
 
   std::vector<std::string> RequiredSimplePredicates;
   std::vector<Record *> RequiredFeatures;
   std::vector<std::unique_ptr<PredicateMatcher>> EpilogueMatchers;
 
   ArrayRef<SMLoc> SrcLoc;
public:
   RuleMatcher &operator=(RuleMatcher &&Other) = default;
 
   uint64_t getRuleID() const { return RuleID; }
 
   InstructionMatcher &addInstructionMatcher(StringRef SymbolicName);
   void addRequiredFeature(Record *Feature);
   const std::vector<Record *> &getRequiredFeatures() const;
 
+  void addHwModeIdx(unsigned Idx) { HwModeIdx = Idx; }
+  int getHwModeIdx() const { return HwModeIdx; }
+
   void addRequiredSimplePredicate(StringRef PredName);
   const std::vector<std::string> &getRequiredSimplePredicates();
 
   // Emplaces an action of the specified Kind at the end of the action list.
   //
   // Returns a reference to the newly created action.
   //
   // Like std::vector::emplace_back(), may invalidate all iterators if the new

llvm/utils/TableGen/GlobalISelMatchTable.cpp

 
 std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
   return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
 }
 } // namespace
 
 //===- Helpers ------------------------------------------------------------===//
 
-std::string
-getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) {
+std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset,
+                                    int HwModeIdx) {
   std::string Name = "GIFBS";
   for (const auto &Feature : FeatureBitset)
     Name += ("_" + Feature->getName()).str();
+  if (HwModeIdx >= 0)
+    Name += ("_HwMode" + std::to_string(HwModeIdx));
   return Name;
 }
 
 template <class GroupT>
 std::vector<Matcher *>
 optimizeRules(ArrayRef<Matcher *> Rules,
               std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
 
void RuleMatcher::emit(MatchTable &Table) {
 
   unsigned LabelID = Table.allocateLabelID();
   Table << MatchTable::Opcode("GIM_Try", +1)
         << MatchTable::Comment("On fail goto")
         << MatchTable::JumpTarget(LabelID)
         << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str())
         << MatchTable::LineBreak;
 
-  if (!RequiredFeatures.empty()) {
+  if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
     Table << MatchTable::Opcode("GIM_CheckFeatures")
-          << MatchTable::NamedValue(getNameForFeatureBitset(RequiredFeatures))
+          << MatchTable::NamedValue(
+                 getNameForFeatureBitset(RequiredFeatures, HwModeIdx))
           << MatchTable::LineBreak;
   }
 
   if (!RequiredSimplePredicates.empty()) {
     for (const auto &Pred : RequiredSimplePredicates) {
       Table << MatchTable::Opcode("GIM_CheckSimplePredicate")
             << MatchTable::NamedValue(Pred) << MatchTable::LineBreak;
     }

llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h

public:
   void emitTemporariesDecl(raw_ostream &OS, StringRef IfDefName);
   void emitTemporariesInit(raw_ostream &OS, unsigned MaxTemporaries,
                            StringRef IfDefName);
   void emitPredicatesDecl(raw_ostream &OS, StringRef IfDefName);
   void emitPredicatesInit(raw_ostream &OS, StringRef IfDefName);
 
   // Map of predicates to their subtarget features.
   SubtargetFeatureInfoMap SubtargetFeatures;
+
+  std::map<std::string, unsigned> HwModes;
 };
 } // namespace llvm
 
 #endif

llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp

 #include "GlobalISelMatchTable.h"
 
 using namespace llvm;
 using namespace llvm::gi;
 
 void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl(
     raw_ostream &OS, ArrayRef<RuleMatcher> Rules) {
   SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures,
-                                                           OS);
+                                                           OS, &HwModes);
 
   // Separate subtarget features by how often they must be recomputed.
   SubtargetFeatureInfoMap ModuleFeatures;
   std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
                std::inserter(ModuleFeatures, ModuleFeatures.end()),
                [](const SubtargetFeatureInfoMap::value_type &X) {
                  return !X.second.mustRecomputePerFunction();
                });
   SubtargetFeatureInfoMap FunctionFeatures;
   std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
                std::inserter(FunctionFeatures, FunctionFeatures.end()),
                [](const SubtargetFeatureInfoMap::value_type &X) {
                  return X.second.mustRecomputePerFunction();
                });
 
   SubtargetFeatureInfo::emitComputeAvailableFeatures(
       getTarget().getName(), getClassName(), "computeAvailableModuleFeatures",
-      ModuleFeatures, OS);
+      ModuleFeatures, OS, "", &HwModes);
 
   OS << "void " << getClassName()
      << "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n"
         "  AvailableFunctionFeatures = computeAvailableFunctionFeatures("
         "(const "
      << getTarget().getName()
      << "Subtarget *)&MF.getSubtarget(), &MF);\n"
         "}\n";
 
   SubtargetFeatureInfo::emitComputeAvailableFeatures(
       getTarget().getName(), getClassName(), "computeAvailableFunctionFeatures",
       FunctionFeatures, OS, "const MachineFunction *MF");
 
   // Emit a table containing the PredicateBitsets objects needed by the matcher
   // and an enum for the matcher to reference them with.
-  std::vector<std::vector<Record *>> FeatureBitsets;
+  std::vector<std::pair<std::vector<Record *>, int>> FeatureBitsets;
   FeatureBitsets.reserve(Rules.size());
   for (auto &Rule : Rules)
-    FeatureBitsets.push_back(Rule.getRequiredFeatures());
-  llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A,
-                                 const std::vector<Record *> &B) {
-    if (A.size() < B.size())
+    FeatureBitsets.emplace_back(Rule.getRequiredFeatures(),
+                                Rule.getHwModeIdx());
+  llvm::sort(FeatureBitsets,
+             [&](const std::pair<std::vector<Record *>, int> &A,
+                 const std::pair<std::vector<Record *>, int> &B) {
+               if (A.first.size() < B.first.size())
                  return true;
-    if (A.size() > B.size())
+               if (A.first.size() > B.first.size())
                  return false;
-    for (auto [First, Second] : zip(A, B)) {
+               for (auto [First, Second] : zip(A.first, B.first)) {
                  if (First->getName() < Second->getName())
                    return true;
                  if (First->getName() > Second->getName())
                    return false;
                }
-    return false;
+
+               return (A.second < B.second);
              });
   FeatureBitsets.erase(
       std::unique(FeatureBitsets.begin(), FeatureBitsets.end()),
       FeatureBitsets.end());
   OS << "// Feature bitsets.\n"
      << "enum {\n"
      << "  GIFBS_Invalid,\n";
   for (const auto &FeatureBitset : FeatureBitsets) {
-    if (FeatureBitset.empty())
+    if (FeatureBitset.first.empty() && FeatureBitset.second < 0)
       continue;
-    OS << "  " << getNameForFeatureBitset(FeatureBitset) << ",\n";
+    OS << "  "
+       << getNameForFeatureBitset(FeatureBitset.first, FeatureBitset.second)
+       << ",\n";
   }
   OS << "};\n"
      << "constexpr static PredicateBitset FeatureBitsets[] {\n"
      << "  {}, // GIFBS_Invalid\n";
   for (const auto &FeatureBitset : FeatureBitsets) {
-    if (FeatureBitset.empty())
+    if (FeatureBitset.first.empty() && FeatureBitset.second < 0)
       continue;
     OS << "  {";
-    for (const auto &Feature : FeatureBitset) {
+    for (const auto &Feature : FeatureBitset.first) {
       const auto &I = SubtargetFeatures.find(Feature);
       assert(I != SubtargetFeatures.end() && "Didn't import predicate?");
       OS << I->second.getEnumBitName() << ", ";
     }
+    // HwModeIdx
+    if (FeatureBitset.second >= 0) {
+      OS << "Feature_HwMode" << FeatureBitset.second << "Bit, ";
+    }
     OS << "},\n";
   }
   OS << "};\n\n";
 }
 
 void GlobalISelMatchTableExecutorEmitter::emitComplexPredicates(
     raw_ostream &OS, ArrayRef<Record *> ComplexOperandMatchers) {
   // Emit complex predicate table and an enum to reference them with.
void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl(
   emitAdditionalImpl(OS);
   emitRunCustomAction(OS);
   emitMatchTable(OS, Table);
   OS << "#endif // ifdef " << IfDefName << "\n\n";
 }
 
 void GlobalISelMatchTableExecutorEmitter::emitPredicateBitset(
     raw_ostream &OS, StringRef IfDefName) {
+  unsigned Size = SubtargetFeatures.size() + HwModes.size();
   OS << "#ifdef " << IfDefName << "\n"
-     << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size()
-     << ";\n"
+     << "const unsigned MAX_SUBTARGET_PREDICATES = " << Size << ";\n"
      << "using PredicateBitset = "
         "llvm::Bitset<MAX_SUBTARGET_PREDICATES>;\n"
      << "#endif // ifdef " << IfDefName << "\n\n";
 }
 
 void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl(
     raw_ostream &OS, StringRef IfDefName) {
   OS << "#ifdef " << IfDefName << "\n"

llvm/utils/TableGen/SubtargetFeatureInfo.h

struct SubtargetFeatureInfo {
   static std::vector<std::pair<Record *, SubtargetFeatureInfo>>
   getAll(const RecordKeeper &Records);
 
   /// Emit the subtarget feature flag definitions.
   ///
   /// This version emits the bit index for the feature and can therefore support
   /// more than 64 feature bits.
   static void emitSubtargetFeatureBitEnumeration(
-      const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS);
+      const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
+      const std::map<std::string, unsigned> *HwModes = nullptr);
 
   static void emitNameTable(SubtargetFeatureInfoMap &SubtargetFeatures,
                             raw_ostream &OS);
 
   /// Emit the function to compute the list of available features given a
   /// subtarget.
   ///
   /// This version is used for subtarget features defined using Predicate<>
   /// and supports more than 64 feature bits.
   ///
   /// \param TargetName The name of the target as used in class prefixes (e.g.
   ///                   <TargetName>Subtarget)
   /// \param ClassName  The name of the class that will contain the generated
   ///                   functions (including the target prefix.)
   /// \param FuncName   The name of the function to emit.
   /// \param SubtargetFeatures A map of TableGen records to the
   ///                          SubtargetFeatureInfo equivalent.
   /// \param ExtraParams Additional arguments to the generated function.
-  static void
-  emitComputeAvailableFeatures(StringRef TargetName, StringRef ClassName,
-                               StringRef FuncName,
-                               SubtargetFeatureInfoMap &SubtargetFeatures,
-                               raw_ostream &OS, StringRef ExtraParams = "");
+  /// \param HwModes Map of HwMode conditions to check.
+  static void emitComputeAvailableFeatures(
+      StringRef TargetName, StringRef ClassName, StringRef FuncName,
+      const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
+      StringRef ExtraParams = "",
+      const std::map<std::string, unsigned> *HwModes = nullptr);
 
   /// Emit the function to compute the list of available features given a
   /// subtarget.
   ///
   /// This version is used for subtarget features defined using
   /// AssemblerPredicate<> and supports up to 64 feature bits.
   ///
   /// \param TargetName The name of the target as used in class prefixes (e.g.

llvm/utils/TableGen/SubtargetFeatureInfo.cpp

for (Record *Pred : AllPredicates) {
 
     SubtargetFeatures.emplace_back(
         Pred, SubtargetFeatureInfo(Pred, SubtargetFeatures.size()));
   }
   return SubtargetFeatures;
 }
 
 void SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(
-    const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
+    const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
+    const std::map<std::string, unsigned> *HwModes) {
   OS << "// Bits for subtarget features that participate in "
      << "instruction matching.\n";
-  OS << "enum SubtargetFeatureBits : "
-     << getMinimalTypeForRange(SubtargetFeatures.size()) << " {\n";
+  unsigned Size = SubtargetFeatures.size();
+  if (HwModes)
+    Size += HwModes->size();
+
+  OS << "enum SubtargetFeatureBits : " << getMinimalTypeForRange(Size)
+     << " {\n";
   for (const auto &SF : SubtargetFeatures) {
     const SubtargetFeatureInfo &SFI = SF.second;
     OS << "  " << SFI.getEnumBitName() << " = " << SFI.Index << ",\n";
   }
+
+  if (HwModes) {
+    unsigned Offset = SubtargetFeatures.size();
+    for (const auto &M : *HwModes) {
+      OS << "  Feature_HwMode" << M.second << "Bit = " << (M.second + Offset)
+         << ",\n";
+    }
+  }
+
   OS << "};\n\n";
 }
 
 void SubtargetFeatureInfo::emitNameTable(
     SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
   // Need to sort the name table so that lookup by the log of the enum value
   // gives the proper name. More specifically, for a feature of value 1<<n,
   // SubtargetFeatureNames[n] should be the name of the feature.
void SubtargetFeatureInfo::emitNameTable(
   // A small number of targets have no predicates. Null terminate the array to
   // avoid a zero-length array.
   OS << "  nullptr\n"
      << "};\n\n";
 }
 
 void SubtargetFeatureInfo::emitComputeAvailableFeatures(
     StringRef TargetName, StringRef ClassName, StringRef FuncName,
-    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
-    StringRef ExtraParams) {
+    const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
+    StringRef ExtraParams, const std::map<std::string, unsigned> *HwModes) {
   OS << "PredicateBitset " << ClassName << "::\n"
      << FuncName << "(const " << TargetName << "Subtarget *Subtarget";
   if (!ExtraParams.empty())
     OS << ", " << ExtraParams;
   OS << ") const {\n";
   OS << "  PredicateBitset Features;\n";
   for (const auto &SF : SubtargetFeatures) {
     const SubtargetFeatureInfo &SFI = SF.second;
     StringRef CondStr = SFI.TheDef->getValueAsString("CondString");
     assert(!CondStr.empty() && "true predicate should have been filtered");
 
     OS << "  if (" << CondStr << ")\n";
     OS << "    Features.set(" << SFI.getEnumBitName() << ");\n";
   }
+
+  if (HwModes) {
+    for (const auto &M : *HwModes) {
+      OS << "  if (" << M.first << ")\n";
+      OS << "    Features.set(Feature_HwMode" << M.second << "Bit);\n";
+    }
+  }
+
   OS << "  return Features;\n";
   OS << "}\n\n";
 }
 
 // If ParenIfBinOp is true, print a surrounding () if Val uses && or ||.
 static bool emitFeaturesAux(StringRef TargetName, const Init &Val,
                             bool ParenIfBinOp, raw_ostream &OS) {
   if (auto *D = dyn_cast<DefInit>(&Val)) {