[globalisel][tablegen] Implement unindexed load, non-extending load, and MemVT checks

[dsanders]

Summary

This includes some context-sensitivity in the MVT to LLT conversion so that
pointer types are tested correctly.
FIXME: I'm not happy with the way this is done since everything is a

special-case. I've yet to find a reasonable way to implement it.

select-load.mir fails because <1 x s64> loads in tablegen get priority over s64
loads. This is fixed in the next patch and as such they should be committed
together, I've posted them separately to help with the review.

Depends on D37456

[dsanders]

Rebase and ping

[qcolombet]

include/llvm/CodeGen/GlobalISel/InstructionSelector.h

124

Doesn't the pointer size depend on the address space?
Thus, given we specific the size this is not really any address space, isn't it?

Basically what I am saying is either don't put the size or change the name of the operator.
BTW, shouldn't we already check the size when checking the type?

include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

247

Could you add a tablegen test case for this operator?

[dsanders]

include/llvm/CodeGen/GlobalISel/InstructionSelector.h

124

Hmm, I agree that it makes sense for pointer sizes to vary between address spaces. However, iPTR doesn't do anything about address spaces so I don't think SelectionDAG rules account for it. I'll have another look and see what I can find out.

BTW, shouldn't we already check the size when checking the type?

This opcode is used instead of GIM_CheckType for pointers. Pointers are a bit tricky since we have to call a target hook to find out the size of a pointer so we don't know the type until run-time.

include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

247

Ok

[dsanders]

Refresh before commit

include/llvm/CodeGen/GlobalISel/InstructionSelector.h

enum {
   /// - InsnID - Instruction ID
   GIM_CheckNonAtomic,
 
   /// Check the type for the specified operand
   /// - InsnID - Instruction ID
   /// - OpIdx - Operand index
   /// - Expected type
   GIM_CheckType,
+  /// Check the type of a pointer to any address space.
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  /// - SizeInBits - The size of the pointer value in bits.
+  GIM_CheckPointerToAny,
   /// Check the register bank for the specified operand
   /// - InsnID - Instruction ID
   /// - OpIdx - Operand index
   /// - Expected register bank (specified as a register class)
   GIM_CheckRegBankForClass,
   /// Check the operand matches a complex predicate
   /// - InsnID - Instruction ID
   /// - OpIdx - Operand index

include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

case GIM_CheckType: {
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
       if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) !=
           MatcherInfo.TypeObjects[TypeID]) {
         if (handleReject() == RejectAndGiveUp)
           return false;
       }
       break;
     }
-
+    case GIM_CheckPointerToAny: {
+      int64_t InsnID = MatchTable[CurrentIdx++];
+      int64_t OpIdx = MatchTable[CurrentIdx++];
+      int64_t SizeInBits = MatchTable[CurrentIdx++];
+      DEBUG(dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" << InsnID
+                   << "]->getOperand(" << OpIdx
+                   << "), SizeInBits=" << SizeInBits << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      const LLT &Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg());
+      if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) {
+        if (handleReject() == RejectAndGiveUp)
+          return false;
+      }
+      break;
+    }
     case GIM_CheckRegBankForClass: {
       int64_t InsnID = MatchTable[CurrentIdx++];
       int64_t OpIdx = MatchTable[CurrentIdx++];
       int64_t RCEnum = MatchTable[CurrentIdx++];
       DEBUG(dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" << InsnID
                    << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum
                    << ")\n");
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

test/CodeGen/AArch64/GlobalISel/select-load.mir

--- |
   define void @load_s16_fpr(i16* %addr) { ret void }
   define void @load_s8_fpr(i8* %addr) { ret void }
 
   define void @load_gep_8_s64_fpr(i64* %addr) { ret void }
   define void @load_gep_16_s32_fpr(i32* %addr) { ret void }
   define void @load_gep_64_s16_fpr(i16* %addr) { ret void }
   define void @load_gep_32_s8_fpr(i8* %addr) { ret void }
 
+  define void @load_v2s32(i64 *%addr) { ret void }
 ...
 
 ---
 # CHECK-LABEL: name: load_s64_gpr
 name:            load_s64_gpr
 legalized:       true
 regBankSelected: true
 
bb.0:
     liveins: %x0
 
     %0(p0) = COPY %x0
     %1(s64) = G_CONSTANT i64 32
     %2(p0) = G_GEP %0, %1
     %3(s8) = G_LOAD %2 :: (load 1 from %ir.addr)
     %b0 = COPY %3
 ...
+---
+# CHECK-LABEL: name: load_v2s32
+name:            load_v2s32
+legalized:       true
+regBankSelected: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: gpr64sp, preferred-register: '' }
+# CHECK-NEXT:  - { id: 1, class: fpr64, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: fpr }
+
+# CHECK:  body:
+# CHECK:    %0 = COPY %x0
+# CHECK:    %1 = LD1Onev2s %0
+# CHECK:    %d0 = COPY %1
+body:             |
+  bb.0:
+    liveins: %x0
+
+    %0(p0) = COPY %x0
+    %1(<2 x s32>) = G_LOAD %0 :: (load 4 from %ir.addr)
+    %d0 = COPY %1(<2 x s32>)
+...

utils/TableGen/GlobalISelEmitter.cpp

void emitCxxEnumValue(raw_ostream &OS) const {
     if (Ty.isScalar()) {
       OS << "GILLT_s" << Ty.getSizeInBits();
       return;
     }
     if (Ty.isVector()) {
       OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits();
       return;
     }
+    if (Ty.isPointer()) {
+      OS << "GILLT_p" << Ty.getAddressSpace();
+      if (Ty.getSizeInBits() > 0)
+        OS << "s" << Ty.getSizeInBits();
+      return;
+    }
     llvm_unreachable("Unhandled LLT");
   }
 
   void emitCxxConstructorCall(raw_ostream &OS) const {
     if (Ty.isScalar()) {
       OS << "LLT::scalar(" << Ty.getSizeInBits() << ")";
       return;
     }
     if (Ty.isVector()) {
       OS << "LLT::vector(" << Ty.getNumElements() << ", "
          << Ty.getScalarSizeInBits() << ")";
       return;
     }
+    if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
+      OS << "LLT::pointer(" << Ty.getAddressSpace() << ", "
+         << Ty.getSizeInBits() << ")";
+      return;
+    }
     llvm_unreachable("Unhandled LLT");
   }
 
   const LLT &get() const { return Ty; }
 
   /// This ordering is used for std::unique() and std::sort(). There's no
   /// particular logic behind the order but either A < B or B < A must be
   /// true if A != B.
public:
   }
 };
 
 class InstructionMatcher;
 /// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for
 /// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...).
 static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
   MVT VT(SVT);
+
   if (VT.isVector() && VT.getVectorNumElements() != 1)
     return LLTCodeGen(
         LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
+
   if (VT.isInteger() || VT.isFloatingPoint())
     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
   return None;
 }
 
 static std::string explainPredicates(const TreePatternNode *N) {
   std::string Explanation = "";
   StringRef Separator = "";
static Error isTrivialOperatorNode(const TreePatternNode *N) {
   bool HasUnsupportedPredicate = false;
   for (const auto &Predicate : N->getPredicateFns()) {
     if (Predicate.isAlwaysTrue())
       continue;
 
     if (Predicate.isImmediatePattern())
       continue;
 
+    if (Predicate.isUnindexedLoad())
+      continue;
+
+    if (Predicate.isNonExtLoad())
+      continue;
     HasUnsupportedPredicate = true;
     Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
     Separator = ", ";
     Explanation += (Separator + "first-failing:" +
                     Predicate.getOrigPatFragRecord()->getRecord()->getName())
                        .str();
     break;
   }
public:
   enum PredicateKind {
     OPM_Tie,
     OPM_ComplexPattern,
     OPM_IntrinsicID,
     OPM_Instruction,
     OPM_Int,
     OPM_LiteralInt,
     OPM_LLT,
+    OPM_PointerToAny,
     OPM_RegBank,
     OPM_MBB,
   };
 
 protected:
   PredicateKind Kind;
 
 public:
Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI")
           << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type")
           << MatchTable::NamedValue(Ty.getCxxEnumValue())
           << MatchTable::LineBreak;
   }
 };
 
 std::set<LLTCodeGen> LLTOperandMatcher::KnownTypes;
 
+/// Generates code to check that an operand is a pointer to any address space.
+///
+/// In SelectionDAG, the types did not describe pointers or address spaces. As a
+/// result, iN is used to describe a pointer of N bits to any address space and
+/// PatFrag predicates are typically used to constrain the address space. There's
+/// no reliable means to derive the missing type information from the pattern so
+/// imported rules must test the components of a pointer separately.
+///
+/// SizeInBits must be non-zero and the matched pointer must be that size.
+/// TODO: Add support for iPTR via SizeInBits==0 and a subtarget query.
+class PointerToAnyOperandMatcher : public OperandPredicateMatcher {
+protected:
+  unsigned SizeInBits;
+
+public:
+  PointerToAnyOperandMatcher(unsigned SizeInBits)
+      : OperandPredicateMatcher(OPM_PointerToAny), SizeInBits(SizeInBits) {}
+
+  static bool classof(const OperandPredicateMatcher *P) {
+    return P->getKind() == OPM_PointerToAny;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule,
+                            unsigned InsnVarID, unsigned OpIdx) const override {
+    Table << MatchTable::Opcode("GIM_CheckPointerToAny") << MatchTable::Comment("MI")
+          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
+          << MatchTable::IntValue(OpIdx) << MatchTable::Comment("SizeInBits")
+          << MatchTable::IntValue(SizeInBits) << MatchTable::LineBreak;
+  }
+};
+
 /// Generates code to check that an operand is a particular target constant.
 class ComplexPatternOperandMatcher : public OperandPredicateMatcher {
 protected:
   const OperandMatcher &Operand;
   const Record &TheDef;
 
   unsigned getAllocatedTemporariesBaseID() const;
 
public:
 
   std::string getOperandExpr(unsigned InsnVarID) const {
     return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" +
            llvm::to_string(OpIdx) + ")";
   }
 
   InstructionMatcher &getInstructionMatcher() const { return Insn; }
 
+  Error addTypeCheckPredicate(const TypeSetByHwMode &VTy,
+                              bool OperandIsAPointer) {
+    auto OpTyOrNone = VTy.isMachineValueType()
+                          ? MVTToLLT(VTy.getMachineValueType().SimpleTy)
+                          : None;
+    if (!OpTyOrNone)
+      return failedImport("unsupported type");
+
+    if (OperandIsAPointer)
+      addPredicate<PointerToAnyOperandMatcher>(
+          OpTyOrNone->get().getSizeInBits());
+    else
+      addPredicate<LLTOperandMatcher>(*OpTyOrNone);
+    return Error::success();
+  }
+
   /// Emit MatchTable opcodes to capture instructions into the MIs table.
   void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule,
                           unsigned InsnVarID) const {
     for (const auto &Predicate : predicates())
       Predicate->emitCaptureOpcodes(Table, Rule, InsnVarID, OpIdx);
   }
 
   /// Emit MatchTable opcodes that test whether the instruction named in
private:
 
   Error importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates);
   Expected<InstructionMatcher &> createAndImportSelDAGMatcher(
       RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
       const TreePatternNode *Src, unsigned &TempOpIdx) const;
   Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R,
                                            unsigned &TempOpIdx) const;
   Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
-                           const TreePatternNode *SrcChild, unsigned OpIdx,
+                           const TreePatternNode *SrcChild,
+                           bool OperandIsAPointer, unsigned OpIdx,
                            unsigned &TempOpIdx) const;
   Expected<BuildMIAction &>
   createAndImportInstructionRenderer(RuleMatcher &M, const TreePatternNode *Dst,
                                      const InstructionMatcher &InsnMatcher);
   Error importExplicitUseRenderer(RuleMatcher &Rule,
                                   BuildMIAction &DstMIBuilder,
                                   TreePatternNode *DstChild,
                                   const InstructionMatcher &InsnMatcher) const;
if (Src->isLeaf()) {
     SrcGIOrNull = &Target.getInstruction(SrcGIEquivOrNull->getValueAsDef("I"));
 
     // The operators look good: match the opcode
     InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull);
   }
 
   unsigned OpIdx = 0;
   for (const TypeSetByHwMode &VTy : Src->getExtTypes()) {
-    auto OpTyOrNone = VTy.isMachineValueType()
-                          ? MVTToLLT(VTy.getMachineValueType().SimpleTy)
-                          : None;
-    if (!OpTyOrNone)
-      return failedImport(
-          "Result of Src pattern operator has an unsupported type");
-
     // Results don't have a name unless they are the root node. The caller will
     // set the name if appropriate.
     OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);
-    OM.addPredicate<LLTOperandMatcher>(*OpTyOrNone);
+    if (auto Error = OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */))
+      return failedImport(toString(std::move(Error)) +
+                          " for result of Src pattern operator");
   }
 
   for (const auto &Predicate : Src->getPredicateFns()) {
     if (Predicate.isAlwaysTrue())
       continue;
 
     if (Predicate.isImmediatePattern()) {
       InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(Predicate);
       continue;
     }
 
+    // No check required. A G_LOAD is an unindexed load.
+    if (Predicate.isUnindexedLoad())
+      continue;
+
+    // No check required. G_LOAD by itself is a non-extending load.
+    if (Predicate.isNonExtLoad())
+      continue;
+
+    if (Predicate.getLoadMemoryVT() != nullptr) {
+      Optional<LLTCodeGen> MemTyOrNone =
+          MVTToLLT(getValueType(Predicate.getLoadMemoryVT()));
+
+      if (!MemTyOrNone)
+        return failedImport("MemVT could not be converted to LLT");
+
+      InsnMatcher.getOperand(0).addPredicate<LLTOperandMatcher>(MemTyOrNone.getValue());
+      continue;
+    }
+
     return failedImport("Src pattern child has predicate (" +
                         explainPredicates(Src) + ")");
   }
   if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))
     InsnMatcher.addPredicate<NonAtomicMMOPredicateMatcher>();
 
   if (Src->isLeaf()) {
     Init *SrcInit = Src->getLeafValue();
if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" ||
       InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);
       return InsnMatcher;
     }
 
     // Match the used operands (i.e. the children of the operator).
     for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) {
       TreePatternNode *SrcChild = Src->getChild(i);
 
+      // SelectionDAG allows pointers to be represented with iN since it doesn't
+      // distinguish between pointers and integers but they are different types in GlobalISel.
+      // Coerce integers to pointers to address space 0 if the context indicates a pointer.
+      // TODO: Find a better way to do this, SDTCisPtrTy?
+      bool OperandIsAPointer =
+          SrcGIOrNull->TheDef->getName() == "G_LOAD" && i == 0;
+
       // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
       // following the defs is an intrinsic ID.
       if ((SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||
            SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS") &&
           i == 0) {
         if (const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP)) {
           OperandMatcher &OM =
               InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx);
           if (!OM.isTied())
             OM.addPredicate<IntrinsicIDOperandMatcher>(II);
           continue;
         }
 
         return failedImport("Expected IntInit containing instrinsic ID)");
       }
 
-      if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild, OpIdx++,
-                                          TempOpIdx))
+      if (auto Error =
+              importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,
+                                 OpIdx++, TempOpIdx))
         return std::move(Error);
     }
   }
 
   return InsnMatcher;
 }
 
 Error GlobalISelEmitter::importComplexPatternOperandMatcher(
     OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const {
   const auto &ComplexPattern = ComplexPatternEquivs.find(R);
   if (ComplexPattern == ComplexPatternEquivs.end())
     return failedImport("SelectionDAG ComplexPattern (" + R->getName() +
                         ") not mapped to GlobalISel");
 
   OM.addPredicate<ComplexPatternOperandMatcher>(OM, *ComplexPattern->second);
   TempOpIdx++;
   return Error::success();
 }
 
 Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule,
                                             InstructionMatcher &InsnMatcher,
                                             const TreePatternNode *SrcChild,
+                                            bool OperandIsAPointer,
                                             unsigned OpIdx,
                                             unsigned &TempOpIdx) const {
   OperandMatcher &OM =
       InsnMatcher.addOperand(OpIdx, SrcChild->getName(), TempOpIdx);
   if (OM.isTied())
     return Error::success();
 
   ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild->getExtTypes();
   if (ChildTypes.size() != 1)
     return failedImport("Src pattern child has multiple results");
 
   // Check MBB's before the type check since they are not a known type.
   if (!SrcChild->isLeaf()) {
     if (SrcChild->getOperator()->isSubClassOf("SDNode")) {
       auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild->getOperator());
       if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") {
         OM.addPredicate<MBBOperandMatcher>();
         return Error::success();
       }
     }
   }
 
-  Optional<LLTCodeGen> OpTyOrNone = None;
-  if (ChildTypes.front().isMachineValueType())
-    OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
-  if (!OpTyOrNone)
-    return failedImport("Src operand has an unsupported type (" + to_string(*SrcChild) + ")");
-  OM.addPredicate<LLTOperandMatcher>(*OpTyOrNone);
+  if (auto Error =
+          OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer))
+    return failedImport(toString(std::move(Error)) + " for Src operand (" +
+                        to_string(*SrcChild) + ")");
 
   // Check for nested instructions.
   if (!SrcChild->isLeaf()) {
     if (SrcChild->getOperator()->isSubClassOf("ComplexPattern")) {
       // When a ComplexPattern is used as an operator, it should do the same
       // thing as when used as a leaf. However, the children of the operator
       // name the sub-operands that make up the complex operand and we must
       // prepare to reference them in the renderer too.