Index: include/llvm/CodeGen/GlobalISel/InstructionSelector.h =================================================================== --- include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -107,6 +107,9 @@ /// - InsnID - Instruction ID /// - The predicate to test GIM_CheckAPFloatImmPredicate, + /// Check a memory operation is non-atomic. + /// - InsnID - Instruction ID + GIM_CheckNonAtomic, /// Check the type for the specified operand /// - InsnID - Instruction ID Index: include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h =================================================================== --- include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -209,6 +209,25 @@ return false; break; } + case GIM_CheckNonAtomic: { + int64_t InsnID = MatchTable[CurrentIdx++]; + DEBUG(dbgs() << CurrentIdx << ": GIM_CheckNonAtomic(MIs[" << InsnID + << "])\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert((State.MIs[InsnID]->getOpcode() == TargetOpcode::G_LOAD || + State.MIs[InsnID]->getOpcode() == TargetOpcode::G_STORE) && + "Expected G_LOAD/G_STORE"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (MMO->getOrdering() != AtomicOrdering::NotAtomic) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckType: { int64_t InsnID = MatchTable[CurrentIdx++]; Index: include/llvm/Target/GlobalISel/SelectionDAGCompat.td =================================================================== --- include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -23,6 +23,11 @@ class GINodeEquiv { Instruction I = i; SDNode Node = node; + + // SelectionDAG has separate nodes for atomic and non-atomic memory operations + // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel + // stores this information in the MachineMemoryOperand. + bit CheckMMOIsNonAtomic = 0; } // These are defined in the same order as the G_* instructions. @@ -72,6 +77,17 @@ def : GINodeEquiv; def : GINodeEquiv; +// Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some +// complications that tablegen must take care of. Predicates such as +// isSignExtLoad require that this is not a perfect 1:1 mapping since a +// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. +def : GINodeEquiv { let CheckMMOIsNonAtomic = 1; } +// Broadly speaking G_STORE is equivalent to ISD::STORE but there are some +// complications that tablegen must take care of. Predicates such as +// isTruncStore require that this is not a perfect 1:1 mapping since a +// truncating store is (G_TRUNCATE (G_LOAD x)) in GlobalISel. +def : GINodeEquiv { let CheckMMOIsNonAtomic = 1; } + // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. class GIComplexPatternEquiv { Index: utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- utils/TableGen/GlobalISelEmitter.cpp +++ utils/TableGen/GlobalISelEmitter.cpp @@ -979,6 +979,7 @@ enum PredicateKind { IPM_Opcode, IPM_ImmPredicate, + IPM_NonAtomicMMO, }; PredicateKind Kind; @@ -1107,6 +1108,24 @@ } }; +/// Generates code to check that a memory instruction has a non-atomic MachineMemoryOperand. +class NonAtomicMMOPredicateMatcher : public InstructionPredicateMatcher { +public: + NonAtomicMMOPredicateMatcher() + : InstructionPredicateMatcher(IPM_NonAtomicMMO) {} + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_NonAtomicMMO; + } + + void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnVarID) const override { + Table << MatchTable::Opcode("GIM_CheckNonAtomic") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::LineBreak; + } +}; + /// Generates code to check that a set of predicates and operands match for a /// particular instruction. /// @@ -1974,7 +1993,7 @@ /// Keep track of the equivalence between SDNodes and Instruction. /// This is defined using 'GINodeEquiv' in the target description. - DenseMap NodeEquivs; + DenseMap NodeEquivs; /// Keep track of the equivalence between ComplexPattern's and /// GIComplexOperandMatcher. Map entries are specified by subclassing @@ -1985,7 +2004,7 @@ SubtargetFeatureInfoMap SubtargetFeatures; void gatherNodeEquivs(); - const CodeGenInstruction *findNodeEquiv(Record *N) const; + Record *findNodeEquiv(Record *N) const; Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); Expected @@ -2022,8 +2041,7 @@ void GlobalISelEmitter::gatherNodeEquivs() { assert(NodeEquivs.empty()); for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv")) - NodeEquivs[Equiv->getValueAsDef("Node")] = - &Target.getInstruction(Equiv->getValueAsDef("I")); + NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv; assert(ComplexPatternEquivs.empty()); for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) { @@ -2034,7 +2052,7 @@ } } -const CodeGenInstruction *GlobalISelEmitter::findNodeEquiv(Record *N) const { +Record *GlobalISelEmitter::findNodeEquiv(Record *N) const { return NodeEquivs.lookup(N); } @@ -2061,6 +2079,7 @@ GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, const TreePatternNode *Src, unsigned &TempOpIdx) const { + Record *SrcGIEquivOrNull = nullptr; const CodeGenInstruction *SrcGIOrNull = nullptr; // Start with the defined operands (i.e., the results of the root operator). @@ -2076,14 +2095,14 @@ return failedImport( "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); } else { - SrcGIOrNull = findNodeEquiv(Src->getOperator()); - if (!SrcGIOrNull) + SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); + if (!SrcGIEquivOrNull) return failedImport("Pattern operator lacks an equivalent Instruction" + explainOperator(Src->getOperator())); - auto &SrcGI = *SrcGIOrNull; + SrcGIOrNull = &Target.getInstruction(SrcGIEquivOrNull->getValueAsDef("I")); // The operators look good: match the opcode - InsnMatcher.addPredicate(&SrcGI); + InsnMatcher.addPredicate(SrcGIOrNull); } unsigned OpIdx = 0; @@ -2113,6 +2132,8 @@ return failedImport("Src pattern child has predicate (" + explainPredicates(Src) + ")"); } + if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) + InsnMatcher.addPredicate(); if (Src->isLeaf()) { Init *SrcInit = Src->getLeafValue();