diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -109,6 +109,13 @@ /// - OpIdx - Operand index GIM_RecordInsn, + /// Variant of GIM_RecordInsn that ignores COPY instructions when + /// possible. + /// - NewInsnID - Instruction ID to define + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + GIM_RecordInsnIgnoreCopies, + /// Check the feature bits /// - Expected features GIM_CheckFeatures, @@ -364,7 +371,8 @@ /// reading from a specific operand. /// - InsnID - Instruction ID to modify /// - OldInsnID - Instruction ID to get the matched operand from - /// - OpIdx - Operand index in OldInsnID the render function should read from.. + /// - OpIdx - Operand index in OldInsnID the render function should read + /// from.. /// - RendererFnID - Custom renderer function to call GIR_CustomOperandRenderer, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -101,7 +101,9 @@ break; } + case GIM_RecordInsnIgnoreCopies: case GIM_RecordInsn: { + const bool IgnoreCopies = (MatcherOpcode == GIM_RecordInsnIgnoreCopies); int64_t NewInsnID = MatchTable[CurrentIdx++]; int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; @@ -126,7 +128,12 @@ break; } - MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); + MachineInstr *NewMI; + if (IgnoreCopies) + NewMI = getDefIgnoringCopies(MO.getReg(), MRI); + else + NewMI = MRI.getVRegDef(MO.getReg()); + if ((size_t)NewInsnID < State.MIs.size()) State.MIs[NewInsnID] = NewMI; else { @@ -135,9 +142,10 @@ State.MIs.push_back(NewMI); } DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), - dbgs() << CurrentIdx << ": MIs[" << NewInsnID - << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx - << ")\n"); + dbgs() << CurrentIdx << ": MIs[" << NewInsnID << "] = " + << (IgnoreCopies ? "GIM_RecordInsnIgnoreCopies" + : "GIM_RecordInsn") + << "(" << InsnID << ", " << OpIdx << ")\n"); break; } @@ -827,8 +835,18 @@ << OtherInsnID << "][" << OtherOpIdx << "])\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( - State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { + + // FIXME: Gross hack, needs a better solution here. + MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); + MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); + if (Op.isReg() && OtherOp.isReg()) { + MachineInstr *MI = getDefIgnoringCopies(Op.getReg(), MRI); + MachineInstr *OtherMI = getDefIgnoringCopies(OtherOp.getReg(), MRI); + if (MI && MI == OtherMI) + break; + } + + if (!Op.isIdenticalTo(OtherOp)) { if (handleReject() == RejectAndGiveUp) return false; } diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1850,6 +1850,10 @@ list ResultInstrs = resultInstrs; list Predicates = []; // See class Instruction in Target.td. int AddedComplexity = 0; // See class Instruction in Target.td. + + // When this is set, the GISel matcher will look through COPY instructions + // before checking opcodes for instruction operands. + bit GISelIgnoreCopies = ?; } // Pat - A simple (but common) form of a pattern, which produces a simple result diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -111,7 +111,9 @@ } class AMDGPUPat : Pat, - PredicateControl; + PredicateControl { + let GISelIgnoreCopies = 1; +} let RecomputePerFunction = 1 in { def FP16Denormals : Predicate<"MF->getInfo()->getMode().allFP64FP16Denormals()">; diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// class GCNPat : Pat, GCNPredicateControl { - + let GISelIgnoreCopies = 1; } class UniformSextInreg : PatFrag< diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -2479,13 +2479,15 @@ class InstructionOperandMatcher : public OperandPredicateMatcher { protected: std::unique_ptr InsnMatcher; + bool IgnoreCopies; public: InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx, RuleMatcher &Rule, StringRef SymbolicName, - bool NumOpsCheck = true) + bool IgnoreCopies, bool NumOpsCheck = true) : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx), - InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)) {} + InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)), + IgnoreCopies(IgnoreCopies) {} static bool classof(const PredicateMatcher *P) { return P->getKind() == OPM_Instruction; @@ -2495,8 +2497,9 @@ void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const { const unsigned NewInsnVarID = InsnMatcher->getInsnVarID(); - Table << MatchTable::Opcode("GIM_RecordInsn") - << MatchTable::Comment("DefineMI") + const StringRef Opcode = + IgnoreCopies ? "GIM_RecordInsnIgnoreCopies" : "GIM_RecordInsn"; + Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("DefineMI") << MatchTable::IntValue(NewInsnVarID) << MatchTable::Comment("MI") << MatchTable::IntValue(getInsnVarID()) << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx()) @@ -3618,16 +3621,16 @@ const TreePatternNode *N) const; Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); - Expected - createAndImportSelDAGMatcher(RuleMatcher &Rule, - InstructionMatcher &InsnMatcher, - const TreePatternNode *Src, unsigned &TempOpIdx); + Expected createAndImportSelDAGMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx, bool IgnoreCopies); Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const; Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher, const TreePatternNode *SrcChild, bool OperandIsAPointer, bool OperandIsImmArg, - unsigned OpIdx, unsigned &TempOpIdx); + bool IgnoreCopies, unsigned OpIdx, + unsigned &TempOpIdx); Expected createAndImportInstructionRenderer( RuleMatcher &M, InstructionMatcher &InsnMatcher, @@ -3982,7 +3985,7 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( RuleMatcher &Rule, InstructionMatcher &InsnMatcher, - const TreePatternNode *Src, unsigned &TempOpIdx) { + const TreePatternNode *Src, unsigned &TempOpIdx, bool IgnoreCopies) { Record *SrcGIEquivOrNull = nullptr; const CodeGenInstruction *SrcGIOrNull = nullptr; @@ -4137,11 +4140,11 @@ TreePatternNode *ValueChild = Src->getChild(1); if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true, - false, 1, TempOpIdx)) + false, IgnoreCopies, 1, TempOpIdx)) return std::move(Error); if (auto Error = importChildMatcher(Rule, InsnMatcher, ValueChild, false, - false, 0, TempOpIdx)) + false, IgnoreCopies, 0, TempOpIdx)) return std::move(Error); return InsnMatcher; } @@ -4190,9 +4193,9 @@ OperandIsImmArg |= II->isParamImmArg(i - 1); } - if (auto Error = - importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer, - OperandIsImmArg, OpIdx++, TempOpIdx)) + if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild, + OperandIsAPointer, OperandIsImmArg, + IgnoreCopies, OpIdx++, TempOpIdx)) return std::move(Error); } } @@ -4230,10 +4233,13 @@ return SrcChildName; } -Error GlobalISelEmitter::importChildMatcher( - RuleMatcher &Rule, InstructionMatcher &InsnMatcher, - const TreePatternNode *SrcChild, bool OperandIsAPointer, - bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) { +Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, + InstructionMatcher &InsnMatcher, + const TreePatternNode *SrcChild, + bool OperandIsAPointer, + bool OperandIsImmArg, + bool IgnoreCopies, unsigned OpIdx, + unsigned &TempOpIdx) { Record *PhysReg = nullptr; std::string SrcChildName = std::string(getSrcChildName(SrcChild, PhysReg)); @@ -4320,7 +4326,7 @@ } auto MaybeInsnOperand = OM.addPredicate( - InsnMatcher.getRuleMatcher(), SrcChild->getName()); + InsnMatcher.getRuleMatcher(), SrcChild->getName(), IgnoreCopies); if (!MaybeInsnOperand) { // This isn't strictly true. If the user were to provide exactly the same // matchers as the original operand then we could allow it. However, it's @@ -4331,7 +4337,7 @@ // Map the node to a gMIR instruction. InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand; auto InsnMatcherOrError = createAndImportSelDAGMatcher( - Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); + Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx, IgnoreCopies); if (auto Error = InsnMatcherOrError.takeError()) return Error; @@ -4409,7 +4415,8 @@ const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV"; if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") { auto MaybeInsnOperand = OM.addPredicate( - InsnMatcher.getRuleMatcher(), SrcChild->getName(), false); + InsnMatcher.getRuleMatcher(), SrcChild->getName(), IgnoreCopies, + false); InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand; ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode(); @@ -5209,8 +5216,16 @@ // before their first use.) InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName()); unsigned TempOpIdx = 0; - auto InsnMatcherOrError = - createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx); + + bool IgnoreCopies = false; + if (P.getSrcRecord()->isSubClassOf("Pattern")) { + bool Unset = false; + IgnoreCopies = + P.getSrcRecord()->getValueAsBitOrUnset("GISelIgnoreCopies", Unset); + } + + auto InsnMatcherOrError = createAndImportSelDAGMatcher( + M, InsnMatcherTemp, Src, TempOpIdx, IgnoreCopies); if (auto Error = InsnMatcherOrError.takeError()) return std::move(Error); InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();