diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -103,11 +103,13 @@ /// - JumpTable... - (UpperBound - LowerBound) (at least 2) jump targets GIM_SwitchType, - /// Record the specified instruction + /// Record the specified instruction. + /// The IgnoreCopies variant ignores COPY instructions. /// - NewInsnID - Instruction ID to define /// - InsnID - Instruction ID /// - OpIdx - Operand index GIM_RecordInsn, + GIM_RecordInsnIgnoreCopies, /// Check the feature bits /// - Expected features @@ -262,11 +264,14 @@ GIM_CheckIsSafeToFold, /// Check the specified operands are identical. + /// The IgnoreCopies variant looks through COPY instructions before + /// comparing the operands. /// - InsnID - Instruction ID /// - OpIdx - Operand index /// - OtherInsnID - Other instruction ID /// - OtherOpIdx - Other operand index GIM_CheckIsSameOperand, + GIM_CheckIsSameOperandIgnoreCopies, /// Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some /// named operands that will be recorded in RecordedOperands. Names of these @@ -364,7 +369,8 @@ /// reading from a specific operand. /// - InsnID - Instruction ID to modify /// - OldInsnID - Instruction ID to get the matched operand from - /// - OpIdx - Operand index in OldInsnID the render function should read from.. + /// - OpIdx - Operand index in OldInsnID the render function should read + /// from.. /// - RendererFnID - Custom renderer function to call GIR_CustomOperandRenderer, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -101,7 +101,8 @@ break; } - case GIM_RecordInsn: { + case GIM_RecordInsn: + case GIM_RecordInsnIgnoreCopies: { int64_t NewInsnID = MatchTable[CurrentIdx++]; int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; @@ -126,7 +127,12 @@ break; } - MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); + MachineInstr *NewMI; + if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) + NewMI = getDefIgnoringCopies(MO.getReg(), MRI); + else + NewMI = MRI.getVRegDef(MO.getReg()); + if ((size_t)NewInsnID < State.MIs.size()) State.MIs[NewInsnID] = NewMI; else { @@ -816,7 +822,8 @@ } break; } - case GIM_CheckIsSameOperand: { + case GIM_CheckIsSameOperand: + case GIM_CheckIsSameOperandIgnoreCopies: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; int64_t OtherInsnID = MatchTable[CurrentIdx++]; @@ -827,8 +834,20 @@ << OtherInsnID << "][" << OtherOpIdx << "])\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( - State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { + + MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); + MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); + + if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { + if (Op.isReg() && OtherOp.isReg()) { + MachineInstr *MI = getDefIgnoringCopies(Op.getReg(), MRI); + MachineInstr *OtherMI = getDefIgnoringCopies(OtherOp.getReg(), MRI); + if (MI && MI == OtherMI) + break; + } + } + + if (!Op.isIdenticalTo(OtherOp)) { if (handleReject() == RejectAndGiveUp) return false; } diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -890,6 +890,12 @@ ValueType ScalarMemoryVT = ?; } +// Patterns and PatFrags can also subclass GISelFlags to set flags that affect +// how GlobalISel behaves when matching them. +class GISelFlags { + bit GIIgnoreCopies = ?; +} + // PatFrag - A version of PatFrags matching only a single fragment. class PatFrag diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -104,7 +104,10 @@ } class AMDGPUPat : Pat, - PredicateControl; + PredicateControl, GISelFlags; + +let GIIgnoreCopies = 1 in +class AMDGPUPatIgnoreCopies : AMDGPUPat; let RecomputePerFunction = 1 in { def FP16Denormals : Predicate<"MF->getInfo()->getMode().allFP64FP16Denormals()">; diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -2051,52 +2051,53 @@ }] >; + // Definition from ISA doc: // (y & x) | (z & ~x) -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))), (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), - (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), - (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) + (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), + (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) >; // (y & C) | (z & ~C) -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (BFIImm32 i32:$x, i32:$y, i32:$z), (V_BFI_B32_e64 VSrc_b32:$x, VSrc_b32:$y, VSrc_b32:$z) >; // 64-bit version -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))), (REG_SEQUENCE VReg_64, (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, + (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), + (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) + (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), + (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) >; // SHA-256 Ch function // z ^ (x & (y ^ z)) -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag i32:$z, (and i32:$x, (xor i32:$y, i32:$z))), (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), - (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), - (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) + (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), + (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) >; // 64-bit version -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), (REG_SEQUENCE VReg_64, (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, + (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), + (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) + (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), + (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) >; def : AMDGPUPat < @@ -3197,27 +3198,27 @@ // SHA-256 Ma patterns // ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))), (V_BFI_B32_e64 (V_XOR_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)), - (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32), - (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)) + (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32), + (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)) >; -def : AMDGPUPat < +def : AMDGPUPatIgnoreCopies < (DivergentBinFrag (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))), (REG_SEQUENCE VReg_64, (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub0)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), sub0, + (i32 (EXTRACT_SUBREG VReg_64:$z, sub0)), + (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), sub0, (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), - (i32 (EXTRACT_SUBREG VReg_64:$z, sub1)), - (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), sub1) + (i32 (EXTRACT_SUBREG VReg_64:$z, sub1)), + (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), sub1) >; multiclass IntMed3Pat; + +class SrlPF: PatFrag< + (ops node:$PATFRAG_Src0, node:$src1), + (srl $PATFRAG_Src0, $src1)>, GISelFlags { + let GIIgnoreCopies = IC; +} + +// GIIgnoreCopies on Pattern +// MIs[1] should be using IgnoreCopies variants. +let GIIgnoreCopies = 1 in +def : Pat< + (i32 (sub (mul i32:$src0, i32:$src0), i32:$src1)), + (InstThreeOperands GPR32:$src0, GPR32:$src1, GPR32:$src1) +>, GISelFlags; + +// GIIgnoreCopies set on "root" PatFrag. +// MIs[1] and MIs[2] should be using IgnoreCopies variants. +def : Pat< + (i32 (SrlPF<1> (shl (mul i32:$src0, i32:$src0), i32:$src1), i32:$src0)), + (InstThreeOperands GPR32:$src0, GPR32:$src1, GPR32:$src1) +>; + +// GIIgnoreCopies set on "root" PatFrag, but a children PatFrag forces it back to zero. +// MIs[1] should be using IgnoreCopies variants. +// MIs[2] should NOT be using them. +def : Pat< + (i32 (SrlPF<1> (SrlPF<0> (add i32:$src0, i32:$src0), i32:$src1), i32:$src0)), + (InstThreeOperands GPR32:$src0, GPR32:$src1, GPR32:$src1) +>; + +// CHECK: GIM_Try +// CHECK: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_LSHR +// CHECK: GIM_RecordInsn, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2] +// CHECK: GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_ADD +// CHECK: GIM_CheckIsSameOperand, /*MI*/2, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1 +// CHECK: GIM_CheckIsSameOperandIgnoreCopies, /*MI*/0, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1 +// CHECK: // (srl:{ *:[i32] } (srl:{ *:[i32] } (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1), i32:{ *:[i32] }:$src0) +// CHECK: GIR_Done +// CHECK: GIM_Try +// CHECK: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SHL +// CHECK: GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2] +// CHECK: GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_MUL +// CHECK: GIM_CheckIsSameOperandIgnoreCopies, /*MI*/2, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1 +// CHECK: GIM_CheckIsSameOperandIgnoreCopies, /*MI*/0, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1 +// CHECK: // (srl:{ *:[i32] } (shl:{ *:[i32] } (mul:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1), i32:{ *:[i32] }:$src0) +// CHECK: GIR_Done +// CHECK: GIM_Try +// CHECK: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_MUL +// CHECK: GIM_CheckIsSameOperandIgnoreCopies, /*MI*/1, /*OpIdx*/2, /*OtherMI*/1, /*OtherOpIdx*/1 +// CHECK: // (sub:{ *:[i32] } (mul:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1) => (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src1) +// CHECK: GIR_Done \ No newline at end of file diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h --- a/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h @@ -665,6 +665,10 @@ std::vector Children; + /// If this was instantiated from a PatFrag node, and the PatFrag was derived + /// from "GISelFlags": the original Record derived from GISelFlags. + const Record *GISelFlags = nullptr; + public: TreePatternNode(Record *Op, std::vector Ch, unsigned NumResults) @@ -794,6 +798,9 @@ /// marked isCommutative. bool isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const; + void setGISelFlagsRecord(const Record *R) { GISelFlags = R; } + const Record *getGISelFlagsRecord() const { return GISelFlags; } + void print(raw_ostream &OS) const; void dump() const; diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp --- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp +++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp @@ -2038,6 +2038,7 @@ New->setNamesAsPredicateArg(getNamesAsPredicateArg()); New->Types = Types; New->setPredicateCalls(getPredicateCalls()); + New->setGISelFlagsRecord(getGISelFlagsRecord()); New->setTransformFn(getTransformFn()); return New; } @@ -2140,6 +2141,7 @@ R->setName(getName()); R->setNamesAsPredicateArg(getNamesAsPredicateArg()); R->setPredicateCalls(getPredicateCalls()); + R->setGISelFlagsRecord(getGISelFlagsRecord()); R->setTransformFn(getTransformFn()); for (unsigned i = 0, e = getNumTypes(); i != e; ++i) R->setType(i, getExtType(i)); @@ -2209,6 +2211,9 @@ for (unsigned i = 0, e = FragTree->getNumTypes(); i != e; ++i) FragTree->UpdateNodeType(i, getExtType(i), TP); + if (Op->isSubClassOf("GISelFlags")) + FragTree->setGISelFlagsRecord(Op); + // Transfer in the old predicates. for (const TreePredicateCall &Pred : getPredicateCalls()) FragTree->addPredicateCall(Pred); @@ -4542,6 +4547,7 @@ R->setName(Orig->getName()); R->setNamesAsPredicateArg(Orig->getNamesAsPredicateArg()); R->setPredicateCalls(Orig->getPredicateCalls()); + R->setGISelFlagsRecord(Orig->getGISelFlagsRecord()); R->setTransformFn(Orig->getTransformFn()); for (unsigned i = 0, e = Orig->getNumTypes(); i != e; ++i) R->setType(i, Orig->getExtType(i)); diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -38,6 +38,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/LowLevelTypeImpl.h" #include "llvm/Support/MachineValueType.h" +#include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" @@ -673,6 +674,12 @@ class MatchAction; class PredicateMatcher; +enum { + GISF_IgnoreCopies = 0x1, +}; + +using GISelFlags = std::uint16_t; + class Matcher { public: virtual ~Matcher() = default; @@ -864,6 +871,9 @@ /// ID for the next temporary register ID allocated with allocateTempRegID() unsigned NextTempRegID; + /// Current GISelFlags + GISelFlags Flags = 0; + std::vector RequiredFeatures; std::vector> EpilogueMatchers; @@ -884,6 +894,18 @@ uint64_t RuleID; static uint64_t NextRuleID; + GISelFlags updateGISelFlag(GISelFlags CurFlags, const Record *R, + StringRef FlagName, GISelFlags FlagBit) { + // If the value of a flag is unset, ignore it. + // If it's set, it always takes precedence over the existing value so + // clear/set the corresponding bit. + bool Unset = false; + bool Value = R->getValueAsBitOrUnset("GIIgnoreCopies", Unset); + if (!Unset) + return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit); + return CurFlags; + } + public: RuleMatcher(ArrayRef SrcLoc) : NextInsnVarID(0), NextOutputInsnID(0), NextTempRegID(0), SrcLoc(SrcLoc), @@ -901,6 +923,23 @@ template action_iterator insertAction(action_iterator InsertPt, Args &&... args); + // Update the active GISelFlags based on the GISelFlags Record R. + // A SaveAndRestore object is returned so the old GISelFlags are restored + // at the end of the scope. + SaveAndRestore setGISelFlags(const Record *R) { + if (!R || !R->isSubClassOf("GISelFlags")) + return {Flags, Flags}; + + assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) && + "GISelFlags is only expected on Pattern/PatFrags!"); + + GISelFlags NewFlags = + updateGISelFlag(Flags, R, "GIIgnoreCopies", GISF_IgnoreCopies); + return {Flags, NewFlags}; + } + + GISelFlags getGISelFlags() const { return Flags; } + /// Define an instruction without emitting any code to do so. unsigned implicitlyDefineInsnVar(InstructionMatcher &Matcher); @@ -1216,11 +1255,13 @@ std::string MatchingName; unsigned OrigOpIdx; + GISelFlags Flags; + public: SameOperandMatcher(unsigned InsnVarID, unsigned OpIdx, StringRef MatchingName, - unsigned OrigOpIdx) + unsigned OrigOpIdx, GISelFlags Flags) : OperandPredicateMatcher(OPM_SameOperand, InsnVarID, OpIdx), - MatchingName(MatchingName), OrigOpIdx(OrigOpIdx) {} + MatchingName(MatchingName), OrigOpIdx(OrigOpIdx), Flags(Flags) {} static bool classof(const PredicateMatcher *P) { return P->getKind() == OPM_SameOperand; @@ -2481,12 +2522,15 @@ protected: std::unique_ptr InsnMatcher; + GISelFlags Flags; + public: InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx, RuleMatcher &Rule, StringRef SymbolicName, bool NumOpsCheck = true) : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx), - InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)) {} + InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)), + Flags(Rule.getGISelFlags()) {} static bool classof(const PredicateMatcher *P) { return P->getKind() == OPM_Instruction; @@ -2496,7 +2540,9 @@ void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const { const unsigned NewInsnVarID = InsnMatcher->getInsnVarID(); - Table << MatchTable::Opcode("GIM_RecordInsn") + const bool IgnoreCopies = Flags & GISF_IgnoreCopies; + Table << MatchTable::Opcode(IgnoreCopies ? "GIM_RecordInsnIgnoreCopies" + : "GIM_RecordInsn") << MatchTable::Comment("DefineMI") << MatchTable::IntValue(NewInsnVarID) << MatchTable::Comment("MI") << MatchTable::IntValue(getInsnVarID()) @@ -3325,8 +3371,10 @@ // If the operand is already defined, then we must ensure both references in // the matcher have the exact same node. + RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher(); OM.addPredicate( - OM.getSymbolicName(), getOperandMatcher(OM.getSymbolicName()).getOpIdx()); + OM.getSymbolicName(), getOperandMatcher(OM.getSymbolicName()).getOpIdx(), + RM.getGISelFlags()); } void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) { @@ -3534,15 +3582,16 @@ const OperandMatcher &OtherOM = Rule.getOperandMatcher(MatchingName); unsigned OtherInsnVarID = Rule.getInsnVarID(OtherOM.getInstructionMatcher()); assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID()); - - Table << MatchTable::Opcode("GIM_CheckIsSameOperand") + const bool IgnoreCopies = Flags & GISF_IgnoreCopies; + Table << MatchTable::Opcode(IgnoreCopies + ? "GIM_CheckIsSameOperandIgnoreCopies" + : "GIM_CheckIsSameOperand") << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx) << MatchTable::Comment("OtherMI") << MatchTable::IntValue(OtherInsnVarID) << MatchTable::Comment("OtherOpIdx") - << MatchTable::IntValue(OtherOM.getOpIdx()) - << MatchTable::LineBreak; + << MatchTable::IntValue(OtherOM.getOpIdx()) << MatchTable::LineBreak; } //===- GlobalISelEmitter class --------------------------------------------===// @@ -3984,6 +4033,8 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( RuleMatcher &Rule, InstructionMatcher &InsnMatcher, const TreePatternNode *Src, unsigned &TempOpIdx) { + const auto SavedFlags = Rule.setGISelFlags(Src->getGISelFlagsRecord()); + Record *SrcGIEquivOrNull = nullptr; const CodeGenInstruction *SrcGIOrNull = nullptr; @@ -5208,6 +5259,9 @@ // before their first use.) InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName()); unsigned TempOpIdx = 0; + + const auto SavedFlags = M.setGISelFlags(P.getSrcRecord()); + auto InsnMatcherOrError = createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx); if (auto Error = InsnMatcherOrError.takeError())