diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst --- a/llvm/docs/GlobalISel/MIRPatterns.rst +++ b/llvm/docs/GlobalISel/MIRPatterns.rst @@ -101,6 +101,46 @@ // using $x again here copies operand 1 from G_AND into the new inst. (apply (COPY $root, $x)) +Builtin Operations +------------------ + +MIR Patterns also offer builtin operations, also called "builtin instructions". +They offer some powerful features that would otherwise require use of C++ code. + +GIReplaceReg +~~~~~~~~~~~~ + +.. code-block:: text + :caption: Usage + + (apply (GIReplaceReg $old, $new)) + +Operands: + +* ``$old`` (out) register defined by a matched instruction +* ``$new`` (in) register + +Semantics: + +* Can only appear in an 'apply' pattern. +* If both old/new are operands of matched instructions, + ``canReplaceReg`` is checked before applying the rule. + + +GIEraseRoot +~~~~~~~~~~~ + +.. code-block:: text + :caption: Usage + + (apply (GIEraseRoot)) + +Semantics: + +* Can only appear as the only pattern of an 'apply' pattern list. +* The root cannot have any output operands. +* The root must be a CodeGenInstruction + Limitations ----------- @@ -114,10 +154,6 @@ * Instructions with multiple defs cannot be the root of a ``GICombinePatFrag``. * Using ``GICombinePatFrag`` in the ``apply`` pattern of a ``GICombineRule`` is not supported. -* Deleting the matched pattern in a ``GICombineRule`` needs to be done using - ``G_IMPLICIT_DEF`` or C++. -* Replacing the root of a pattern with another instruction needs to be done - using COPY. * We cannot rewrite a matched instruction other than the root. * Matching/creating a (CImm) immediate >64 bits is not supported (see comment in ``GIM_CheckConstantInt``) @@ -179,33 +215,41 @@ Common Pattern #1: Replace a Register with Another ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The 'apply' pattern must always redefine its root. -It cannot just replace it with something else directly. -A simple workaround is to just use a COPY that'll be eliminated later. +The 'apply' pattern must always redefine all operands defined by the match root. +Sometimes, we do not need to create instructions, simply replace a def with +another matched register. The ``GIReplaceReg`` builtin can do just that. .. code-block:: text def Foo : GICombineRule< (defs root:$dst), (match (G_FNEG $tmp, $src), (G_FNEG $dst, $tmp)), - (apply (COPY $dst, $src))>; + (apply (GIReplaceReg $dst, $src))>; -Common Pattern #2: Erasing a Pattern -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This also works if the replacement register is a temporary register from the +``apply`` pattern. -As said before, we must always emit something in the 'apply' pattern. -If we wish to delete the matched instruction, we can simply replace its -definition with a ``G_IMPLICIT_DEF``. +.. code-block:: text + + def ReplaceTemp : GICombineRule< + (defs root:$a), + (match (G_BUILD_VECTOR $tmp, $x, $y), + (G_UNMERGE_VALUES $a, $b, $tmp)), + (apply (G_UNMERGE_VALUES $a, i32:$new, $y), + (GIReplaceReg $b, $new))> + +Common Pattern #2: Erasing a Def-less Root +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we simply want to erase a def-less match root, we can use the +``GIEraseRoot`` builtin. .. code-block:: text def Foo : GICombineRule< - (defs root:$dst), - (match (G_FOO $tmp, $src), (G_BAR $dst, $tmp)), - (apply (G_IMPLICIT_DEF $dst))>; - -If the instruction has no definition, like ``G_STORE``, we cannot use -an instruction pattern in 'apply' - C++ has to be used. + (defs root:$mi), + (match (G_STORE $a, $b):$mi), + (apply (GIEraseRoot))>; Common Pattern #3: Emitting a Constant Value ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h @@ -258,6 +258,13 @@ GIM_CheckIsSameOperand, GIM_CheckIsSameOperandIgnoreCopies, + /// Check we can replace all uses of a register with another. + /// - OldInsnID + /// - OldOpIdx + /// - NewInsnID + /// - NewOpIdx + GIM_CheckCanReplaceReg, + /// Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some /// named operands that will be recorded in RecordedOperands. Names of these /// operands are referenced in predicate argument list. Emitter determines @@ -431,6 +438,20 @@ /// - Expected type GIR_MakeTempReg, + /// Replaces all references to a register from an instruction + /// with another register from another instruction. + /// - OldInsnID + /// - OldOpIdx + /// - NewInsnID + /// - NewOpIdx + GIR_ReplaceReg, + + /// Replaces all references to a register with a temporary register. + /// - OldInsnID + /// - OldOpIdx + /// - TempRegIdx + GIR_ReplaceRegWithTempReg, + /// A successful emission GIR_Done, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h @@ -866,6 +866,25 @@ } break; } + case GIM_CheckCanReplaceReg: { + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OldOpIdx = MatchTable[CurrentIdx++]; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t NewOpIdx = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs[" + << OldInsnID << "][" << OldOpIdx << "] = MIs[" + << NewInsnID << "][" << NewOpIdx << "])\n"); + + Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); + Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); + if (!canReplaceReg(Old, New, MRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } case GIM_Reject: DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx << ": GIM_Reject\n"); @@ -1237,7 +1256,45 @@ << "] = GIR_MakeTempReg(" << TypeID << ")\n"); break; } + case GIR_ReplaceReg: { + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OldOpIdx = MatchTable[CurrentIdx++]; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t NewOpIdx = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs[" + << OldInsnID << "][" << OldOpIdx << "] = MIs[" + << NewInsnID << "][" << NewOpIdx << "])\n"); + Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); + Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); + if (Observer) + Observer->changingAllUsesOfReg(MRI, Old); + MRI.replaceRegWith(Old, New); + if (Observer) + Observer->finishedChangingAllUsesOfReg(); + break; + } + case GIR_ReplaceRegWithTempReg: { + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OldOpIdx = MatchTable[CurrentIdx++]; + int64_t TempRegID = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs[" + << OldInsnID << "][" << OldOpIdx << "] = TempRegs[" + << TempRegID << "])\n"); + + Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); + Register New = State.TempRegisters[TempRegID]; + if (Observer) + Observer->changingAllUsesOfReg(MRI, Old); + MRI.replaceRegWith(Old, New); + if (Observer) + Observer->finishedChangingAllUsesOfReg(); + break; + } case GIR_Coverage: { int64_t RuleID = MatchTable[CurrentIdx++]; assert(CoverageInfo); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -110,6 +110,42 @@ list Alternatives = alts; } +//===----------------------------------------------------------------------===// +// Pattern Builtins +//===----------------------------------------------------------------------===// + +// "Magic" Builtin instructions for MIR patterns. +// The definitions that implement +class GIBuiltinInst; + +// Replace all references to a register with another one. +// +// Usage: +// (apply (GIReplaceReg $old, $new)) +// +// Operands: +// - $old (out) register defined by a matched instruction +// - $new (in) register +// +// Semantics: +// - Can only appear in an 'apply' pattern. +// - If both old/new are operands of matched instructions, +// "canReplaceReg" is checked before applying the rule. +def GIReplaceReg : GIBuiltinInst; + +// Apply action that erases the match root. +// +// Usage: +// (apply (GIEraseRoot)) +// +// Semantics: +// - Can only appear as the only pattern of an 'apply' pattern list. +// - The root cannot have any output operands. +// - The root must be a CodeGenInstruction +// +// TODO: Allow using this directly, like (apply GIEraseRoot) +def GIEraseRoot : GIBuiltinInst; + //===----------------------------------------------------------------------===// def extending_load_matchdata : GIDefMatchData<"PreferredTuple">; @@ -141,7 +177,7 @@ def idempotent_prop : GICombineRule< (defs root:$dst), (match (idempotent_prop_frags $dst, $src)), - (apply (COPY $dst, $src))>; + (apply (GIReplaceReg $dst, $src))>; def extending_loads : GICombineRule< @@ -337,7 +373,7 @@ (defs root:$dst), (match (G_IMPLICIT_DEF $undef), (G_SELECT $dst, $undef, $x, $y)), - (apply (COPY $dst, $y)) + (apply (GIReplaceReg $dst, $y)) >; // Fold (true ? x : y) -> x @@ -384,14 +420,14 @@ def right_identity_zero: GICombineRule< (defs root:$dst), (match (right_identity_zero_frags $dst, $lhs)), - (apply (COPY $dst, $lhs)) + (apply (GIReplaceReg $dst, $lhs)) >; // Fold x op 1 -> x def right_identity_one: GICombineRule< (defs root:$dst), (match (G_MUL $dst, $x, 1)), - (apply (COPY $dst, $x)) + (apply (GIReplaceReg $dst, $x)) >; // Fold (x op x) - > x @@ -405,7 +441,7 @@ def binop_same_val: GICombineRule< (defs root:$dst), (match (binop_same_val_frags $dst, $src)), - (apply (COPY $dst, $src)) + (apply (GIReplaceReg $dst, $src)) >; // Fold (0 op x) - > 0 @@ -456,7 +492,7 @@ def binop_right_to_zero: GICombineRule< (defs root:$dst), (match (G_MUL $dst, $lhs, 0:$zero)), - (apply (COPY $dst, $zero)) + (apply (GIReplaceReg $dst, $zero)) >; // Erase stores of undef values. @@ -623,7 +659,7 @@ (defs root:$dst), (match (G_FNEG $t, $src), (G_FNEG $dst, $t)), - (apply (COPY $dst, $src)) + (apply (GIReplaceReg $dst, $src)) >; // Fold (unmerge(merge x, y, z)) -> z, y, z. @@ -1033,7 +1069,7 @@ def add_sub_reg: GICombineRule < (defs root:$dst), (match (add_sub_reg_frags $dst, $src)), - (apply (COPY $dst, $src))>; + (apply (GIReplaceReg $dst, $src))>; def buildvector_identity_fold : GICombineRule< (defs root:$build_vector, register_matchinfo:$matchinfo), diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir @@ -74,8 +74,9 @@ ; CHECK-LABEL: name: mul_0_cant_replace ; CHECK: liveins: $w0 ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s32) = COPY $w0 ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 0 - ; CHECK-NEXT: %op:gpr(s32) = COPY %cst(s32) + ; CHECK-NEXT: %op:gpr(s32) = G_MUL %x, %cst ; CHECK-NEXT: $w0 = COPY %op(s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 %x:_(s32) = COPY $w0 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td new file mode 100644 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td @@ -0,0 +1,94 @@ +// RUN: not llvm-tblgen -I %p/../../../../include -gen-global-isel-combiner \ +// RUN: -combiners=MyCombiner %s 2>&1| \ +// RUN: FileCheck %s -implicit-check-not=error: + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: expected operand 1 of 'GIReplaceReg' to be a name +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$dst, (i32 0))' +def builtinpat_immop : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIReplaceReg $dst, (i32 0)))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: expected operand 1 of 'GIReplaceReg' to be a name +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$dst, (i32 0):$k)' +def builtinpat_namedimmop : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIReplaceReg $dst, (i32 0):$k))>; + + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: 'GIEraseRoot' cannot be used in a 'match' pattern +def eraseroot_in_match : GICombineRule< + (defs root:$dst), + (match (GIEraseRoot):$mi), + (apply (COPY $dst, $src))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'GIEraseRoot' expected 0 operands, got 1 +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIEraseRoot ?:$dst)' +def eraseroot_ops : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIEraseRoot $dst), (COPY $dst, $src))>; + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot must be the only 'apply' pattern +def eraseroot_multiapply : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIEraseRoot), (COPY $dst, $src))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GIEraseRoot can only be used if on roots that do not have any output operand +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: 'COPY' has 1 output operands +def eraseroot_root_has_def: GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIEraseRoot))>; + +def TestPF: GICombinePatFrag< + (outs root:$def), + (ins), + [(pattern (COPY $def, $src))]>; +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction +def eraseroot_notinstmatch: GICombineRule< + (defs root:$mi), + (match (TestPF $dst):$mi), + (apply (GIEraseRoot))>; + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: 'GIReplaceReg' cannot be used in a 'match' pattern +def replacereg_in_match : GICombineRule< + (defs root:$dst), + (match (GIReplaceReg $dst, $src)), + (apply (COPY $dst, $src))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'GIReplaceReg' expected 2 operands, got 1 +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$dst)' +def replacereg_ops : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $src)), + (apply (GIReplaceReg $dst))>; + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIReplaceReg cannot replace 'tmp': this builtin can only replace a register defined by the match root +def replacereg_nonroot : GICombineRule< + (defs root:$dst), + (match (COPY $dst, $tmp), (COPY $tmp, $src)), + (apply (GIReplaceReg $dst, $src), (GIReplaceReg $tmp, $src))>; + +// CHECK: error: Failed to parse one or more rules + +def MyCombiner: GICombiner<"GenMyCombiner", [ + builtinpat_immop, + builtinpat_namedimmop, + eraseroot_in_match, + eraseroot_ops, + eraseroot_multiapply, + eraseroot_root_has_def, + eraseroot_notinstmatch, + replacereg_in_match, + replacereg_ops, + replacereg_nonroot +]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-parrsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-parrsing.td new file mode 100644 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-parrsing.td @@ -0,0 +1,55 @@ +// RUN: llvm-tblgen -I %p/../../../../include -gen-global-isel-combiner \ +// RUN: -gicombiner-stop-after-parse -combiners=MyCombiner %s | \ +// RUN: FileCheck %s + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +// CHECK: (CombineRule name:BuiltinTest0 id:0 root:a +// CHECK-NEXT: (MatchPats +// CHECK-NEXT: __BuiltinTest0_match_0:(CodeGenInstructionPattern G_TRUNC operands:[$a, $b]) +// CHECK-NEXT: ) +// CHECK-NEXT: (ApplyPats +// CHECK-NEXT: __BuiltinTest0_apply_0:(BuiltinPattern GIReplaceReg operands:[$a, $b]) +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable MatchPats +// CHECK-NEXT: a -> __BuiltinTest0_match_0 +// CHECK-NEXT: b -> +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable ApplyPats +// CHECK-NEXT: a -> __BuiltinTest0_apply_0 +// CHECK-NEXT: b -> +// CHECK-NEXT: ) +// CHECK-NEXT: ) +def BuiltinTest0 : GICombineRule< + (defs root:$a), + (match (G_TRUNC $a, $b)), + (apply (GIReplaceReg $a, $b)) +>; + +// CHECK: (CombineRule name:BuiltinTest1 id:1 root:mi +// CHECK-NEXT: (MatchPats +// CHECK-NEXT: mi:(CodeGenInstructionPattern G_STORE operands:[$a, $b]) +// CHECK-NEXT: ) +// CHECK-NEXT: (ApplyPats +// CHECK-NEXT: __BuiltinTest1_apply_0:(BuiltinPattern GIEraseRoot operands:[]) +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable MatchPats +// CHECK-NEXT: a -> +// CHECK-NEXT: b -> +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable ApplyPats ) +// CHECK-NEXT: ) +def BuiltinTest1 : GICombineRule< + (defs root:$mi), + (match (G_STORE $a, $b):$mi), + (apply (GIEraseRoot)) +>; + +def MyCombiner: GICombiner<"GenMyCombiner", [ + BuiltinTest0, + BuiltinTest1 +]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td new file mode 100644 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td @@ -0,0 +1,36 @@ +// RUN: llvm-tblgen -I %p/../../../../include -gen-global-isel-combiner \ +// RUN: -combiners=MyCombiner %s | \ +// RUN: FileCheck %s + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +def Test0 : GICombineRule< + (defs root:$mi), + (match (G_STORE $a, $b):$mi), + (apply (GIEraseRoot))>; + +def MyCombiner: GICombiner<"GenMyCombiner", [ + Test0, +]>; + +// CHECK: const int64_t *GenMyCombiner::getMatchTable() const { +// CHECK-NEXT: constexpr static int64_t MatchTable0[] = { +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 0*/ 10, // Rule ID 0 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE, +// CHECK-NEXT: // MIs[0] a +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] b +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // Combiner Rule #0: Test0 +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 0: @10 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: }; +// CHECK-NEXT: return MatchTable0; +// CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td new file mode 100644 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -0,0 +1,78 @@ +// RUN: llvm-tblgen -I %p/../../../../include -gen-global-isel-combiner \ +// RUN: -combiners=MyCombiner %s | \ +// RUN: FileCheck %s + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +def ReplaceMatched : GICombineRule< + (defs root:$dst), + (match (G_FNEG $tmp, $src), + (G_FNEG $dst, $tmp)), + (apply (GIReplaceReg $dst, $src))>; + +def ReplaceTemp : GICombineRule< + (defs root:$a), + (match (G_BUILD_VECTOR $tmp, $x, $y), + (G_UNMERGE_VALUES $a, $b, $tmp)), + (apply (G_UNMERGE_VALUES $a, i32:$new, $y), + (GIReplaceReg $b, $new))>; + +def MyCombiner: GICombiner<"GenMyCombiner", [ + ReplaceMatched, + ReplaceTemp +]>; + +// CHECK: const int64_t *GenMyCombiner::getMatchTable() const { +// CHECK-NEXT: constexpr static int64_t MatchTable0[] = { +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 0*/ 29, // Rule ID 0 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FNEG, +// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] tmp +// CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_FNEG, +// CHECK-NEXT: // MIs[1] src +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: GIM_CheckCanReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, +// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, +// CHECK-NEXT: // Combiner Rule #0: ReplaceMatched +// CHECK-NEXT: GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 0: @29 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 1*/ 76, // Rule ID 1 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_UNMERGE_VALUES, +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// CHECK-NEXT: // MIs[0] a +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] b +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] tmp +// CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1] +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR, +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, +// CHECK-NEXT: // MIs[1] x +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[1] y +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, +// CHECK-NEXT: // Combiner Rule #1: ReplaceTemp +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_UNMERGE_VALUES, +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // y +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 1: @76 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: }; +// CHECK-NEXT: return MatchTable0; +// CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td @@ -133,7 +133,7 @@ // CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: cannot use wip_match_opcode in combination with apply instruction patterns def instpat_with_wipmatch : GICombineRule< - (defs root:$a), + (defs root:$d), (match (wip_match_opcode COPY):$d), (apply (COPY $x, $b):$d)>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td @@ -297,6 +297,7 @@ (apply (COPY $a, (i32 0)), (COPY $b, (i32 0)))>; + def MyCombiner: GICombiner<"GenMyCombiner", [ WipOpcodeTest0, WipOpcodeTest1, diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -72,6 +72,7 @@ constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply"; constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_"; constexpr StringLiteral PatFragClassName = "GICombinePatFrag"; +constexpr StringLiteral BuiltinInstClassName = "GIBuiltinInst"; std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) { return "GICXXPred_Simple_IsRule" + to_string(CombinerRuleID) + "Enabled"; @@ -311,6 +312,7 @@ K_CodeGenInstruction, K_PatFrag, + K_Builtin, }; virtual ~Pattern() = default; @@ -348,6 +350,8 @@ return "CodeGenInstructionPattern"; case K_PatFrag: return "PatFragPattern"; + case K_Builtin: + return "BuiltinPattern"; } llvm_unreachable("unknown pattern kind!"); @@ -574,7 +578,8 @@ virtual ~InstructionPattern() = default; static bool classof(const Pattern *P) { - return P->getKind() == K_CodeGenInstruction || P->getKind() == K_PatFrag; + return P->getKind() == K_CodeGenInstruction || P->getKind() == K_PatFrag || + P->getKind() == K_Builtin; } template void addOperand(Ty &&...Init) { @@ -591,11 +596,13 @@ /// operands that must be redefined in the 'apply' pattern for the rule to be /// valid. /// - /// For CodeGenInstructionPatterns, this just returns the defs of the CGI. + /// For most patterns, this just returns the defs. /// For PatFrag this only returns the root of the PF. /// /// Returns an empty array on error. - virtual ArrayRef getApplyDefsNeeded() const = 0; + virtual ArrayRef getApplyDefsNeeded() const { + return {operands().begin(), getNumInstDefs()}; + } auto named_operands() { return make_filter_range(Operands, @@ -776,8 +783,6 @@ unsigned getNumInstDefs() const override; unsigned getNumInstOperands() const override; - ArrayRef getApplyDefsNeeded() const override; - const CodeGenInstruction &getInst() const { return I; } StringRef getInstName() const override { return I.TheDef->getName(); } @@ -816,11 +821,6 @@ : NumCGIOps; } -ArrayRef -CodeGenInstructionPattern::getApplyDefsNeeded() const { - return {operands().begin(), getNumInstDefs()}; -} - //===- OperandTypeChecker -------------------------------------------------===// /// This is a trivial type checker for all operands in a set of @@ -936,7 +936,9 @@ SmallVector, 4> Pats; }; - PatFrag(const Record &Def) : Def(Def) {} + explicit PatFrag(const Record &Def) : Def(Def) { + assert(Def.isSubClassOf(PatFragClassName)); + } static StringRef getParamKindStr(ParamKind OK); @@ -1051,6 +1053,10 @@ case Pattern::K_AnyOpcode: PrintError("wip_match_opcode cannot be used in " + PatFragClassName); return false; + case Pattern::K_Builtin: + PrintError("Builtin instructions cannot be used in " + + PatFragClassName); + return false; case Pattern::K_CXX: case Pattern::K_CodeGenInstruction: continue; @@ -1374,6 +1380,74 @@ return true; } +//===- BuiltinPattern -----------------------------------------------------===// + +enum BuiltinKind { + BI_ReplaceReg, + BI_EraseRoot, +}; + +class BuiltinPattern : public InstructionPattern { + struct BuiltinInfo { + StringLiteral DefName; + BuiltinKind Kind; + unsigned NumOps; + unsigned NumDefs; + }; + + static constexpr std::array KnownBuiltins = {{ + {"GIReplaceReg", BI_ReplaceReg, 2, 1}, + {"GIEraseRoot", BI_EraseRoot, 0, 0}, + }}; + +public: + BuiltinPattern(const Record &Def, StringRef Name) + : InstructionPattern(K_Builtin, Name), I(getBuiltinInfo(Def)) {} + + static bool classof(const Pattern *P) { return P->getKind() == K_Builtin; } + + unsigned getNumInstOperands() const override { return I.NumOps; } + unsigned getNumInstDefs() const override { return I.NumDefs; } + StringRef getInstName() const override { return I.DefName; } + BuiltinKind getBuiltinKind() const { return I.Kind; } + + bool checkSemantics(ArrayRef Loc) override; + +private: + static BuiltinInfo getBuiltinInfo(const Record &Def); + + BuiltinInfo I; +}; + +BuiltinPattern::BuiltinInfo BuiltinPattern::getBuiltinInfo(const Record &Def) { + assert(Def.isSubClassOf(BuiltinInstClassName)); + + StringRef Name = Def.getName(); + for (const auto &KBI : KnownBuiltins) { + if (KBI.DefName == Name) + return KBI; + } + + PrintFatalError(Def.getLoc(), "Unimplemented " + BuiltinInstClassName + + " def '" + Name + "'"); +} + +bool BuiltinPattern::checkSemantics(ArrayRef Loc) { + if (!InstructionPattern::checkSemantics(Loc)) + return false; + + // For now all builtins just take names, no immediates. + for (const auto &[Idx, Op] : enumerate(operands())) { + if (!Op.isNamedOperand() || Op.isNamedImmediate()) { + PrintError(Loc, "expected operand " + to_string(Idx) + " of '" + + getInstName() + "' to be a name"); + return false; + } + } + + return true; +} + //===- PrettyStackTrace Helpers ------------------------------------------===// class PrettyStackTraceParse : public PrettyStackTraceEntry { @@ -1485,11 +1559,7 @@ const CXXPattern &P); bool hasOnlyCXXApplyPatterns() const; - bool hasAnyOpcodeMatchPattern() const { - return any_of(MatchPats, [&](const auto &E) { - return isa(E.second.get()); - }); - } + bool hasEraseRoot() const; // Infer machine operand types and check their consistency. bool typecheckPatterns(); @@ -1503,6 +1573,9 @@ /// PatFrags from generating enormous amounts of rules. bool buildPermutationsToEmit(); + /// Checks additional semantics of the Patterns. + bool checkSemantics(); + /// Creates a new RuleMatcher with some boilerplate /// settings/actions/predicates, and and adds it to \p OutRMs. /// \see addFeaturePredicates too. @@ -1550,19 +1623,22 @@ bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M); - // Recursively visits CodeGenInstructionPattern from P to build up the + // Recursively visits InstructionPatterns from P to build up the // RuleMatcher actions. - bool - emitCodeGenInstructionApplyPattern(CodeExpansions &CE, RuleMatcher &M, - const CodeGenInstructionPattern &P, - DenseSet &SeenPats, - StringMap &OperandToTempRegID); + bool emitInstructionApplyPattern(CodeExpansions &CE, RuleMatcher &M, + const InstructionPattern &P, + DenseSet &SeenPats, + StringMap &OperandToTempRegID); bool emitCodeGenInstructionApplyImmOperand(RuleMatcher &M, BuildMIAction &DstMI, const CodeGenInstructionPattern &P, const InstructionOperand &O); + bool emitBuiltinApplyPattern(CodeExpansions &CE, RuleMatcher &M, + const BuiltinPattern &P, + StringMap &OperandToTempRegID); + // Recursively visits CodeGenInstructionPattern from P to build up the // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as // needed. @@ -1595,7 +1671,7 @@ /// Operand tables to tie match/apply patterns together. OperandTable<> MatchOpTable; - OperandTable ApplyOpTable; + OperandTable<> ApplyOpTable; /// Set by findRoots. Pattern *MatchRoot = nullptr; @@ -1627,7 +1703,7 @@ return false; if (!buildRuleOperandsTable() || !typecheckPatterns() || !findRoots() || - !buildPermutationsToEmit()) + !checkSemantics() || !buildPermutationsToEmit()) return false; LLVM_DEBUG(verify()); return true; @@ -1649,6 +1725,7 @@ break; } case Pattern::K_PatFrag: + case Pattern::K_Builtin: case Pattern::K_CodeGenInstruction: if (!emitMatchPattern(CE, Alts, *cast(MatchRoot))) return false; @@ -1799,18 +1876,10 @@ return false; } - if (isa(Pat.get())) { - if (hasAnyOpcodeMatchPattern()) { - PrintError("cannot use wip_match_opcode in combination with apply " - "instruction patterns!"); - return false; - } - - if (isa(Pat.get())) { - PrintError("'" + Name + "': using " + PatFragClassName + - " is not supported in apply patterns"); - return false; - } + if (isa(Pat.get())) { + PrintError("'" + Name + "': using " + PatFragClassName + + " is not supported in apply patterns"); + return false; } if (auto *CXXPat = dyn_cast(Pat.get())) @@ -1827,18 +1896,11 @@ return false; } - // TODO: Could move this to some "checkSemantics" method? - if (isa(Pat.get())) { - if (hasAnyOpcodeMatchPattern()) { - PrintError("wip_opcode_match can only be present once"); - return false; - } - } - - if (const auto *CXXPat = dyn_cast(Pat.get())) { - if (!CXXPat->getRawCode().contains("return ")) { - PrintWarning("'match' C++ code does not seem to return!"); - } + // For now, none of the builtins can appear in 'match'. + if (const auto *BP = dyn_cast(Pat.get())) { + PrintError("'" + BP->getInstName() + + "' cannot be used in a 'match' pattern"); + return false; } MatchPats[Name] = std::move(Pat); @@ -1881,6 +1943,14 @@ }); } +bool CombineRuleBuilder::hasEraseRoot() const { + return any_of(ApplyPats, [&](auto &Entry) { + if (const auto *BP = dyn_cast(Entry.second.get())) + return BP->getBuiltinKind() == BI_EraseRoot; + return false; + }); +} + bool CombineRuleBuilder::typecheckPatterns() { OperandTypeChecker OTC(RuleDef.getLoc()); @@ -1955,6 +2025,97 @@ return true; } +bool CombineRuleBuilder::checkSemantics() { + assert(MatchRoot && "Cannot call this before findRoots()"); + + bool UsesWipMatchOpcode = false; + for (const auto &Match : MatchPats) { + const auto *Pat = Match.second.get(); + + if (const auto *CXXPat = dyn_cast(Pat)) { + if (!CXXPat->getRawCode().contains("return ")) + PrintWarning("'match' C++ code does not seem to return!"); + continue; + } + + const auto *AOP = dyn_cast(Pat); + if (!AOP) + continue; + + if (UsesWipMatchOpcode) { + PrintError("wip_opcode_match can only be present once"); + return false; + } + + UsesWipMatchOpcode = true; + } + + for (const auto &Apply : ApplyPats) { + assert(Apply.second.get()); + const auto *IP = dyn_cast(Apply.second.get()); + if (!IP) + continue; + + if (UsesWipMatchOpcode) { + PrintError("cannot use wip_match_opcode in combination with apply " + "instruction patterns!"); + return false; + } + + const auto *BIP = dyn_cast(IP); + if (!BIP) + continue; + StringRef Name = BIP->getInstName(); + + // (GIEraseInst) has to be the only apply pattern, or it can not be used at + // all. The root cannot have any defs either. + switch (BIP->getBuiltinKind()) { + case BI_EraseRoot: { + if (ApplyPats.size() > 1) { + PrintError(Name + " must be the only 'apply' pattern"); + return false; + } + + const auto *IRoot = dyn_cast(MatchRoot); + if (!IRoot) { + PrintError(Name + + " can only be used if the root is a CodeGenInstruction"); + return false; + } + + if (IRoot->getNumInstDefs() != 0) { + PrintError(Name + " can only be used if on roots that do " + "not have any output operand"); + PrintNote("'" + IRoot->getInstName() + "' has " + + Twine(IRoot->getNumInstDefs()) + " output operands"); + return false; + } + break; + } + case BI_ReplaceReg: { + // (GIReplaceReg can only be used on the root instruction) + // TODO: When we allow rewriting non-root instructions, also allow this. + StringRef OldRegName = BIP->getOperand(0).getOperandName(); + auto *Def = MatchOpTable.getDef(OldRegName); + if (!Def) { + PrintError(Name + " cannot find a matched pattern that defines '" + + OldRegName + "'"); + return false; + } + if (MatchOpTable.getDef(OldRegName) != MatchRoot) { + PrintError(Name + " cannot replace '" + OldRegName + + "': this builtin can only replace a register defined by the " + "match root"); + return false; + } + break; + } + } + } + + return true; +} + RuleMatcher &CombineRuleBuilder::addRuleMatcher(const PatternAlternatives &Alts, Twine AdditionalComment) { auto &RM = OutRMs.emplace_back(RuleDef.getLoc()); @@ -2009,7 +2170,7 @@ const auto Finish = [&]() { assert(MatchRoot); - if (hasOnlyCXXApplyPatterns()) + if (hasOnlyCXXApplyPatterns() || hasEraseRoot()) return true; auto *IPRoot = dyn_cast(MatchRoot); @@ -2107,7 +2268,7 @@ } for (auto &Pat : values(ApplyPats)) { - auto *IP = dyn_cast(Pat.get()); + auto *IP = dyn_cast(Pat.get()); if (IP && !ApplyOpTable.addPattern(IP, DiagnoseRedefApply)) return false; } @@ -2230,6 +2391,10 @@ if (!PF) return nullptr; // Already diagnosed by parsePatFrag Pat = std::make_unique(*PF, Name); + } else if (const DagInit *BP = + getDagWithOperatorOfSubClass(Arg, BuiltinInstClassName)) { + Pat = std::make_unique( + *BP->getOperatorAsDef(RuleDef.getLoc()), Name); } else { return nullptr; } @@ -2496,6 +2661,8 @@ if (!emitPatFragMatchPattern(CE, Alts, M, &IM, *PFP, SeenPats)) return false; + } else if (isa(&IP)) { + llvm_unreachable("No match builtins known!"); } else llvm_unreachable("Unknown kind of InstructionPattern!"); @@ -2514,6 +2681,9 @@ return false; continue; } + case Pattern::K_Builtin: + PrintError("No known match builtins"); + return false; case Pattern::K_CodeGenInstruction: cast(Pat.get())->reportUnreachable(RuleDef.getLoc()); return false; @@ -2563,6 +2733,9 @@ return false; continue; } + case Pattern::K_Builtin: + PrintError("No known match builtins"); + return false; case Pattern::K_CodeGenInstruction: cast(Pat.get())->reportUnreachable( RuleDef.getLoc()); @@ -2728,11 +2901,11 @@ StringMap OperandToTempRegID; for (auto *ApplyRoot : ApplyRoots) { - assert(isa(ApplyRoot) && - "PatFragPatterns are not supported in apply patterns yet!"); - if (!emitCodeGenInstructionApplyPattern( - CE, M, *cast(ApplyRoot), SeenPats, - OperandToTempRegID)) + assert(isa(ApplyRoot) && + "Root can only be a InstructionPattern!"); + if (!emitInstructionApplyPattern(CE, M, + cast(*ApplyRoot), + SeenPats, OperandToTempRegID)) return false; } @@ -2746,9 +2919,13 @@ case Pattern::K_PatFrag: // TODO: We could support pure C++ PatFrags as a temporary thing. llvm_unreachable("Unexpected pattern in apply!"); + case Pattern::K_Builtin: + if (!emitInstructionApplyPattern(CE, M, cast(*Pat), + SeenPats, OperandToTempRegID)) + return false; + break; case Pattern::K_CodeGenInstruction: - cast(Pat.get())->reportUnreachable( - RuleDef.getLoc()); + cast(*Pat).reportUnreachable(RuleDef.getLoc()); return false; case Pattern::K_CXX: { addCXXAction(M, CE, *cast(Pat.get())); @@ -2762,8 +2939,8 @@ return true; } -bool CombineRuleBuilder::emitCodeGenInstructionApplyPattern( - CodeExpansions &CE, RuleMatcher &M, const CodeGenInstructionPattern &P, +bool CombineRuleBuilder::emitInstructionApplyPattern( + CodeExpansions &CE, RuleMatcher &M, const InstructionPattern &P, DenseSet &SeenPats, StringMap &OperandToTempRegID) { auto StackTrace = PrettyStackTraceEmit(RuleDef, &P); @@ -2780,8 +2957,8 @@ StringRef OpName = Op.getOperandName(); if (const auto *DefPat = ApplyOpTable.getDef(OpName)) { - if (!emitCodeGenInstructionApplyPattern(CE, M, *DefPat, SeenPats, - OperandToTempRegID)) + if (!emitInstructionApplyPattern(CE, M, *DefPat, SeenPats, + OperandToTempRegID)) return false; } else { // If we have no def, check this exists in the MatchRoot. @@ -2794,9 +2971,17 @@ } } + if (const auto *BP = dyn_cast(&P)) + return emitBuiltinApplyPattern(CE, M, *BP, OperandToTempRegID); + + if (isa(&P)) + llvm_unreachable("PatFragPatterns is not supported in 'apply'!"); + + auto &CGIP = cast(P); + // Now render this inst. auto &DstMI = - M.addAction(M.allocateOutputInsnID(), &P.getInst()); + M.addAction(M.allocateOutputInsnID(), &CGIP.getInst()); for (auto &Op : P.operands()) { if (Op.isNamedImmediate()) { @@ -2808,7 +2993,7 @@ } if (Op.hasImmValue()) { - if (!emitCodeGenInstructionApplyImmOperand(M, DstMI, P, Op)) + if (!emitCodeGenInstructionApplyImmOperand(M, DstMI, CGIP, Op)) return false; continue; } @@ -2934,6 +3119,55 @@ return true; } +bool CombineRuleBuilder::emitBuiltinApplyPattern( + CodeExpansions &CE, RuleMatcher &M, const BuiltinPattern &P, + StringMap &OperandToTempRegID) { + const auto Error = [&](Twine Reason) { + PrintError("cannot emit '" + P.getInstName() + "' builtin: " + Reason); + return false; + }; + + switch (P.getBuiltinKind()) { + case BI_EraseRoot: { + // Root is always inst 0. + M.addAction(/*InsnID*/ 0); + return true; + } + case BI_ReplaceReg: { + StringRef Old = P.getOperand(0).getOperandName(); + StringRef New = P.getOperand(1).getOperandName(); + + if (!ApplyOpTable.lookup(New).Found && !MatchOpTable.lookup(New).Found) + return Error("unknown operand '" + Old + "'"); + + auto &OldOM = M.getOperandMatcher(Old); + if (auto It = OperandToTempRegID.find(New); + It != OperandToTempRegID.end()) { + // Replace with temp reg. + M.addAction(OldOM.getInsnVarID(), OldOM.getOpIdx(), + It->second); + } else { + // Replace with matched reg. + auto &NewOM = M.getOperandMatcher(New); + M.addAction(OldOM.getInsnVarID(), OldOM.getOpIdx(), + NewOM.getInsnVarID(), NewOM.getOpIdx()); + } + // checkSemantics should have ensured that we can only rewrite the root. + // Ensure we're deleting it. + assert(MatchOpTable.getDef(Old) == MatchRoot); + // TODO: We could avoid adding the action again if it's already in. The + // MatchTable is smart enough to only emit one opcode even if + // EraseInstAction is present multiple times. I think searching for a copy + // is more expensive than just blindly adding it though. + M.addAction(/*InsnID*/ 0); + + return true; + } + } + + llvm_unreachable("Unknown BuiltinKind!"); +} + bool isLiteralImm(const InstructionPattern &P, unsigned OpIdx) { if (const auto *CGP = dyn_cast(&P)) { StringRef InstName = CGP->getInst().TheDef->getName(); diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/GlobalISelMatchTable.h @@ -469,6 +469,8 @@ std::vector RequiredFeatures; std::vector> EpilogueMatchers; + DenseSet ErasedInsnIDs; + ArrayRef SrcLoc; typedef std::tuple @@ -508,6 +510,11 @@ void addRequiredSimplePredicate(StringRef PredName); const std::vector &getRequiredSimplePredicates(); + /// Attempts to mark \p ID as erased (GIR_EraseFromParent called on it). + /// If \p ID has already been erased, returns false and GIR_EraseFromParent + /// should NOT be emitted. + bool tryEraseInsnID(unsigned ID) { return ErasedInsnIDs.insert(ID).second; } + // Emplaces an action of the specified Kind at the end of the action list. // // Returns a reference to the newly created action. @@ -2086,6 +2093,8 @@ AK_DebugComment, AK_CustomCXX, AK_BuildMI, + AK_EraseInst, + AK_ReplaceReg, AK_ConstraintOpsToDef, AK_ConstraintOpsToRC, AK_MakeTempReg, @@ -2097,6 +2106,10 @@ virtual ~MatchAction() {} + // Some actions may need to add extra predicates to ensure they can run. + virtual void emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const {} + /// Emit the MatchTable opcodes to implement the action. virtual void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const = 0; @@ -2174,6 +2187,46 @@ void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; }; +class EraseInstAction : public MatchAction { + unsigned InsnID; + +public: + EraseInstAction(unsigned InsnID) + : MatchAction(AK_EraseInst), InsnID(InsnID) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_EraseInst; + } + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; + static void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnID); +}; + +class ReplaceRegAction : public MatchAction { + unsigned OldInsnID, OldOpIdx; + unsigned NewInsnId = -1, NewOpIdx; + unsigned TempRegID = -1; + +public: + ReplaceRegAction(unsigned OldInsnID, unsigned OldOpIdx, unsigned NewInsnId, + unsigned NewOpIdx) + : MatchAction(AK_EraseInst), OldInsnID(OldInsnID), OldOpIdx(OldOpIdx), + NewInsnId(NewInsnId), NewOpIdx(NewOpIdx) {} + + ReplaceRegAction(unsigned OldInsnID, unsigned OldOpIdx, unsigned TempRegID) + : MatchAction(AK_EraseInst), OldInsnID(OldInsnID), OldOpIdx(OldOpIdx), + TempRegID(TempRegID) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_ReplaceReg; + } + + void emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const override; + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; +}; + /// Generates code to constrain the operands of an output instruction to the /// register classes specified by the definition of that instruction. class ConstrainOperandsToDefinitionAction : public MatchAction { diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -869,6 +869,10 @@ Matchers.front()->emitPredicateOpcodes(Table, *this); + // Check if it's safe to replace registers. + for (const auto &MA : Actions) + MA->emitAdditionalPredicates(Table, *this); + // We must also check if it's safe to fold the matched instructions. if (InsnVariableIDs.size() >= 2) { // Invert the map to create stable ordering (by var names) @@ -2007,9 +2011,58 @@ // better for combines. Particularly when there are multiple match // roots. if (InsnID == 0) - Table << MatchTable::Opcode("GIR_EraseFromParent") - << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + EraseInstAction::emitActionOpcodes(Table, Rule, /*InsnID*/ 0); +} + +//===- EraseInstAction ----------------------------------------------------===// + +void EraseInstAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnID) { + // Avoid erasing the same inst twice. + if (!Rule.tryEraseInsnID(InsnID)) + return; + + Table << MatchTable::Opcode("GIR_EraseFromParent") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::LineBreak; +} + +void EraseInstAction::emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + emitActionOpcodes(Table, Rule, InsnID); +} + +//===- ReplaceRegAction ---------------------------------------------------===// + +void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const { + if (TempRegID != (unsigned)-1) + return; + + Table << MatchTable::Opcode("GIM_CheckCanReplaceReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId) + << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx) + << MatchTable::LineBreak; +} + +void ReplaceRegAction::emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + if (TempRegID != (unsigned)-1) { + Table << MatchTable::Opcode("GIR_ReplaceRegWithTempReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) << MatchTable::LineBreak; + } else { + Table << MatchTable::Opcode("GIR_ReplaceReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId) + << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx) + << MatchTable::LineBreak; + } } //===- ConstrainOperandToRegClassAction -----------------------------------===//