diff --git a/llvm/test/TableGen/GICombinerEmitter/match-invalid.td b/llvm/test/TableGen/GICombinerEmitter/match-invalid.td --- a/llvm/test/TableGen/GICombinerEmitter/match-invalid.td +++ b/llvm/test/TableGen/GICombinerEmitter/match-invalid.td @@ -10,6 +10,8 @@ def dummy; +def int_dummy : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty]>; + def R0 : Register<"r0"> { let Namespace = "MyTarget"; } def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>; class I Pat> @@ -69,13 +71,49 @@ (dummy)>; // CHECK-NOT: :[[@LINE-5]]:{{[0-9]+}}: error: +def not_enough_instr_operands : GICombineRule< + (defs root:$a), + (match (MOV $a)), + (dummy)>; +// CHECK: :[[@LINE-4]]:{{[0-9]+}}: error: MOV expected 2 args, got 1 +// CHECK: :[[@LINE-5]]:{{[0-9]+}}: error: Expected a subclass of GIMatchKind or a sub-dag whose operator is either of a GIMatchKindWithArgs or Instruction +// CHECK: :[[@LINE-6]]:{{[0-9]+}}: error: Failed to parse rule + +def too_many_instr_operands : GICombineRule< + (defs root:$a), + (match (MOV $a, $b, $c)), + (dummy)>; +// CHECK: :[[@LINE-4]]:{{[0-9]+}}: error: MOV expected 2 args, got 3 +// CHECK: :[[@LINE-5]]:{{[0-9]+}}: error: Expected a subclass of GIMatchKind or a sub-dag whose operator is either of a GIMatchKindWithArgs or Instruction +// CHECK: :[[@LINE-6]]:{{[0-9]+}}: error: Failed to parse rule + +def not_enough_intrin_operands : GICombineRule< + (defs root:$a), + (match (int_dummy $a)), + (dummy)>; +// CHECK: :[[@LINE-4]]:{{[0-9]+}}: error: intrinsic 'llvm.dummy' expected 2 args, got 1 +// CHECK: :[[@LINE-5]]:{{[0-9]+}}: error: Expected a subclass of GIMatchKind or a sub-dag whose operator is either of a GIMatchKindWithArgs or Instruction +// CHECK: :[[@LINE-6]]:{{[0-9]+}}: error: Failed to parse rule + +def too_many_intrin_operands : GICombineRule< + (defs root:$a), + (match (int_dummy $a, $b, $c)), + (dummy)>; +// CHECK: :[[@LINE-4]]:{{[0-9]+}}: error: intrinsic 'llvm.dummy' expected 2 args, got 3 +// CHECK: :[[@LINE-5]]:{{[0-9]+}}: error: Expected a subclass of GIMatchKind or a sub-dag whose operator is either of a GIMatchKindWithArgs or Instruction +// CHECK: :[[@LINE-6]]:{{[0-9]+}}: error: Failed to parse rule + def MyCombiner: GICombinerHelper<"GenMyCombiner", [ // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: Failed to parse one or more rules missing_match_node, null_matcher, unknown_kind1, unknown_kind2, - multidef + multidef, + not_enough_instr_operands, + too_many_instr_operands, + not_enough_intrin_operands, + too_many_intrin_operands // Rules omitted from a matcher can be as broken as you like. They will not be read. // multidef_but_not_an_error ]>; diff --git a/llvm/test/TableGen/GICombinerEmitter/match-tree.td b/llvm/test/TableGen/GICombinerEmitter/match-tree.td --- a/llvm/test/TableGen/GICombinerEmitter/match-tree.td +++ b/llvm/test/TableGen/GICombinerEmitter/match-tree.td @@ -32,6 +32,11 @@ def ZEXT : I<(outs GPR32:$dst), (ins GPR32:$src1), []>; def ICMP : I<(outs GPR32:$dst), (ins GPR32:$tst, GPR32:$src1, GPR32:$src2), []>; +def int_dummy : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty]>; +def int_dummy_multi_ret : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_ptr_ty], [llvm_ptr_ty]>; +let TargetPrefix = "foo" in +def int_foo_dummy_target : DefaultAttrsIntrinsic<[llvm_ptr_ty], []>; + def HasFoo : Predicate<"Subtarget->hasFoo()">; def HasAnswerToEverything : Predicate<"Subtarget->getAnswerToUniverse() == 42 && Subtarget->getAnswerToLife() == 42">; @@ -102,6 +107,21 @@ (match (MUL $d, $t, $z)), (apply [{ APPLY }])>; +def Rule10 : GICombineRule< + (defs root:$d), + (match (int_dummy $d, $s)), + (apply [{ APPLY (${d}), (${s}) }])>; + +def Rule11 : GICombineRule< + (defs root:$d), + (match (int_foo_dummy_target $d)), + (apply [{ APPLY (${d}) }])>; + +def Rule12 : GICombineRule< + (defs root:$d1), + (match (int_dummy_multi_ret $d1, $d2, $s1)), + (apply [{ APPLY (${d1}) (${d2}) (${s1}) }])>; + def MyCombinerHelper: GICombinerHelper<"GenMyCombinerHelper", [ Rule0, Rule1, @@ -112,11 +132,14 @@ Rule6, Rule7, Rule8, - Rule9 + Rule9, + Rule10, + Rule11, + Rule12 ]>; // CHECK-LABEL: digraph "matchtree" { -// CHECK-DAG: Node[[N0:(0x)?[0-9a-fA-F]+]] [shape=record,label="{MI[0].getOpcode()|5 partitions|Rule0,Rule1,Rule2,Rule3,Rule4,Rule5,Rule6,Rule7,Rule8,Rule9}"] +// CHECK-DAG: Node[[N0:(0x)?[0-9a-fA-F]+]] [shape=record,label="{MI[0].getOpcode()|7 partitions|Rule0,Rule1,Rule2,Rule3,Rule4,Rule5,Rule6,Rule7,Rule8,Rule9,Rule10,Rule11,Rule12}"] // CHECK-DAG: Node[[N1:(0x)?[0-9a-fA-F]+]] [shape=record,label="{MI[1] = getVRegDef(MI[0].getOperand(1))|2 partitions|Rule0,Rule5}"] // CHECK-DAG: Node[[N2:(0x)?[0-9a-fA-F]+]] [shape=record,label="{MI[1].getOpcode()|2 partitions|Rule0,Rule5}"] // CHECK-DAG: Node[[N3:(0x)?[0-9a-fA-F]+]] [shape=record,label="{No partitioner|Rule0}"] @@ -208,6 +231,8 @@ // CODE-NEXT: case MyTarget::ADD: Partition = 2; break; // CODE-NEXT: case MyTarget::TRUNC: Partition = 3; break; // CODE-NEXT: case MyTarget::MUL: Partition = 4; break; +// CODE-NEXT: case TargetOpcode::G_INTRINSIC: Partition = 5; break; +// CODE-NEXT: case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: Partition = 6; break; // CODE-NEXT: } // Check that the correct partition is choosen if operand 1 is a register. @@ -313,3 +338,93 @@ // CODE-NEXT: return true; // CODE-NEXT: } // CODE-NEXT: } + +// Check intrinsic matching. +// CODE: if (Partition == 5 /* TargetOpcode::G_INTRINSIC */) { +// CODE-NEXT: // Leaf name: Rule10 +// CODE-NEXT: // Rule: Rule10 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(10)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(1).getIntrinsicID() == Intrinsic::dummy +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule10'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)), (MIs[0]->getOperand(2)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: // Leaf name: Rule11 +// CODE-NEXT: // Rule: Rule11 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(11)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(1).getIntrinsicID() == Intrinsic::foo_dummy_target +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule11'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: // Leaf name: Rule12 +// CODE-NEXT: // Rule: Rule12 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(12)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(2).getIntrinsicID() == Intrinsic::dummy_multi_ret +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule12'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)) (MIs[0]->getOperand(1)) (MIs[0]->getOperand(3)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: return false; +// CODE-NEXT:} + +// TODO: This should be merged with the code above. The tree currently emits the same +// code once for each G_INTRINSIC opcode variant which is useless bloat really. + +// CODE: if (Partition == 6 /* TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS */) { +// CODE-NEXT: // Leaf name: Rule10 +// CODE-NEXT: // Rule: Rule10 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(10)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(1).getIntrinsicID() == Intrinsic::dummy +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule10'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)), (MIs[0]->getOperand(2)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: // Leaf name: Rule11 +// CODE-NEXT: // Rule: Rule11 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(11)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(1).getIntrinsicID() == Intrinsic::foo_dummy_target +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule11'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: // Leaf name: Rule12 +// CODE-NEXT: // Rule: Rule12 +// CODE-NEXT: if (!RuleConfig->isRuleDisabled(12)) { +// CODE-NEXT: if (1 +// CODE-NEXT: && ( +// CODE-NEXT: MIs[0]->getOperand(2).getIntrinsicID() == Intrinsic::dummy_multi_ret +// CODE-NEXT: ) +// CODE-NEXT: ) { +// CODE-NEXT: LLVM_DEBUG(dbgs() << "Applying rule 'Rule12'\n"); +// CODE-NEXT: APPLY (MIs[0]->getOperand(0)) (MIs[0]->getOperand(1)) (MIs[0]->getOperand(3)) +// CODE-NEXT: return true; +// CODE-NEXT: } +// CODE-NEXT: } +// CODE-NEXT: return false; +// CODE-NEXT:} diff --git a/llvm/test/TableGen/GICombinerEmitter/parse-match-pattern.td b/llvm/test/TableGen/GICombinerEmitter/parse-match-pattern.td --- a/llvm/test/TableGen/GICombinerEmitter/parse-match-pattern.td +++ b/llvm/test/TableGen/GICombinerEmitter/parse-match-pattern.td @@ -23,6 +23,8 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), []>; def MOV2 : I<(outs GPR32:$dst), (ins GPR32:$src1), []>; +def int_dummy : DefaultAttrsIntrinsic<[llvm_ptr_ty]>; + def trivial : GICombineRule< (defs root:$d), (match (MOV $d, $s)), @@ -197,12 +199,33 @@ // CHECK-NEXT: Node[[N2]]:s1:n -> Pred[[P3]]:d2:s [style=dotted] // CHECK-NEXT: {{^}$}} +def intrinsic_trivial : GICombineRule< + (defs root:$d), + (match (int_dummy $d)), + (apply [{ APPLY }])>; + +// CHECK-LABEL: Parsed rule defs/match for 'intrinsic_trivial' + +// CHECK-NEXT: matchdag { +// CHECK-NEXT: (G_INTRINSIC 0:op0, 1:$intrin_id, 2:op2):$__anon5_0 // $d=getOperand(0), $$intrin_id=getOperand(1) +// CHECK-NEXT: <<$mi.getOpcode() == oneof(G_INTRINSIC,G_INTRINSIC_W_SIDE_EFFECTS) && $intrin_id.getIntrinsicID() == Intrinsic::dummy>>:$__anonpred5_1 +// CHECK-NEXT: __anon5_0 ==> __anonpred5_1[mi] +// CHECK-NEXT: } + +// CHECK-NEXT: digraph "intrinsic_trivial" { +// CHECK-NEXT: rankdir="BT" +// CHECK-NEXT: Node[[N1:(0x)?[0-9a-fA-F]+]] [shape=record,label="{{[{]{}}#0 $op0|#1 $$intrin_id|#2 $op2}|__anon5_0|G_INTRINSIC|Match starts here|$d=getOperand(0), $$intrin_id=getOperand(1)|{{(0x)?[0-9a-fA-F]+}}|{#0 $op0|#1 $$intrin_id|#2 $op2}}",color=red] +// CHECK-NEXT: Pred[[P1:(0x)?[0-9a-fA-F]+]] [shape=record,label="{{[{]{}}#0 $$|#1 $mi}|__anonpred5_1|$mi.getOpcode() == oneof(G_INTRINSIC,G_INTRINSIC_W_SIDE_EFFECTS) && $intrin_id.getIntrinsicID() == Intrinsic::dummy|{{(0x)?[0-9a-fA-F]+}}|{#0 $$|#1 $mi}}",style=dotted] +// CHECK-NEXT: Node[[N1]]:e -> Pred[[P1]]:d1:s [style=dotted] +// CHECK-NEXT: } + def MyCombiner: GICombinerHelper<"GenMyCombiner", [ trivial, simple, multiroot, nonstandardroot, - multiref_use + multiref_use, + intrinsic_trivial ]>; // Verify we're sharing operand lists correctly @@ -211,5 +234,6 @@ // CHECK-NEXT: 0:dst, 1:src1{{$}} // CHECK-NEXT: 0:$, 1:mi{{$}} // CHECK-NEXT: 0:$, 1:mi0, 2:mi1{{$}} +// CHECK-NEXT: 0:op0, 1:$intrin_id, 2:op2 // CHECK-NEXT: } // CHECK-NEXT: } diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/CodeGenIntrinsics.h --- a/llvm/utils/TableGen/CodeGenIntrinsics.h +++ b/llvm/utils/TableGen/CodeGenIntrinsics.h @@ -152,6 +152,9 @@ bool isParamImmArg(unsigned ParamIdx) const; + unsigned getNumDefs() const { return IS.RetTys.size(); } + unsigned getNumParams() const { return IS.ParamTys.size(); } + CodeGenIntrinsic(Record *R, std::vector DefaultProperties); }; diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/CodeGenTarget.h @@ -164,6 +164,8 @@ return *I->second; } + CodeGenInstruction &getInstructionByName(StringRef Name) const; + /// Returns the number of predefined instructions. static unsigned getNumFixedInstructions(); diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp --- a/llvm/utils/TableGen/CodeGenTarget.cpp +++ b/llvm/utils/TableGen/CodeGenTarget.cpp @@ -499,6 +499,10 @@ #include "llvm/Support/TargetOpcodes.def" nullptr}; +CodeGenInstruction &CodeGenTarget::getInstructionByName(StringRef Name) const { + return getInstruction(Records.getDef(Name)); +} + unsigned CodeGenTarget::getNumFixedInstructions() { return std::size(FixedInstrs) - 1; } diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp --- a/llvm/utils/TableGen/GICombinerEmitter.cpp +++ b/llvm/utils/TableGen/GICombinerEmitter.cpp @@ -11,6 +11,8 @@ /// //===----------------------------------------------------------------------===// +#include "CodeGenInstruction.h" +#include "CodeGenIntrinsics.h" #include "CodeGenTarget.h" #include "GlobalISel/CodeExpander.h" #include "GlobalISel/CodeExpansions.h" @@ -59,6 +61,8 @@ cl::cat(GICombinerEmitterCat)); namespace { +static constexpr StringLiteral IntrinsicIDVarName = "$intrin_id"; + typedef uint64_t RuleID; // We're going to be referencing the same small strings quite a lot for operand @@ -402,53 +406,118 @@ const CodeGenTarget &Target, StringInit *ArgName, const Init &Arg, StringMap> &NamedEdgeDefs, StringMap> &NamedEdgeUses) { - if (const DagInit *Matcher = - getDagWithOperatorOfSubClass(Arg, "Instruction")) { - auto &Instr = - Target.getInstruction(Matcher->getOperatorAsDef(TheDef.getLoc())); - StringRef Name = ArgName ? ArgName->getValue() : ""; + const DagInit *Matcher = getDagWithOperatorOfSubClass(Arg, "Instruction"); + std::optional Intrinsic; - GIMatchDagInstr *N = - MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), - MatchDag.getContext().makeOperandList(Instr)); + if (!Matcher) { + Matcher = getDagWithOperatorOfSubClass(Arg, "Intrinsic"); + if (!Matcher) + return false; - N->setOpcodeAnnotation(&Instr); - const auto &P = MatchDag.addPredicateNode( - makeNameForAnonPredicate(*this), Instr); - MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); - unsigned OpIdx = 0; - for (const auto &NameInit : Matcher->getArgNames()) { - StringRef Name = insertStrTab(NameInit->getAsUnquotedString()); - if (Name.empty()) - continue; - N->assignNameToOperand(OpIdx, Name); - - // Record the endpoints of any named edges. We'll add the cartesian - // product of edges later. - const auto &InstrOperand = N->getOperandInfo()[OpIdx]; - if (InstrOperand.isDef()) { - NamedEdgeDefs.try_emplace(Name); - NamedEdgeDefs[Name].emplace_back(N, &InstrOperand, Matcher); - } else { - NamedEdgeUses.try_emplace(Name); - NamedEdgeUses[Name].emplace_back(N, &InstrOperand, Matcher); - } + Intrinsic.emplace(Matcher->getOperatorAsDef(TheDef.getLoc()), + std::vector{}); + } - if (InstrOperand.isDef()) { - if (any_of(Roots, [&](const RootInfo &X) { - return X.getPatternSymbol() == Name; - })) { - N->setMatchRoot(); - } - } + // Collect all CodeGenInstructions. For non-intrinsics, this always only has + // one instruction. For intrinsics, it has all the variants of G_INTRINSICS + // supported. + SmallVector Instrs; + if (Intrinsic) { + Instrs.push_back(&Target.getInstructionByName("G_INTRINSIC")); + Instrs.push_back( + &Target.getInstructionByName("G_INTRINSIC_W_SIDE_EFFECTS")); + } else { + Instrs.push_back( + &Target.getInstruction(Matcher->getOperatorAsDef(TheDef.getLoc()))); + } + + StringRef Name = ArgName ? ArgName->getValue() : ""; + + auto &MDCtx = MatchDag.getContext(); + const auto &OpList = Intrinsic ? MDCtx.makeIntrinsicPredicateOperandList( + *Intrinsic, IntrinsicIDVarName) + : MDCtx.makeOperandList(*Instrs.front()); + GIMatchDagInstr *N = MatchDag.addInstrNode(makeDebugName(*this, Name), + insertStrTab(Name), OpList); + + N->setOpcodeAnnotation(Instrs.front()); + + StringRef PredName = makeNameForAnonPredicate(*this); + GIMatchDagPredicate *P = nullptr; + + if (Intrinsic) { + // Add all variants of G_INTRINSIC. + auto IntrinPred = MatchDag.addPredicateNode( + *Intrinsic, IntrinsicIDVarName, PredName); + for (const auto *Instr : Instrs) + IntrinPred->addOpcode(Instr); + P = IntrinPred; + } else { + P = MatchDag.addPredicateNode(PredName, + *Instrs.front()); + } + + MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); + + // Check that the number of arguments is correct. + const unsigned NumMatcherArgs = Matcher->getNumArgs(); + const unsigned ExpectedMatcherArgs = + Intrinsic ? (Intrinsic->getNumDefs() + Intrinsic->getNumParams()) + : Instrs.front()->Operands.size(); + if (NumMatcherArgs != ExpectedMatcherArgs) { + if (Intrinsic) + PrintError(TheDef.getLoc(), "intrinsic '" + Intrinsic->Name + + "' expected " + + Twine(ExpectedMatcherArgs) + + " args, got " + Twine(NumMatcherArgs)); + else + PrintError(TheDef.getLoc(), Instrs.front()->TheDef->getName() + + " expected " + + Twine(ExpectedMatcherArgs) + + " args, got " + Twine(NumMatcherArgs)); + return false; + } + + unsigned OpIdx = 0; + for (const auto &NameInit : Matcher->getArgNames()) { + // Insert IntrinsicID operand after the defs. + if (Intrinsic && Intrinsic->getNumDefs() == OpIdx) + N->assignNameToOperand(OpIdx++, IntrinsicIDVarName); - OpIdx++; + StringRef Name = insertStrTab(NameInit->getAsUnquotedString()); + if (Name.empty()) + continue; + N->assignNameToOperand(OpIdx, Name); + + // Record the endpoints of any named edges. We'll add the cartesian + // product of edges later. + const auto &InstrOperand = N->getOperandInfo()[OpIdx]; + if (InstrOperand.isDef()) { + NamedEdgeDefs.try_emplace(Name); + NamedEdgeDefs[Name].emplace_back(N, &InstrOperand, Matcher); + } else { + NamedEdgeUses.try_emplace(Name); + NamedEdgeUses[Name].emplace_back(N, &InstrOperand, Matcher); } - return true; + if (InstrOperand.isDef()) { + if (any_of(Roots, [&](const RootInfo &X) { + return X.getPatternSymbol() == Name; + })) { + N->setMatchRoot(); + } + } + + OpIdx++; } - return false; + + // Insert IntrinsicID operand after the defs (for intrinsics without any + // parameters). + if (Intrinsic && Intrinsic->getNumDefs() == OpIdx) + N->assignNameToOperand(OpIdx++, IntrinsicIDVarName); + + return true; } // Parse the wip_match_opcode placeholder that's temporarily present in lieu of @@ -799,19 +868,21 @@ // partitioner(s) would be sufficiently complete to prevent us from having // untested predicates left over. for (const GIMatchDagPredicate *Predicate : Leaf.untested_predicates()) { - if (Predicate->generateCheckCode(OS, (Indent + " ").str(), - Expansions)) - continue; - PrintError(RuleDef.getLoc(), - "Unable to test predicate used in rule"); - PrintNote(SMLoc(), - "This indicates an incomplete implementation in tablegen"); - Predicate->print(errs()); - errs() << "\n"; - OS << Indent - << "llvm_unreachable(\"TableGen did not emit complete code for this " - "path\");\n"; - break; + OS << Indent << " && (\n"; + if (!Predicate->generateCheckCode(OS, (Indent + " ").str(), + Expansions)) { + PrintError(RuleDef.getLoc(), "Unable to test predicate used in rule"); + PrintNote(SMLoc(), + "This indicates an incomplete implementation in tablegen"); + Predicate->print(errs()); + errs() << "\n"; + OS << Indent + << "llvm_unreachable(\"TableGen did not emit complete code for this " + "path\");\n"; + break; + } + OS << Indent << " )\n"; + continue; } if (Rule->getMatchingFixupCode() && diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h b/llvm/utils/TableGen/GlobalISel/GIMatchDag.h --- a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h +++ b/llvm/utils/TableGen/GlobalISel/GIMatchDag.h @@ -34,6 +34,12 @@ return OperandListCtx.makeMIPredicateOperandList(); } + const GIMatchDagOperandList & + makeIntrinsicPredicateOperandList(const CodeGenIntrinsic &I, + StringRef IntrinsicIDVarName) { + return OperandListCtx.makeIntrinsicPredicateOperandList(I, + IntrinsicIDVarName); + } const GIMatchDagOperandList &makeTwoMOPredicateOperandList() { return OperandListCtx.makeTwoMOPredicateOperandList(); diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h +++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h @@ -15,12 +15,15 @@ #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Support/raw_ostream.h" #include namespace llvm { class CodeGenInstruction; +struct CodeGenIntrinsic; + /// Describes an operand of a MachineInstr w.r.t the DAG Matching. This /// information is derived from CodeGenInstruction::Operands but is more /// readily available for context-less access as we don't need to know which @@ -117,11 +120,20 @@ FoldingSet OperandLists; std::vector> OperandListsOwner; + StringSet<> StrTab; + + StringRef createOperandName(unsigned Idx) { + return StrTab.insert("op" + std::to_string(Idx)).first->first(); + } + public: const GIMatchDagOperandList &makeEmptyOperandList(); const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I); const GIMatchDagOperandList &makeMIPredicateOperandList(); const GIMatchDagOperandList &makeTwoMOPredicateOperandList(); + const GIMatchDagOperandList & + makeIntrinsicPredicateOperandList(const CodeGenIntrinsic &I, + StringRef IntrinsicIDVarName); void print(raw_ostream &OS) const; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp +++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp @@ -9,6 +9,7 @@ #include "GIMatchDagOperands.h" #include "../CodeGenInstruction.h" +#include "../CodeGenIntrinsics.h" using namespace llvm; @@ -116,6 +117,36 @@ return *OperandListsOwner.back().get(); } +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeIntrinsicPredicateOperandList( + const CodeGenIntrinsic &I, StringRef IntrinsicIDVarName) { + FoldingSetNodeID ID; + const unsigned NumOuts = I.getNumDefs(); + const unsigned NumIns = I.getNumParams(); + + for (unsigned i = 0; i < (NumOuts + NumIns + 1); ++i) { + if (i == NumOuts) + GIMatchDagOperand::Profile(ID, i++, IntrinsicIDVarName, false); + GIMatchDagOperand::Profile(ID, i, createOperandName(i), i < NumOuts); + } + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr NewValue = + std::make_unique(); + for (unsigned i = 0; i < (NumOuts + NumIns + 1); ++i) { + if (i == NumOuts) + NewValue->add(IntrinsicIDVarName, i++, false); + NewValue->add(createOperandName(i), i, i < NumOuts); + } + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} const GIMatchDagOperandList & GIMatchDagOperandListContext::makeTwoMOPredicateOperandList() { diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h +++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h @@ -19,6 +19,7 @@ namespace llvm { class CodeExpansions; class CodeGenInstruction; +struct CodeGenIntrinsic; class GIMatchDagOperandList; class GIMatchDagContext; class raw_ostream; @@ -33,6 +34,7 @@ enum GIMatchDagPredicateKind { GIMatchDagPredicateKind_Opcode, GIMatchDagPredicateKind_OneOfOpcodes, + GIMatchDagPredicateKind_Intrinsic, GIMatchDagPredicateKind_SameMO, }; @@ -103,13 +105,20 @@ class GIMatchDagOneOfOpcodesPredicate : public GIMatchDagPredicate { SmallVector Instrs; +protected: + GIMatchDagOneOfOpcodesPredicate(GIMatchDagPredicateKind Kind, + GIMatchDagContext &Ctx, StringRef Name); + public: - GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name); + GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name) + : GIMatchDagOneOfOpcodesPredicate(GIMatchDagPredicateKind_OneOfOpcodes, + Ctx, Name) {} void addOpcode(const CodeGenInstruction *Instr) { Instrs.push_back(Instr); } static bool classof(const GIMatchDagPredicate *P) { - return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes; + return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes || + P->getKind() == GIMatchDagPredicateKind_Intrinsic; } const SmallVectorImpl &getInstrs() const { @@ -123,6 +132,33 @@ #endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) }; +class GIMatchDagIntrinsicPredicate : public GIMatchDagOneOfOpcodesPredicate { + std::string IntrinEnumName; + StringRef IntrinsicIDVarName; + +public: + GIMatchDagIntrinsicPredicate(GIMatchDagContext &Ctx, + const CodeGenIntrinsic &I, + StringRef IntrinsicIDVarName, StringRef Name); + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_Intrinsic; + } + + std::string getIntrinsicEnumName() const { return IntrinEnumName; } + StringRef getIntrinsicIDVarName() const { return IntrinsicIDVarName; } + + void printDescription(raw_ostream &OS) const override; + + virtual bool + generateCheckCode(raw_ostream &OS, StringRef Indent, + const CodeExpansions &Expansions) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + class GIMatchDagSameMOPredicate : public GIMatchDagPredicate { public: GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name); diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp +++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp @@ -11,6 +11,9 @@ #include "llvm/TableGen/Record.h" #include "../CodeGenInstruction.h" +#include "../CodeGenIntrinsics.h" +#include "../CodeGenTarget.h" +#include "CodeExpansions.h" #include "GIMatchDag.h" using namespace llvm; @@ -34,9 +37,8 @@ } GIMatchDagOneOfOpcodesPredicate::GIMatchDagOneOfOpcodesPredicate( - GIMatchDagContext &Ctx, StringRef Name) - : GIMatchDagPredicate(GIMatchDagPredicateKind_OneOfOpcodes, Name, - Ctx.makeMIPredicateOperandList()) {} + GIMatchDagPredicateKind Kind, GIMatchDagContext &Ctx, StringRef Name) + : GIMatchDagPredicate(Kind, Name, Ctx.makeMIPredicateOperandList()) {} void GIMatchDagOneOfOpcodesPredicate::printDescription(raw_ostream &OS) const { OS << "$mi.getOpcode() == oneof("; @@ -48,6 +50,30 @@ OS << ")"; } +GIMatchDagIntrinsicPredicate::GIMatchDagIntrinsicPredicate( + GIMatchDagContext &Ctx, const CodeGenIntrinsic &I, + StringRef IntrinsicIDVarName, StringRef Name) + : GIMatchDagOneOfOpcodesPredicate(GIMatchDagPredicateKind_Intrinsic, Ctx, + Name), + IntrinEnumName(I.EnumName), IntrinsicIDVarName(IntrinsicIDVarName) {} + +void GIMatchDagIntrinsicPredicate::printDescription(raw_ostream &OS) const { + GIMatchDagOneOfOpcodesPredicate::printDescription(OS); + OS << " && " << IntrinsicIDVarName + << ".getIntrinsicID() == Intrinsic::" << IntrinEnumName; +} + +bool GIMatchDagIntrinsicPredicate::generateCheckCode( + raw_ostream &OS, StringRef Indent, const CodeExpansions &Expansions) const { + assert(!getInstrs().empty() && "No G_INTRINSIC opcode added!"); + + const auto IDVar = Expansions.lookup(IntrinsicIDVarName); + assert(!IDVar.empty() && "Expansion of intrinsic ID operand not defined!"); + OS << Indent << IDVar << ".getIntrinsicID() == Intrinsic::" << IntrinEnumName + << "\n"; + return true; +} + GIMatchDagSameMOPredicate::GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name) : GIMatchDagPredicate(GIMatchDagPredicateKind_SameMO, Name, diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp --- a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp +++ b/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp @@ -457,8 +457,15 @@ } for (const CodeGenInstruction *Expected : OpcodesForThisPredicate) { - // Mark this predicate as one we're testing. - TestedPredicatesForLeaf.set(PIdx); + // Mark this predicate as tested, unless it needs additional match code. + + // FIXME: This is needed, but prevents patterns such as + // (match (G_FPTRUNC $fptrunc_dst, $fmed3_dst):$fptrunc, + // (int_amdgcn_fmed3 $fmed3_dst, $src0, $src1, $src2)) + // because G_FPTRUNC would be fully tested, and int_amdgcn_fmed3 becomes + // unreachble. + if (!isa(P)) + TestedPredicatesForLeaf.set(PIdx); // Partitions must be numbered 0, 1, .., N but instructions don't meet // that requirement. Assign a partition number to each opcode if we